From a306f2c601b46eea1997a5a65fff9a16a5f4f6f7 Mon Sep 17 00:00:00 2001 From: Thomas von Dein Date: Tue, 21 Jan 2025 18:37:45 +0100 Subject: [PATCH 1/9] implement multiple regex support and icase and negate flags --- cfg/config.go | 45 +++++++++++++++++++++++++++++++------- cfg/config_test.go | 49 +++++++++++++++++++++++++++++++++++------ cmd/tablizer.go | 47 +++++++++++++++++++++++++--------------- lib/filter.go | 46 ++++++++++++++++++++++++++++++++------- lib/filter_test.go | 24 ++++++++++----------- lib/helpers.go | 18 +++++++++++----- lib/io.go | 20 +++++++++-------- lib/parser.go | 2 +- lib/parser_test.go | 32 ++++++++++++++++----------- tablizer.1 | 54 +++++++++++++++++++++++++++++----------------- tablizer.pod | 48 +++++++++++++++++++++++++---------------- 11 files changed, 267 insertions(+), 118 deletions(-) diff --git a/cfg/config.go b/cfg/config.go index 3d4840a..b7a867d 100644 --- a/cfg/config.go +++ b/cfg/config.go @@ -52,6 +52,12 @@ type Transposer struct { Replace string } +type Pattern struct { + Pattern string + PatternRe *regexp.Regexp + Negate bool +} + // internal config type Config struct { Debug bool @@ -62,8 +68,7 @@ type Config struct { Separator string OutputMode int InvertMatch bool - Pattern string - PatternR *regexp.Regexp + Patterns []*Pattern UseFuzzySearch bool UseHighlight bool @@ -333,15 +338,39 @@ func (conf *Config) ApplyDefaults() { } } -func (conf *Config) PreparePattern(pattern string) error { - PatternR, err := regexp.Compile(pattern) +func (conf *Config) PreparePattern(patterns []*Pattern) error { + // regex checks if a pattern looks like /$pattern/[i!] + flagre := regexp.MustCompile(`^/(.*)/([i!]+)$`) - if err != nil { - return fmt.Errorf("regexp pattern %s is invalid: %w", conf.Pattern, err) + for _, pattern := range patterns { + matches := flagre.FindAllStringSubmatch(pattern.Pattern, -1) + + if matches != nil { + // we have a regex with flags + for _, match := range matches { + pattern.Pattern = match[1] // the inner part is our actual pattern + flags := match[2] // the flags + + for _, flag := range flags { + switch flag { + case 'i': + pattern.Pattern = `(?i)` + pattern.Pattern + case '!': + pattern.Negate = true + } + } + } + } + + PatternRe, err := regexp.Compile(pattern.Pattern) + if err != nil { + return fmt.Errorf("regexp pattern %s is invalid: %w", pattern.Pattern, err) + } + + pattern.PatternRe = PatternRe } - conf.PatternR = PatternR - conf.Pattern = pattern + conf.Patterns = patterns return nil } diff --git a/cfg/config_test.go b/cfg/config_test.go index 84a477f..7b60f79 100644 --- a/cfg/config_test.go +++ b/cfg/config_test.go @@ -79,20 +79,55 @@ func TestPrepareSortFlags(t *testing.T) { func TestPreparePattern(t *testing.T) { var tests = []struct { - pattern string - wanterr bool + patterns []*Pattern + name string + wanterr bool + wanticase bool + wantneg bool }{ - {"[A-Z]+", false}, - {"[a-z", true}, + { + []*Pattern{{Pattern: "[A-Z]+"}}, + "simple", + false, + false, + false, + }, + { + []*Pattern{{Pattern: "[a-z"}}, + "regfail", + true, + false, + false, + }, + { + []*Pattern{{Pattern: "/[A-Z]+/i"}}, + "icase", + false, + true, + false, + }, + { + []*Pattern{{Pattern: "/[A-Z]+/!"}}, + "negate", + false, + false, + true, + }, + { + []*Pattern{{Pattern: "/[A-Z]+/!i"}}, + "negicase", + false, + true, + true, + }, } for _, testdata := range tests { - testname := fmt.Sprintf("PreparePattern-pattern-%s-wanterr-%t", - testdata.pattern, testdata.wanterr) + testname := fmt.Sprintf("PreparePattern-pattern-%s-wanterr-%t", testdata.name, testdata.wanterr) t.Run(testname, func(t *testing.T) { conf := Config{} - err := conf.PreparePattern(testdata.pattern) + err := conf.PreparePattern(testdata.patterns) if err != nil { if !testdata.wanterr { diff --git a/cmd/tablizer.go b/cmd/tablizer.go index 71f5b21..d4768d2 100644 --- a/cmd/tablizer.go +++ b/cmd/tablizer.go @@ -6,7 +6,7 @@ NAME SYNOPSIS Usage: - tablizer [regex] [file, ...] [flags] + tablizer [regex,...] [file, ...] [flags] Operational Flags: -c, --columns string Only show the speficied columns (separated by ,) @@ -130,30 +130,43 @@ DESCRIPTION for the developer. PATTERNS AND FILTERING - You can reduce the rows being displayed by using a regular expression - pattern. The regexp is PCRE compatible, refer to the syntax cheat sheet - here: . If you want to read a - more comprehensive documentation about the topic and have perl installed - you can read it with: + You can reduce the rows being displayed by using one or more regular + expression patterns. The regexp language being used is the one of + GOLANG, refer to the syntax cheat sheet here: + . + + If you want to read a more comprehensive documentation about the topic + and have perl installed you can read it with: perldoc perlre - Or read it online: . + Or read it online: . But please note + that the GO regexp engine does NOT support all perl regex terms, + especially look-ahead and look-behind. - A note on modifiers: the regexp engine used in tablizer uses another - modifier syntax: + If you want to supply flags to a regex, then surround it with slashes + and append the flag. The following flags are supported: - (?MODIFIER) - - The most important modifiers are: - - "i" ignore case "m" multiline mode "s" single line mode + i => case insensitive + ! => negative match Example for a case insensitive search: - kubectl get pods -A | tablizer "(?i)account" + kubectl get pods -A | tablizer "/account/i" - You can use the experimental fuzzy search feature by providing the + If you use the "!" flag, then the regex match will be negated, that is, + if a line in the input matches the given regex, but "!" is supplied, + tablizer will NOT include it in the output. + + For example, here we want to get all lines matching "foo" but not "bar": + + cat table | tablizer foo '/bar/!' + + This would match a line "foo zorro" but not "foo bar". + + The flags can also be combined. + + You can also use the experimental fuzzy search feature by providing the option -z, in which case the pattern is regarded as a fuzzy search term, not a regexp. @@ -392,7 +405,7 @@ AUTHORS var usage = ` Usage: - tablizer [regex] [file, ...] [flags] + tablizer [regex,...] [file, ...] [flags] Operational Flags: -c, --columns string Only show the speficied columns (separated by ,) diff --git a/lib/filter.go b/lib/filter.go index dadbb06..816732d 100644 --- a/lib/filter.go +++ b/lib/filter.go @@ -27,15 +27,42 @@ import ( ) /* - * [!]Match a line, use fuzzy search for normal pattern strings and - * regexp otherwise. - */ +* [!]Match a line, use fuzzy search for normal pattern strings and +* regexp otherwise. + + 'foo bar' foo, /bar/! => false => line contains foo and not (not bar) + 'foo nix' foo, /bar/! => ture => line contains foo and (not bar) + 'foo bar' foo, /bar/ => true => line contains both foo and bar + 'foo nix' foo, /bar/ => false => line does not contain bar + 'foo bar' foo, /nix/ => false => line does not contain nix +*/ func matchPattern(conf cfg.Config, line string) bool { - if conf.UseFuzzySearch { - return fuzzy.MatchFold(conf.Pattern, line) + if len(conf.Patterns) == 0 { + // any line always matches "" + return true } - return conf.PatternR.MatchString(line) + if conf.UseFuzzySearch { + // fuzzy search only considers the 1st pattern + return fuzzy.MatchFold(conf.Patterns[0].Pattern, line) + } + + var match bool + + for _, re := range conf.Patterns { + patmatch := re.PatternRe.MatchString(line) + if re.Negate { + // toggle the meaning of match + patmatch = !patmatch + } + + if match != patmatch { + // toggles match if the last match and current match are different + match = !match + } + } + + return match } /* @@ -123,8 +150,11 @@ func Exists[K comparable, V any](m map[K]V, v K) bool { return false } +/* + * Filters the whole input lines, returns filtered lines + */ func FilterByPattern(conf cfg.Config, input io.Reader) (io.Reader, error) { - if conf.Pattern == "" { + if len(conf.Patterns) == 0 { return input, nil } @@ -136,7 +166,7 @@ func FilterByPattern(conf cfg.Config, input io.Reader) (io.Reader, error) { line := strings.TrimSpace(scanner.Text()) if hadFirst { // don't match 1st line, it's the header - if conf.Pattern != "" && matchPattern(conf, line) == conf.InvertMatch { + if matchPattern(conf, line) == conf.InvertMatch { // by default -v is false, so if a line does NOT // match the pattern, we will ignore it. However, // if the user specified -v, the matching is inverted, diff --git a/lib/filter_test.go b/lib/filter_test.go index e779d31..5562fc0 100644 --- a/lib/filter_test.go +++ b/lib/filter_test.go @@ -27,21 +27,21 @@ import ( func TestMatchPattern(t *testing.T) { var input = []struct { - name string - fuzzy bool - pattern string - line string + name string + fuzzy bool + patterns []*cfg.Pattern + line string }{ { - name: "normal", - pattern: "haus", - line: "hausparty", + name: "normal", + patterns: []*cfg.Pattern{{Pattern: "haus"}}, + line: "hausparty", }, { - name: "fuzzy", - pattern: "hpt", - line: "haus-party-termin", - fuzzy: true, + name: "fuzzy", + patterns: []*cfg.Pattern{{Pattern: "hpt"}}, + line: "haus-party-termin", + fuzzy: true, }, } @@ -55,7 +55,7 @@ func TestMatchPattern(t *testing.T) { conf.UseFuzzySearch = true } - err := conf.PreparePattern(inputdata.pattern) + err := conf.PreparePattern(inputdata.patterns) if err != nil { t.Errorf("PreparePattern returned error: %s", err) } diff --git a/lib/helpers.go b/lib/helpers.go index 32b6705..c2ae9e9 100644 --- a/lib/helpers.go +++ b/lib/helpers.go @@ -293,12 +293,20 @@ func colorizeData(conf cfg.Config, output string) string { return colorized - case len(conf.Pattern) > 0 && !conf.NoColor && color.IsConsole(os.Stdout): - r := regexp.MustCompile("(" + conf.Pattern + ")") + case len(conf.Patterns) > 0 && !conf.NoColor && color.IsConsole(os.Stdout): + out := output - return r.ReplaceAllStringFunc(output, func(in string) string { - return conf.ColorStyle.Sprint(in) - }) + for _, re := range conf.Patterns { + if !re.Negate { + r := regexp.MustCompile("(" + re.Pattern + ")") + + out = r.ReplaceAllStringFunc(out, func(in string) string { + return conf.ColorStyle.Sprint(in) + }) + } + } + + return out default: return output diff --git a/lib/io.go b/lib/io.go index 86954b8..3e39ef8 100644 --- a/lib/io.go +++ b/lib/io.go @@ -29,13 +29,13 @@ import ( const RWRR = 0755 func ProcessFiles(conf *cfg.Config, args []string) error { - fd, pattern, err := determineIO(conf, args) + fd, patterns, err := determineIO(conf, args) if err != nil { return err } - if err := conf.PreparePattern(pattern); err != nil { + if err := conf.PreparePattern(patterns); err != nil { return err } @@ -63,9 +63,9 @@ func ProcessFiles(conf *cfg.Config, args []string) error { return nil } -func determineIO(conf *cfg.Config, args []string) (io.Reader, string, error) { +func determineIO(conf *cfg.Config, args []string) (io.Reader, []*cfg.Pattern, error) { var filehandle io.Reader - var pattern string + var patterns []*cfg.Pattern var haveio bool switch { @@ -76,7 +76,7 @@ func determineIO(conf *cfg.Config, args []string) (io.Reader, string, error) { fd, err := os.OpenFile(conf.InputFile, os.O_RDONLY, RWRR) if err != nil { - return nil, "", fmt.Errorf("failed to read input file %s: %w", conf.InputFile, err) + return nil, nil, fmt.Errorf("failed to read input file %s: %w", conf.InputFile, err) } filehandle = fd @@ -93,13 +93,15 @@ func determineIO(conf *cfg.Config, args []string) (io.Reader, string, error) { } if len(args) > 0 { - pattern = args[0] - conf.Pattern = args[0] + patterns = make([]*cfg.Pattern, len(args)) + for i, arg := range args { + patterns[i] = &cfg.Pattern{Pattern: arg} + } } if !haveio { - return nil, "", errors.New("no file specified and nothing to read on stdin") + return nil, nil, errors.New("no file specified and nothing to read on stdin") } - return filehandle, pattern, nil + return filehandle, patterns, nil } diff --git a/lib/parser.go b/lib/parser.go index 40fe69b..c664bf5 100644 --- a/lib/parser.go +++ b/lib/parser.go @@ -137,7 +137,7 @@ func parseTabular(conf cfg.Config, input io.Reader) (Tabdata, error) { } } else { // data processing - if conf.Pattern != "" && matchPattern(conf, line) == conf.InvertMatch { + if matchPattern(conf, line) == conf.InvertMatch { // by default -v is false, so if a line does NOT // match the pattern, we will ignore it. However, // if the user specified -v, the matching is inverted, diff --git a/lib/parser_test.go b/lib/parser_test.go index e382f59..a9abbe5 100644 --- a/lib/parser_test.go +++ b/lib/parser_test.go @@ -83,36 +83,42 @@ func TestParser(t *testing.T) { func TestParserPatternmatching(t *testing.T) { var tests = []struct { - entries [][]string - pattern string - invert bool - want bool + name string + entries [][]string + patterns []*cfg.Pattern + invert bool + want bool }{ { + name: "match", entries: [][]string{ {"asd", "igig", "cxxxncnc"}, }, - pattern: "ig", - invert: false, + patterns: []*cfg.Pattern{{Pattern: "ig"}}, + invert: false, }, { + name: "invert", entries: [][]string{ {"19191", "EDD 1", "X"}, }, - pattern: "ig", - invert: true, + patterns: []*cfg.Pattern{{Pattern: "ig"}}, + invert: true, }, } for _, inputdata := range input { for _, testdata := range tests { testname := fmt.Sprintf("parse-%s-with-pattern-%s-inverted-%t", - inputdata.name, testdata.pattern, testdata.invert) + inputdata.name, testdata.name, testdata.invert) t.Run(testname, func(t *testing.T) { - conf := cfg.Config{InvertMatch: testdata.invert, Pattern: testdata.pattern, - Separator: inputdata.separator} + conf := cfg.Config{ + InvertMatch: testdata.invert, + Patterns: testdata.patterns, + Separator: inputdata.separator, + } - _ = conf.PreparePattern(testdata.pattern) + _ = conf.PreparePattern(testdata.patterns) readFd := strings.NewReader(strings.TrimSpace(inputdata.text)) gotdata, err := Parse(conf, readFd) @@ -125,7 +131,7 @@ func TestParserPatternmatching(t *testing.T) { } else { if !reflect.DeepEqual(testdata.entries, gotdata.entries) { t.Errorf("Parser returned invalid data (pattern: %s, invert: %t)\nExp: %+v\nGot: %+v\n", - testdata.pattern, testdata.invert, testdata.entries, gotdata.entries) + testdata.name, testdata.invert, testdata.entries, gotdata.entries) } } }) diff --git a/tablizer.1 b/tablizer.1 index 73dd6b4..aa74953 100644 --- a/tablizer.1 +++ b/tablizer.1 @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "TABLIZER 1" -.TH TABLIZER 1 "2025-01-15" "1" "User Commands" +.TH TABLIZER 1 "2025-01-21" "1" "User Commands" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l @@ -144,7 +144,7 @@ tablizer \- Manipulate tabular output of other programs .IX Header "SYNOPSIS" .Vb 2 \& Usage: -\& tablizer [regex] [file, ...] [flags] +\& tablizer [regex,...] [file, ...] [flags] \& \& Operational Flags: \& \-c, \-\-columns string Only show the speficied columns (separated by ,) @@ -278,38 +278,52 @@ Finally the \fB\-d\fR option enables debugging output which is mostly useful for the developer. .SS "\s-1PATTERNS AND FILTERING\s0" .IX Subsection "PATTERNS AND FILTERING" -You can reduce the rows being displayed by using a regular expression -pattern. The regexp is \s-1PCRE\s0 compatible, refer to the syntax cheat -sheet here: . If you want -to read a more comprehensive documentation about the topic and have -perl installed you can read it with: +You can reduce the rows being displayed by using one or more regular +expression patterns. The regexp language being used is the one of +\&\s-1GOLANG,\s0 refer to the syntax cheat sheet here: +. +.PP +If you want to read a more comprehensive documentation about the +topic and have perl installed you can read it with: .PP .Vb 1 \& perldoc perlre .Ve .PP -Or read it online: . +Or read it online: . But please note +that the \s-1GO\s0 regexp engine does \s-1NOT\s0 support all perl regex terms, +especially look-ahead and look-behind. .PP -A note on modifiers: the regexp engine used in tablizer uses another -modifier syntax: +If you want to supply flags to a regex, then surround it with slashes +and append the flag. The following flags are supported: .PP -.Vb 1 -\& (?MODIFIER) +.Vb 2 +\& i => case insensitive +\& ! => negative match .Ve .PP -The most important modifiers are: -.PP -\&\f(CW\*(C`i\*(C'\fR ignore case -\&\f(CW\*(C`m\*(C'\fR multiline mode -\&\f(CW\*(C`s\*(C'\fR single line mode -.PP Example for a case insensitive search: .PP .Vb 1 -\& kubectl get pods \-A | tablizer "(?i)account" +\& kubectl get pods \-A | tablizer "/account/i" .Ve .PP -You can use the experimental fuzzy search feature by providing the +If you use the \f(CW\*(C`!\*(C'\fR flag, then the regex match will be negated, that +is, if a line in the input matches the given regex, but \f(CW\*(C`!\*(C'\fR is +supplied, tablizer will \s-1NOT\s0 include it in the output. +.PP +For example, here we want to get all lines matching \*(L"foo\*(R" but not +\&\*(L"bar\*(R": +.PP +.Vb 1 +\& cat table | tablizer foo \*(Aq/bar/!\*(Aq +.Ve +.PP +This would match a line \*(L"foo zorro\*(R" but not \*(L"foo bar\*(R". +.PP +The flags can also be combined. +.PP +You can also use the experimental fuzzy search feature by providing the option \fB\-z\fR, in which case the pattern is regarded as a fuzzy search term, not a regexp. .PP diff --git a/tablizer.pod b/tablizer.pod index 8f170ae..11846ea 100644 --- a/tablizer.pod +++ b/tablizer.pod @@ -5,7 +5,7 @@ tablizer - Manipulate tabular output of other programs =head1 SYNOPSIS Usage: - tablizer [regex] [file, ...] [flags] + tablizer [regex,...] [file, ...] [flags] Operational Flags: -c, --columns string Only show the speficied columns (separated by ,) @@ -142,32 +142,44 @@ useful for the developer. =head2 PATTERNS AND FILTERING -You can reduce the rows being displayed by using a regular expression -pattern. The regexp is PCRE compatible, refer to the syntax cheat -sheet here: L. If you want -to read a more comprehensive documentation about the topic and have -perl installed you can read it with: +You can reduce the rows being displayed by using one or more regular +expression patterns. The regexp language being used is the one of +GOLANG, refer to the syntax cheat sheet here: +L. + +If you want to read a more comprehensive documentation about the +topic and have perl installed you can read it with: perldoc perlre -Or read it online: L. +Or read it online: L. But please note +that the GO regexp engine does NOT support all perl regex terms, +especially look-ahead and look-behind. -A note on modifiers: the regexp engine used in tablizer uses another -modifier syntax: +If you want to supply flags to a regex, then surround it with slashes +and append the flag. The following flags are supported: - (?MODIFIER) - -The most important modifiers are: - -C ignore case -C multiline mode -C single line mode + i => case insensitive + ! => negative match Example for a case insensitive search: - kubectl get pods -A | tablizer "(?i)account" + kubectl get pods -A | tablizer "/account/i" -You can use the experimental fuzzy search feature by providing the +If you use the C flag, then the regex match will be negated, that +is, if a line in the input matches the given regex, but C is +supplied, tablizer will NOT include it in the output. + +For example, here we want to get all lines matching "foo" but not +"bar": + + cat table | tablizer foo '/bar/!' + +This would match a line "foo zorro" but not "foo bar". + +The flags can also be combined. + +You can also use the experimental fuzzy search feature by providing the option B<-z>, in which case the pattern is regarded as a fuzzy search term, not a regexp. From ea3dd75fecd93ec693f5abf64235dc24c3a30b60 Mon Sep 17 00:00:00 2001 From: Thomas von Dein Date: Tue, 21 Jan 2025 18:43:56 +0100 Subject: [PATCH 2/9] fix linting error --- cfg/config.go | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/cfg/config.go b/cfg/config.go index b7a867d..93376a7 100644 --- a/cfg/config.go +++ b/cfg/config.go @@ -345,19 +345,17 @@ func (conf *Config) PreparePattern(patterns []*Pattern) error { for _, pattern := range patterns { matches := flagre.FindAllStringSubmatch(pattern.Pattern, -1) - if matches != nil { - // we have a regex with flags - for _, match := range matches { - pattern.Pattern = match[1] // the inner part is our actual pattern - flags := match[2] // the flags + // we have a regex with flags + for _, match := range matches { + pattern.Pattern = match[1] // the inner part is our actual pattern + flags := match[2] // the flags - for _, flag := range flags { - switch flag { - case 'i': - pattern.Pattern = `(?i)` + pattern.Pattern - case '!': - pattern.Negate = true - } + for _, flag := range flags { + switch flag { + case 'i': + pattern.Pattern = `(?i)` + pattern.Pattern + case '!': + pattern.Negate = true } } } From 1593799c036f84f8dc497c0e2bc197050bfb48fb Mon Sep 17 00:00:00 2001 From: Thomas von Dein Date: Wed, 22 Jan 2025 10:47:42 +0100 Subject: [PATCH 3/9] added multi pattern tests --- t/test-multipatterns.txtar | 46 ++++++++++++++++++++++++++++++++++++++ t/testtable5 | 6 +++++ 2 files changed, 52 insertions(+) create mode 100644 t/test-multipatterns.txtar create mode 100644 t/testtable5 diff --git a/t/test-multipatterns.txtar b/t/test-multipatterns.txtar new file mode 100644 index 0000000..90852a1 --- /dev/null +++ b/t/test-multipatterns.txtar @@ -0,0 +1,46 @@ +# filtering + +# a AND b +exec tablizer -r testtable.txt -H -cspecies invasive imperium +stdout 'namak' +! stdout human + +# a AND !b +exec tablizer -r testtable.txt -H -cspecies invasive '/imperium/!' +stdout 'human' +! stdout namak + +# a AND !b AND c +exec tablizer -r testtable.txt -H -cspecies peaceful '/imperium/!' planetary +stdout 'kenaha' +! stdout 'namak|heduu|riedl' + +# case insensitive +exec tablizer -r testtable.txt -H -cspecies '/REGIONAL/i' +stdout namak +! stdout 'human|riedl|heduu|kenaa' + +# case insensitive negated +exec tablizer -r testtable.txt -H -cspecies '/REGIONAL/!i' +stdout 'human|riedl|heduu|kenaa' +! stdout namak + +# !a AND !b +exec tablizer -r testtable.txt -H -cspecies '/galactic/!' '/planetary/!' +stdout namak +! stdout 'human|riedl|heduu|kenaa' + +# same case insensitive +exec tablizer -r testtable.txt -H -cspecies '/GALACTIC/i!' '/PLANETARY/!i' +stdout namak +! stdout 'human|riedl|heduu|kenaa' + +# will be automatically created in work dir +-- testtable.txt -- +SPECIES TYPE HOME STAGE SPREAD +human invasive earth brink planetary +riedl peaceful keauna civilized pangalactic +namak invasive namak imperium regional +heduu peaceful iu imperium galactic +kenaha peaceful kohi hunter-gatherer planetary + diff --git a/t/testtable5 b/t/testtable5 new file mode 100644 index 0000000..f8f9eb3 --- /dev/null +++ b/t/testtable5 @@ -0,0 +1,6 @@ +SPECIES TYPE HOME STAGE +human invasive earth brink +riedl peaceful keauna civilized +namak invasive namak imperium +heduu peaceful iu imperium +kenaha peaceful kohi hunter-gatherer From 6566dd66f0b55667915cdce8d53e18f50c1e7284 Mon Sep 17 00:00:00 2001 From: Thomas von Dein Date: Wed, 22 Jan 2025 10:48:06 +0100 Subject: [PATCH 4/9] fixed pattern regex, fixed pattern AND operation --- cfg/config.go | 2 +- lib/filter.go | 14 +++++++++----- tablizer.1 | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/cfg/config.go b/cfg/config.go index 93376a7..fbb9917 100644 --- a/cfg/config.go +++ b/cfg/config.go @@ -340,7 +340,7 @@ func (conf *Config) ApplyDefaults() { func (conf *Config) PreparePattern(patterns []*Pattern) error { // regex checks if a pattern looks like /$pattern/[i!] - flagre := regexp.MustCompile(`^/(.*)/([i!]+)$`) + flagre := regexp.MustCompile(`^/(.*)/([i!]*)$`) for _, pattern := range patterns { matches := flagre.FindAllStringSubmatch(pattern.Pattern, -1) diff --git a/lib/filter.go b/lib/filter.go index 816732d..8227e23 100644 --- a/lib/filter.go +++ b/lib/filter.go @@ -47,8 +47,9 @@ func matchPattern(conf cfg.Config, line string) bool { return fuzzy.MatchFold(conf.Patterns[0].Pattern, line) } - var match bool + var match int + //fmt.Printf("<%s>\n", line) for _, re := range conf.Patterns { patmatch := re.PatternRe.MatchString(line) if re.Negate { @@ -56,13 +57,16 @@ func matchPattern(conf cfg.Config, line string) bool { patmatch = !patmatch } - if match != patmatch { - // toggles match if the last match and current match are different - match = !match + if patmatch { + match++ } + + //fmt.Printf("patmatch: %t, match: %d, pattern: %s, negate: %t\n", patmatch, match, re.Pattern, re.Negate) } - return match + // fmt.Printf("result: %t\n", match == len(conf.Patterns)) + //fmt.Println() + return match == len(conf.Patterns) } /* diff --git a/tablizer.1 b/tablizer.1 index aa74953..d4d21ec 100644 --- a/tablizer.1 +++ b/tablizer.1 @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "TABLIZER 1" -.TH TABLIZER 1 "2025-01-21" "1" "User Commands" +.TH TABLIZER 1 "2025-01-22" "1" "User Commands" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l From 2c08687c2903813c27558ff89bc2e12745888d4c Mon Sep 17 00:00:00 2001 From: Thomas von Dein Date: Thu, 30 Jan 2025 17:30:02 +0100 Subject: [PATCH 5/9] add support for negative filters (-F field!=regex) --- README.md | 2 +- cfg/config.go | 28 +++++++++++++++++++++------- cmd/root.go | 4 ++-- cmd/tablizer.go | 8 ++++++-- lib/filter.go | 14 +++++++++----- lib/filter_test.go | 16 +++++++++++++++- tablizer.1 | 10 ++++++++-- tablizer.pod | 6 +++++- 8 files changed, 67 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index a66a97d..31888b7 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Operational Flags: -s, --separator string Custom field separator -k, --sort-by int Sort by column (default: 1) -z, --fuzzy Use fuzzy search [experimental] - -F, --filter field=reg Filter given field with regex, can be used multiple times + -F, --filter field[!]=reg Filter given field with regex, can be used multiple times -T, --transpose-columns string Transpose the speficied columns (separated by ,) -R, --regex-transposer /from/to/ Apply /search/replace/ regexp to fields given in -T diff --git a/cfg/config.go b/cfg/config.go index fbb9917..172012b 100644 --- a/cfg/config.go +++ b/cfg/config.go @@ -1,5 +1,5 @@ /* -Copyright © 2022-2024 Thomas von Dein +Copyright © 2022-2025 Thomas von Dein This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -58,6 +58,11 @@ type Pattern struct { Negate bool } +type Filter struct { + Regex *regexp.Regexp + Negate bool +} + // internal config type Config struct { Debug bool @@ -100,7 +105,7 @@ type Config struct { // used for field filtering Rawfilters []string - Filters map[string]*regexp.Regexp + Filters map[string]Filter //map[string]*regexp.Regexp // -r InputFile string @@ -270,12 +275,20 @@ func (conf *Config) PrepareModeFlags(flag Modeflag) { } func (conf *Config) PrepareFilters() error { - conf.Filters = make(map[string]*regexp.Regexp, len(conf.Rawfilters)) + conf.Filters = make(map[string]Filter, len(conf.Rawfilters)) - for _, filter := range conf.Rawfilters { - parts := strings.Split(filter, "=") + for _, rawfilter := range conf.Rawfilters { + filter := Filter{} + + parts := strings.Split(rawfilter, "!=") if len(parts) != MAXPARTS { - return errors.New("filter field and value must be separated by =") + parts = strings.Split(rawfilter, "=") + + if len(parts) != MAXPARTS { + return errors.New("filter field and value must be separated by '=' or '!='") + } + } else { + filter.Negate = true } reg, err := regexp.Compile(parts[1]) @@ -284,7 +297,8 @@ func (conf *Config) PrepareFilters() error { parts[0], err) } - conf.Filters[strings.ToLower(strings.ToLower(parts[0]))] = reg + filter.Regex = reg + conf.Filters[strings.ToLower(parts[0])] = filter } return nil diff --git a/cmd/root.go b/cmd/root.go index afbf0df..1d347db 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,5 +1,5 @@ /* -Copyright © 2022-2024 Thomas von Dein +Copyright © 2022-2025 Thomas von Dein This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -190,7 +190,7 @@ func Execute() { // filters rootCmd.PersistentFlags().StringArrayVarP(&conf.Rawfilters, - "filter", "F", nil, "Filter by field (field=regexp)") + "filter", "F", nil, "Filter by field (field=regexp || field!=regexp)") rootCmd.PersistentFlags().StringArrayVarP(&conf.Transposers, "regex-transposer", "R", nil, "apply /search/replace/ regexp to fields given in -T") diff --git a/cmd/tablizer.go b/cmd/tablizer.go index d4768d2..7d6504b 100644 --- a/cmd/tablizer.go +++ b/cmd/tablizer.go @@ -17,7 +17,7 @@ SYNOPSIS -s, --separator string Custom field separator -k, --sort-by int|name Sort by column (default: 1) -z, --fuzzy Use fuzzy search [experimental] - -F, --filter field=reg Filter given field with regex, can be used multiple times + -F, --filter field[!]=reg Filter given field with regex, can be used multiple times -T, --transpose-columns string Transpose the speficied columns (separated by ,) -R, --regex-transposer /from/to/ Apply /search/replace/ regexp to fields given in -T @@ -181,6 +181,10 @@ DESCRIPTION If you specify more than one filter, both filters have to match (AND operation). + These field filters can also be negated: + + fieldname!=regexp + If the option -v is specified, the filtering is inverted. COLUMNS @@ -416,7 +420,7 @@ Operational Flags: -s, --separator string Custom field separator -k, --sort-by int|name Sort by column (default: 1) -z, --fuzzy Use fuzzy search [experimental] - -F, --filter field=reg Filter given field with regex, can be used multiple times + -F, --filter field[!]=reg Filter given field with regex, can be used multiple times -T, --transpose-columns string Transpose the speficied columns (separated by ,) -R, --regex-transposer /from/to/ Apply /search/replace/ regexp to fields given in -T diff --git a/lib/filter.go b/lib/filter.go index 8227e23..8b6271e 100644 --- a/lib/filter.go +++ b/lib/filter.go @@ -1,5 +1,5 @@ /* -Copyright © 2022-2024 Thomas von Dein +Copyright © 2022-2025 Thomas von Dein This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -86,15 +86,19 @@ func FilterByFields(conf cfg.Config, data *Tabdata) (*Tabdata, bool, error) { keep := true for idx, header := range data.headers { - if !Exists(conf.Filters, strings.ToLower(header)) { + lcheader := strings.ToLower(header) + if !Exists(conf.Filters, lcheader) { // do not filter by unspecified field continue } - if !conf.Filters[strings.ToLower(header)].MatchString(row[idx]) { - // there IS a filter, but it doesn't match - keep = false + match := conf.Filters[lcheader].Regex.MatchString(row[idx]) + if conf.Filters[lcheader].Negate { + match = !match + } + if !match { + keep = false break } } diff --git a/lib/filter_test.go b/lib/filter_test.go index 5562fc0..7e5381d 100644 --- a/lib/filter_test.go +++ b/lib/filter_test.go @@ -1,5 +1,5 @@ /* -Copyright © 2024 Thomas von Dein +Copyright © 2024-2025 Thomas von Dein This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -98,6 +98,20 @@ func TestFilterByFields(t *testing.T) { }, }, + { + name: "one-field-negative", + filter: []string{"one!=asd"}, + expect: Tabdata{ + headers: []string{ + "ONE", "TWO", "THREE", + }, + entries: [][]string{ + {"19191", "EDD 1", "x"}, + {"8d8", "AN 1", "y"}, + }, + }, + }, + { name: "one-field-inverted", filter: []string{"one=19"}, diff --git a/tablizer.1 b/tablizer.1 index d4d21ec..4bc9cb1 100644 --- a/tablizer.1 +++ b/tablizer.1 @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "TABLIZER 1" -.TH TABLIZER 1 "2025-01-22" "1" "User Commands" +.TH TABLIZER 1 "2025-01-30" "1" "User Commands" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l @@ -155,7 +155,7 @@ tablizer \- Manipulate tabular output of other programs \& \-s, \-\-separator string Custom field separator \& \-k, \-\-sort\-by int|name Sort by column (default: 1) \& \-z, \-\-fuzzy Use fuzzy search [experimental] -\& \-F, \-\-filter field=reg Filter given field with regex, can be used multiple times +\& \-F, \-\-filter field[!]=reg Filter given field with regex, can be used multiple times \& \-T, \-\-transpose\-columns string Transpose the speficied columns (separated by ,) \& \-R, \-\-regex\-transposer /from/to/ Apply /search/replace/ regexp to fields given in \-T \& @@ -340,6 +340,12 @@ Fieldnames (== columns headers) are case insensitive. If you specify more than one filter, both filters have to match (\s-1AND\s0 operation). .PP +These field filters can also be negated: +.PP +.Vb 1 +\& fieldname!=regexp +.Ve +.PP If the option \fB\-v\fR is specified, the filtering is inverted. .SS "\s-1COLUMNS\s0" .IX Subsection "COLUMNS" diff --git a/tablizer.pod b/tablizer.pod index 11846ea..5ebfc3d 100644 --- a/tablizer.pod +++ b/tablizer.pod @@ -16,7 +16,7 @@ tablizer - Manipulate tabular output of other programs -s, --separator string Custom field separator -k, --sort-by int|name Sort by column (default: 1) -z, --fuzzy Use fuzzy search [experimental] - -F, --filter field=reg Filter given field with regex, can be used multiple times + -F, --filter field[!]=reg Filter given field with regex, can be used multiple times -T, --transpose-columns string Transpose the speficied columns (separated by ,) -R, --regex-transposer /from/to/ Apply /search/replace/ regexp to fields given in -T @@ -194,6 +194,10 @@ Fieldnames (== columns headers) are case insensitive. If you specify more than one filter, both filters have to match (AND operation). +These field filters can also be negated: + + fieldname!=regexp + If the option B<-v> is specified, the filtering is inverted. From a455f6b79a9e5ea88ed73a8320f339224f9243b9 Mon Sep 17 00:00:00 2001 From: Thomas von Dein Date: Thu, 30 Jan 2025 17:31:56 +0100 Subject: [PATCH 6/9] bump version --- cfg/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cfg/config.go b/cfg/config.go index 172012b..93f026d 100644 --- a/cfg/config.go +++ b/cfg/config.go @@ -28,7 +28,7 @@ import ( ) const DefaultSeparator string = `(\s\s+|\t)` -const Version string = "v1.3.1" +const Version string = "v1.3.2" const MAXPARTS = 2 var DefaultConfigfile = os.Getenv("HOME") + "/.config/tablizer/config" From 3949411c5759c649f992253c40ffe8260e0654d9 Mon Sep 17 00:00:00 2001 From: Thomas von Dein Date: Wed, 5 Feb 2025 17:51:14 +0100 Subject: [PATCH 7/9] add change log generator, update release builder --- .github/workflows/release.yaml | 63 +++++++++++++++++++++++++++++++--- 1 file changed, 59 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ab0e8c5..493ff04 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,8 +1,8 @@ -name: build-and-test +name: build-release on: push: tags: - - "*" + - "v*.*.*" jobs: release: @@ -10,10 +10,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up Go - uses: actions/setup-go@v1 + uses: actions/setup-go@v5 with: go-version: 1.22.11 @@ -30,3 +30,58 @@ jobs: tag: ${{ github.ref_name }} file: ./releases/* file_glob: true + + - name: Build Changelog + id: github_release + uses: mikepenz/release-changelog-builder-action@v5 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + mode: "PR" + configurationJson: | + { + "template": "#{{CHANGELOG}}\n\n**Full Changelog**: #{{RELEASE_DIFF}}", + "pr_template": "- #{{TITLE}} (##{{NUMBER}}) by #{{AUTHOR}}\n#{{BODY}}", + "empty_template": "- no changes", + "categories": [ + { + "title": "## New Features", + "labels": ["add", "feature"] + }, + { + "title": "## Bug Fixes", + "labels": ["fix", "bug", "revert"] + }, + { + "title": "## Documentation Enhancements", + "labels": ["doc"] + }, + { + "title": "## Refactoring Efforts", + "labels": ["refactor"] + }, + { + "title": "## Miscellaneus Changes", + "labels": [] + } + ], + "ignore_labels": [ + "duplicate", "good first issue", "help wanted", "invalid", "question", "wontfix" + ], + "label_extractor": [ + { + "pattern": "(.) (.+)", + "target": "$1" + }, + { + "pattern": "(.) (.+)", + "target": "$1", + "on_property": "title" + } + ] + } + + - name: Create Release + uses: softprops/action-gh-release@v2 + with: + body: ${{steps.github_release.outputs.changelog}} From 74d82fa356e07ec03b896f132cb31d0bb33d0750 Mon Sep 17 00:00:00 2001 From: Thomas von Dein Date: Wed, 12 Feb 2025 14:04:40 +0100 Subject: [PATCH 8/9] fix ci tests on windows: make clean before running test --- .github/workflows/ci.yaml | 4 +--- Makefile | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1624835..e56e0ca 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -5,9 +5,7 @@ jobs: strategy: matrix: version: ['1.22'] - # windows-latest removed, see: - # https://github.com/rogpeppe/go-internal/issues/284 - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest, macos-latest, windows-latest] name: Build runs-on: ${{ matrix.os }} steps: diff --git a/Makefile b/Makefile index 2870e99..5ce09dd 100644 --- a/Makefile +++ b/Makefile @@ -64,7 +64,7 @@ install: buildlocal clean: rm -rf $(tool) releases coverage.out -test: +test: clean go test ./... $(OPTS) singletest: From 6b659773f105d0f9efa7fa4a259c1b6b3492f2e4 Mon Sep 17 00:00:00 2001 From: Thomas von Dein Date: Wed, 19 Feb 2025 18:09:05 +0100 Subject: [PATCH 9/9] build release bins w/o symbols and debug, +static --- mkrel.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mkrel.sh b/mkrel.sh index 894deaf..ec20ca1 100755 --- a/mkrel.sh +++ b/mkrel.sh @@ -42,8 +42,15 @@ for D in $DIST; do binfile="releases/${tool}-${os}-${arch}-${version}" tardir="${tool}-${os}-${arch}-${version}" tarfile="releases/${tool}-${os}-${arch}-${version}.tar.gz" + pie="" + + if test "$D" = "linux/amd64"; then + pie="-buildmode=pie" + fi + set -x - GOOS=${os} GOARCH=${arch} go build -o ${binfile} -ldflags "-X 'github.com/tlinden/tablizer/cfg.VERSION=${version}'" + GOOS=${os} GOARCH=${arch} go build -tags osusergo,netgo -ldflags "-extldflags=-static -w -X 'github.com/tlinden/tablizer/cfg.VERSION=${version}'" --trimpath $pie -o ${binfile} + strip --strip-all ${binfile} mkdir -p ${tardir} cp ${binfile} README.md LICENSE ${tardir}/ echo 'tool = tablizer