use non format logger

* add color table support (using alternating colorization of rows) using new flag `-L`
* add config file support (HCL format) using `~/.config/tablizer/config` or `-f <file>` so the user can customize colors
* removed golang 1.17 support

.github/ISSUE_TEMPLATE/bug_report.md

@@ -7,7 +7,7 @@ assignees: TLINDEN
 
 ---
 
-**Describtion**
+**Description**
 <!-- Please provide a clear and concise description of the issue: -->
 
 

.github/ISSUE_TEMPLATE/feature_request.md

@@ -7,7 +7,7 @@ assignees: TLINDEN
 
 ---
 
-**Describtion**
+**Description**
 <!-- Please provide a clear and concise description of the feature you desire: -->
 
 

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "monthly"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+        interval: "monthly"

.github/workflows/ci.yaml

@@ -4,7 +4,7 @@ jobs:
   build:
     strategy:
       matrix:
-        version: [1.17, 1.18, 1.19]
+        version: [1.18, 1.19]
         os: [ubuntu-latest, windows-latest, macos-latest]
     name: Build
     runs-on: ${{ matrix.os }}
@@ -30,29 +30,9 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.17
+          go-version: 1.18
       - uses: actions/checkout@v3
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
-        #with:
+        with:
-          # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
+          skip-cache: true
-          # version: v1.29
-
-          # Optional: working directory, useful for monorepos
-          # working-directory: somedir
-
-          # Optional: golangci-lint command line arguments.
-          # args: --issues-exit-code=0
-
-          # Optional: show only new issues if it's a pull request. The default value is `false`.
-          # only-new-issues: true
-
-          # Optional: if set to true then the all caching functionality will be complete disabled,
-          #           takes precedence over all other caching options.
-          # skip-cache: true
-
-          # Optional: if set to true then the action don't cache or restore ~/go/pkg.
-          # skip-pkg-cache: true
-
-          # Optional: if set to true then the action don't cache or restore ~/.cache/go-build.
-          # skip-build-cache: true

.gitignore

@@ -0,0 +1,2 @@
+releases
+tablizer

Makefile

@@ -91,3 +91,11 @@ show-versions: buildlocal
 
 goupdate:
 	go get -t -u=patch ./...
+
+lint:
+	golangci-lint run
+
+# keep til ireturn
+lint-full:
+	golangci-lint run --enable-all --exclude-use-default --disable exhaustivestruct,exhaustruct,depguard,interfacer,deadcode,golint,structcheck,scopelint,varcheck,ifshort,maligned,nosnakecase,godot,funlen,gofumpt,cyclop,noctx,gochecknoglobals,paralleltest,forbidigo,gci,godox,goimports,ireturn,stylecheck,testpackage,mirror,nestif,revive,goerr113,gomnd
+	gocritic check -enableAll *.go

README.md

@@ -70,6 +70,17 @@ NAME(1)                      READY(2) STATUS(3)  RESTARTS(4)    AGE(5)
 repldepl-7bcd8d5b64-q2bf4    1/1      Running    1 (69m ago)    5h26m
 ```
 
+Sometimes a filter regex is to broad  and you wish to filter only on a
+particular column. This is possible using `-F`:
+```
+% kubectl get pods | tablizer -n -Fname=2
+NAME                            READY   STATUS  RESTARTS        AGE
+repldepl-7bcd8d5b64-q2bf4       1/1     Running 1 (69m ago)     5h26m
+```
+
+Here we filtered  the `NAME` column for `2`, which  would have matched
+otherwise on all rows.
+
 There are more output modes like org-mode (orgtbl) and markdown.
 
 ## Demo

TODO.md

@@ -6,4 +6,13 @@
 
 - add --no-headers option
 
+### Lisp Plugin Infrastructure using zygo
 
+Hooks:
+
+| Filter    | Purpose                                                     | Args                | Return |
+|-----------|-------------------------------------------------------------|---------------------|--------|
+| filter    | include or exclude lines                                    | row as hash         | bool   |
+| process   | do calculations with data, store results in global lisp env | whole dataset       | nil    |
+| transpose | modify a cell                                               | headername and cell | cell   |
+| append    | add one or more rows to the dataset (use this to add stats) | nil                 | rows   |

cfg/config.go

@@ -1,5 +1,5 @@
 /*
-Copyright © 2022 Thomas von Dein
+Copyright © 2022-2024 Thomas von Dein
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -19,27 +19,50 @@ package cfg
 import (
 	"errors"
 	"fmt"
-	"github.com/gookit/color"
 	"os"
 	"regexp"
+	"strings"
+
+	"github.com/glycerine/zygomys/zygo"
+	"github.com/gookit/color"
+	"github.com/hashicorp/hcl/v2/hclsimple"
 )
 
 const DefaultSeparator string = `(\s\s+|\t)`
-const Version string = "v1.0.15"
+const Version string = "v1.2.0"
+const MAXPARTS = 2
+
+var DefaultLoadPath = os.Getenv("HOME") + "/.config/tablizer/lisp"
+var DefaultConfigfile = os.Getenv("HOME") + "/.config/tablizer/config"
 
 var VERSION string // maintained by -x
 
+// public config, set via config file or using defaults
+type Settings struct {
+	FG             string `hcl:"FG"`
+	BG             string `hcl:"BG"`
+	HighlightFG    string `hcl:"HighlightFG"`
+	HighlightBG    string `hcl:"HighlightBG"`
+	NoHighlightFG  string `hcl:"NoHighlightFG"`
+	NoHighlightBG  string `hcl:"NoHighlightBG"`
+	HighlightHdrFG string `hcl:"HighlightHdrFG"`
+	HighlightHdrBG string `hcl:"HighlightHdrBG"`
+}
+
+// internal config
 type Config struct {
-	Debug       bool
+	Debug          bool
-	NoNumbering bool
+	NoNumbering    bool
-	NoHeaders   bool
+	NoHeaders      bool
-	Columns     string
+	Columns        string
-	UseColumns  []int
+	UseColumns     []int
-	Separator   string
+	Separator      string
-	OutputMode  int
+	OutputMode     int
-	InvertMatch bool
+	InvertMatch    bool
-	Pattern     string
+	Pattern        string
-	PatternR    *regexp.Regexp
+	PatternR       *regexp.Regexp
+	UseFuzzySearch bool
+	UseHighlight   bool
 
 	SortMode       string
 	SortDescending bool
@@ -49,9 +72,28 @@ type Config struct {
 	 FIXME: make configurable somehow, config file or ENV
 	 see https://github.com/gookit/color.
 	*/
-	ColorStyle color.Style
+	ColorStyle        color.Style
+	HighlightStyle    color.Style
+	NoHighlightStyle  color.Style
+	HighlightHdrStyle color.Style
 
 	NoColor bool
+
+	// special  case: we use the  config struct to transport  the lisp
+	// env trough the program
+	Lisp *zygo.Zlisp
+
+	// a path containing lisp scripts to be loaded on startup
+	LispLoadPath string
+
+	// config file, optional
+	Configfile string
+
+	Settings Settings
+
+	// used for field filtering
+	Rawfilters []string
+	Filters    map[string]*regexp.Regexp
 }
 
 // maps outputmode short flags to output mode, ie. -O => -o orgtbl
@@ -73,7 +115,7 @@ const (
 	Shell
 	Yaml
 	CSV
-	Ascii
+	ASCII
 )
 
 // various sort types
@@ -83,50 +125,108 @@ type Sortmode struct {
 	Age     bool
 }
 
+// valid lisp hooks
+var ValidHooks []string
+
 // default color schemes
-func Colors() map[color.Level]map[string]color.Color {
+func (conf *Config) Colors() map[color.Level]map[string]color.Color {
-	return map[color.Level]map[string]color.Color{
+	colors := map[color.Level]map[string]color.Color{
 		color.Level16: {
-			"bg": color.BgGreen, "fg": color.FgBlack,
+			"bg": color.BgGreen, "fg": color.FgWhite,
+			"hlbg": color.BgGray, "hlfg": color.FgWhite,
 		},
 		color.Level256: {
-			"bg": color.BgLightGreen, "fg": color.FgBlack,
+			"bg": color.BgLightGreen, "fg": color.FgWhite,
+			"hlbg": color.BgLightBlue, "hlfg": color.FgWhite,
 		},
 		color.LevelRgb: {
-			// FIXME: maybe use something nicer
+			"bg": color.BgLightGreen, "fg": color.FgWhite,
-			"bg": color.BgLightGreen, "fg": color.FgBlack,
+			"hlbg": color.BgHiGreen, "hlfg": color.FgWhite,
+			"nohlbg": color.BgWhite, "nohlfg": color.FgLightGreen,
+			"hdrbg": color.BgBlue, "hdrfg": color.FgWhite,
 		},
 	}
+
+	if len(conf.Settings.BG) > 0 {
+		colors[color.Level16]["bg"] = ColorStringToBGColor(conf.Settings.BG)
+		colors[color.Level256]["bg"] = ColorStringToBGColor(conf.Settings.BG)
+		colors[color.LevelRgb]["bg"] = ColorStringToBGColor(conf.Settings.BG)
+	}
+
+	if len(conf.Settings.FG) > 0 {
+		colors[color.Level16]["fg"] = ColorStringToColor(conf.Settings.FG)
+		colors[color.Level256]["fg"] = ColorStringToColor(conf.Settings.FG)
+		colors[color.LevelRgb]["fg"] = ColorStringToColor(conf.Settings.FG)
+	}
+
+	if len(conf.Settings.HighlightBG) > 0 {
+		colors[color.Level16]["hlbg"] = ColorStringToBGColor(conf.Settings.HighlightBG)
+		colors[color.Level256]["hlbg"] = ColorStringToBGColor(conf.Settings.HighlightBG)
+		colors[color.LevelRgb]["hlbg"] = ColorStringToBGColor(conf.Settings.HighlightBG)
+	}
+
+	if len(conf.Settings.HighlightFG) > 0 {
+		colors[color.Level16]["hlfg"] = ColorStringToColor(conf.Settings.HighlightFG)
+		colors[color.Level256]["hlfg"] = ColorStringToColor(conf.Settings.HighlightFG)
+		colors[color.LevelRgb]["hlfg"] = ColorStringToColor(conf.Settings.HighlightFG)
+	}
+
+	if len(conf.Settings.NoHighlightBG) > 0 {
+		colors[color.Level16]["nohlbg"] = ColorStringToBGColor(conf.Settings.NoHighlightBG)
+		colors[color.Level256]["nohlbg"] = ColorStringToBGColor(conf.Settings.NoHighlightBG)
+		colors[color.LevelRgb]["nohlbg"] = ColorStringToBGColor(conf.Settings.NoHighlightBG)
+	}
+
+	if len(conf.Settings.NoHighlightFG) > 0 {
+		colors[color.Level16]["nohlfg"] = ColorStringToColor(conf.Settings.NoHighlightFG)
+		colors[color.Level256]["nohlfg"] = ColorStringToColor(conf.Settings.NoHighlightFG)
+		colors[color.LevelRgb]["nohlfg"] = ColorStringToColor(conf.Settings.NoHighlightFG)
+	}
+
+	if len(conf.Settings.HighlightHdrBG) > 0 {
+		colors[color.Level16]["hdrbg"] = ColorStringToBGColor(conf.Settings.HighlightHdrBG)
+		colors[color.Level256]["hdrbg"] = ColorStringToBGColor(conf.Settings.HighlightHdrBG)
+		colors[color.LevelRgb]["hdrbg"] = ColorStringToBGColor(conf.Settings.HighlightHdrBG)
+	}
+
+	if len(conf.Settings.HighlightHdrFG) > 0 {
+		colors[color.Level16]["hdrfg"] = ColorStringToColor(conf.Settings.HighlightHdrFG)
+		colors[color.Level256]["hdrfg"] = ColorStringToColor(conf.Settings.HighlightHdrFG)
+		colors[color.LevelRgb]["hdrfg"] = ColorStringToColor(conf.Settings.HighlightHdrFG)
+	}
+
+	return colors
 }
 
 // find supported color mode, modifies config based on constants
-func (c *Config) DetermineColormode() {
+func (conf *Config) DetermineColormode() {
 	if !isTerminal(os.Stdout) {
 		color.Disable()
 	} else {
 		level := color.TermColorLevel()
-		colors := Colors()
+		colors := conf.Colors()
-		c.ColorStyle = color.New(colors[level]["bg"], colors[level]["fg"])
+
+		conf.ColorStyle = color.New(colors[level]["bg"], colors[level]["fg"])
+		conf.HighlightStyle = color.New(colors[level]["hlbg"], colors[level]["hlfg"])
+		conf.NoHighlightStyle = color.New(colors[level]["nohlbg"], colors[level]["nohlfg"])
+		conf.HighlightHdrStyle = color.New(colors[level]["hdrbg"], colors[level]["hdrfg"])
 	}
 }
 
 // Return true if current terminal is interactive
 func isTerminal(f *os.File) bool {
 	o, _ := f.Stat()
-	if (o.Mode() & os.ModeCharDevice) == os.ModeCharDevice {
+
-		return true
+	return (o.Mode() & os.ModeCharDevice) == os.ModeCharDevice
-	} else {
-		return false
-	}
 }
 
+// main program version
+// generated  version string, used  by -v contains  lib.Version on
+//
+//	main branch, and lib.Version-$branch-$lastcommit-$date on
+//
+// development branch
 func Getversion() string {
-	// main program version
-
-	// generated  version string, used  by -v contains  lib.Version on
-	//  main  branch,   and  lib.Version-$branch-$lastcommit-$date  on
-	// development branch
-
 	return fmt.Sprintf("This is tablizer version %s", VERSION)
 }
 
@@ -158,44 +258,129 @@ func (conf *Config) PrepareModeFlags(flag Modeflag) {
 	case flag.C:
 		conf.OutputMode = CSV
 	default:
-		conf.OutputMode = Ascii
+		conf.OutputMode = ASCII
 	}
 }
 
-func (c *Config) CheckEnv() {
+func (conf *Config) PrepareFilters() error {
-	// check for environment vars, command line flags have precedence,
+	conf.Filters = make(map[string]*regexp.Regexp, len(conf.Rawfilters))
-	// NO_COLOR is being checked by the color module itself.
+
-	if !c.NoNumbering {
+	for _, filter := range conf.Rawfilters {
-		_, set := os.LookupEnv("T_NO_HEADER_NUMBERING")
+		parts := strings.Split(filter, "=")
-		if set {
+		if len(parts) != MAXPARTS {
-			c.NoNumbering = true
+			return errors.New("filter field and value must be separated by =")
 		}
-	}
 
-	if len(c.Columns) == 0 {
+		reg, err := regexp.Compile(parts[1])
-		cols := os.Getenv("T_COLUMNS")
+		if err != nil {
-		if len(cols) > 1 {
+			return fmt.Errorf("failed to compile filter regex for field %s: %w",
-			c.Columns = cols
+				parts[0], err)
 		}
+
+		conf.Filters[strings.ToLower(parts[0])] = reg
 	}
-}
-
-func (c *Config) ApplyDefaults() {
-	// mode specific defaults
-	if c.OutputMode == Yaml || c.OutputMode == CSV {
-		c.NoNumbering = true
-	}
-}
-
-func (c *Config) PreparePattern(pattern string) error {
-	PatternR, err := regexp.Compile(pattern)
-
-	if err != nil {
-		return errors.Unwrap(fmt.Errorf("Regexp pattern %s is invalid: %w", c.Pattern, err))
-	}
-
-	c.PatternR = PatternR
-	c.Pattern = pattern
 
 	return nil
 }
+
+func (conf *Config) CheckEnv() {
+	// check for environment vars, command line flags have precedence,
+	// NO_COLOR is being checked by the color module itself.
+	if !conf.NoNumbering {
+		_, set := os.LookupEnv("T_NO_HEADER_NUMBERING")
+		if set {
+			conf.NoNumbering = true
+		}
+	}
+
+	if len(conf.Columns) == 0 {
+		cols := os.Getenv("T_COLUMNS")
+		if len(cols) > 1 {
+			conf.Columns = cols
+		}
+	}
+}
+
+func (conf *Config) ApplyDefaults() {
+	// mode specific defaults
+	if conf.OutputMode == Yaml || conf.OutputMode == CSV {
+		conf.NoNumbering = true
+	}
+
+	ValidHooks = []string{"filter", "process", "transpose", "append"}
+}
+
+func (conf *Config) PreparePattern(pattern string) error {
+	PatternR, err := regexp.Compile(pattern)
+
+	if err != nil {
+		return fmt.Errorf("regexp pattern %s is invalid: %w", conf.Pattern, err)
+	}
+
+	conf.PatternR = PatternR
+	conf.Pattern = pattern
+
+	return nil
+}
+
+// Parse config file.  Ignore if the file doesn't exist  but return an
+// error if it exists but fails to read or parse
+func (conf *Config) ParseConfigfile() error {
+	path, err := os.Stat(conf.Configfile)
+
+	if os.IsNotExist(err) || path.IsDir() {
+		// ignore non-existent or dirs
+		return nil
+	}
+
+	configstring, err := os.ReadFile(path.Name())
+	if err != nil {
+		return fmt.Errorf("failed to read config file %s: %w", path.Name(), err)
+	}
+
+	err = hclsimple.Decode(
+		path.Name(),
+		configstring,
+		nil,
+		&conf.Settings)
+	if err != nil {
+		return fmt.Errorf("failed to load configuration file %s: %w",
+			path.Name(), err)
+	}
+
+	return nil
+}
+
+// translate color string to internal color value
+func ColorStringToColor(colorname string) color.Color {
+	for name, color := range color.FgColors {
+		if name == colorname {
+			return color
+		}
+	}
+
+	for name, color := range color.ExFgColors {
+		if name == colorname {
+			return color
+		}
+	}
+
+	return color.Normal
+}
+
+// same, for background colors
+func ColorStringToBGColor(colorname string) color.Color {
+	for name, color := range color.BgColors {
+		if name == colorname {
+			return color
+		}
+	}
+
+	for name, color := range color.ExBgColors {
+		if name == colorname {
+			return color
+		}
+	}
+
+	return color.Normal
+}

cfg/config_test.go

@@ -34,18 +34,18 @@ func TestPrepareModeFlags(t *testing.T) {
 		{Modeflag{O: true}, Orgtbl},
 		{Modeflag{Y: true}, Yaml},
 		{Modeflag{M: true}, Markdown},
-		{Modeflag{}, Ascii},
+		{Modeflag{}, ASCII},
 	}
 
 	// FIXME: use a map for easier printing
-	for _, tt := range tests {
+	for _, testdata := range tests {
-		testname := fmt.Sprintf("PrepareModeFlags-expect-%d", tt.expect)
+		testname := fmt.Sprintf("PrepareModeFlags-expect-%d", testdata.expect)
 		t.Run(testname, func(t *testing.T) {
-			c := Config{}
+			conf := Config{}
 
-			c.PrepareModeFlags(tt.flag)
+			conf.PrepareModeFlags(testdata.flag)
-			if c.OutputMode != tt.expect {
+			if conf.OutputMode != testdata.expect {
-				t.Errorf("got: %d, expect: %d", c.OutputMode, tt.expect)
+				t.Errorf("got: %d, expect: %d", conf.OutputMode, testdata.expect)
 			}
 		})
 	}
@@ -63,15 +63,15 @@ func TestPrepareSortFlags(t *testing.T) {
 		{Sortmode{}, "string"},
 	}
 
-	for _, tt := range tests {
+	for _, testdata := range tests {
-		testname := fmt.Sprintf("PrepareSortFlags-expect-%s", tt.expect)
+		testname := fmt.Sprintf("PrepareSortFlags-expect-%s", testdata.expect)
 		t.Run(testname, func(t *testing.T) {
-			c := Config{}
+			conf := Config{}
 
-			c.PrepareSortFlags(tt.flag)
+			conf.PrepareSortFlags(testdata.flag)
 
-			if c.SortMode != tt.expect {
+			if conf.SortMode != testdata.expect {
-				t.Errorf("got: %s, expect: %s", c.SortMode, tt.expect)
+				t.Errorf("got: %s, expect: %s", conf.SortMode, testdata.expect)
 			}
 		})
 	}
@@ -86,15 +86,16 @@ func TestPreparePattern(t *testing.T) {
 		{"[a-z", true},
 	}
 
-	for _, tt := range tests {
+	for _, testdata := range tests {
-		testname := fmt.Sprintf("PreparePattern-pattern-%s-wanterr-%t", tt.pattern, tt.wanterr)
+		testname := fmt.Sprintf("PreparePattern-pattern-%s-wanterr-%t",
+			testdata.pattern, testdata.wanterr)
 		t.Run(testname, func(t *testing.T) {
-			c := Config{}
+			conf := Config{}
 
-			err := c.PreparePattern(tt.pattern)
+			err := conf.PreparePattern(testdata.pattern)
 
 			if err != nil {
-				if !tt.wanterr {
+				if !testdata.wanterr {
 					t.Errorf("PreparePattern returned error: %s", err)
 				}
 			}

cmd/root.go

@@ -1,5 +1,5 @@
 /*
-Copyright © 2022 Thomas von Dein
+Copyright © 2022-2024 Thomas von Dein
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -20,23 +20,25 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"github.com/spf13/cobra"
-	"github.com/tlinden/tablizer/cfg"
-	"github.com/tlinden/tablizer/lib"
 	"log"
 	"os"
 	"os/exec"
 	"strings"
+
+	"github.com/spf13/cobra"
+	"github.com/tlinden/tablizer/cfg"
+	"github.com/tlinden/tablizer/lib"
 )
 
 func man() {
 	man := exec.Command("less", "-")
 
-	var b bytes.Buffer
+	var buffer bytes.Buffer
-	b.Write([]byte(manpage))
+
+	buffer.Write([]byte(manpage))
 
 	man.Stdout = os.Stdout
-	man.Stdin = &b
+	man.Stdin = &buffer
 	man.Stderr = os.Stderr
 
 	err := man.Run()
@@ -57,7 +59,7 @@ func completion(cmd *cobra.Command, mode string) error {
 	case "powershell":
 		return cmd.Root().GenPowerShellCompletionWithDesc(os.Stdout)
 	default:
-		return errors.New("Invalid shell parameter! Valid ones: bash|zsh|fish|powershell")
+		return errors.New("invalid shell parameter! Valid ones: bash|zsh|fish|powershell")
 	}
 }
 
@@ -78,11 +80,13 @@ func Execute() {
 		RunE: func(cmd *cobra.Command, args []string) error {
 			if ShowVersion {
 				fmt.Println(cfg.Getversion())
+
 				return nil
 			}
 
 			if ShowManual {
 				man()
+
 				return nil
 			}
 
@@ -91,48 +95,103 @@ func Execute() {
 			}
 
 			// Setup
+			err := conf.ParseConfigfile()
+			if err != nil {
+				return err
+			}
+
 			conf.CheckEnv()
 			conf.PrepareModeFlags(modeflag)
 			conf.PrepareSortFlags(sortmode)
+
+			if err = conf.PrepareFilters(); err != nil {
+				return err
+			}
+
 			conf.DetermineColormode()
 			conf.ApplyDefaults()
 
+			// setup lisp env, load plugins etc
+			err = lib.SetupLisp(&conf)
+			if err != nil {
+				return err
+			}
+
 			// actual execution starts here
 			return lib.ProcessFiles(&conf, args)
 		},
 	}
 
 	// options
-	rootCmd.PersistentFlags().BoolVarP(&conf.Debug, "debug", "d", false, "Enable debugging")
+	rootCmd.PersistentFlags().BoolVarP(&conf.Debug, "debug", "d", false,
-	rootCmd.PersistentFlags().BoolVarP(&conf.NoNumbering, "no-numbering", "n", false, "Disable header numbering")
+		"Enable debugging")
-	rootCmd.PersistentFlags().BoolVarP(&conf.NoHeaders, "no-headers", "", false, "Disable header display")
+	rootCmd.PersistentFlags().BoolVarP(&conf.NoNumbering, "no-numbering", "n", false,
-	rootCmd.PersistentFlags().BoolVarP(&conf.NoColor, "no-color", "N", false, "Disable pattern highlighting")
+		"Disable header numbering")
-	rootCmd.PersistentFlags().BoolVarP(&ShowVersion, "version", "V", false, "Print program version")
+	rootCmd.PersistentFlags().BoolVarP(&conf.NoHeaders, "no-headers", "H", false,
-	rootCmd.PersistentFlags().BoolVarP(&conf.InvertMatch, "invert-match", "v", false, "select non-matching rows")
+		"Disable header display")
-	rootCmd.PersistentFlags().BoolVarP(&ShowManual, "man", "m", false, "Display manual page")
+	rootCmd.PersistentFlags().BoolVarP(&conf.NoColor, "no-color", "N", false,
-	rootCmd.PersistentFlags().StringVarP(&ShowCompletion, "completion", "", "", "Display completion code")
+		"Disable pattern highlighting")
-	rootCmd.PersistentFlags().StringVarP(&conf.Separator, "separator", "s", cfg.DefaultSeparator, "Custom field separator")
+	rootCmd.PersistentFlags().BoolVarP(&ShowVersion, "version", "V", false,
-	rootCmd.PersistentFlags().StringVarP(&conf.Columns, "columns", "c", "", "Only show the speficied columns (separated by ,)")
+		"Print program version")
+	rootCmd.PersistentFlags().BoolVarP(&conf.InvertMatch, "invert-match", "v", false,
+		"select non-matching rows")
+	rootCmd.PersistentFlags().BoolVarP(&ShowManual, "man", "m", false,
+		"Display manual page")
+	rootCmd.PersistentFlags().BoolVarP(&conf.UseFuzzySearch, "fuzzy", "z", false,
+		"Use fuzzy searching")
+	rootCmd.PersistentFlags().BoolVarP(&conf.UseHighlight, "highlight-lines", "L", false,
+		"Use alternating background colors")
+	rootCmd.PersistentFlags().StringVarP(&ShowCompletion, "completion", "", "",
+		"Display completion code")
+	rootCmd.PersistentFlags().StringVarP(&conf.Separator, "separator", "s", cfg.DefaultSeparator,
+		"Custom field separator")
+	rootCmd.PersistentFlags().StringVarP(&conf.Columns, "columns", "c", "",
+		"Only show the speficied columns (separated by ,)")
 
 	// sort options
-	rootCmd.PersistentFlags().IntVarP(&conf.SortByColumn, "sort-by", "k", 0, "Sort by column (default: 1)")
+	rootCmd.PersistentFlags().IntVarP(&conf.SortByColumn, "sort-by", "k", 0,
+		"Sort by column (default: 1)")
 
 	// sort mode, only 1 allowed
-	rootCmd.PersistentFlags().BoolVarP(&conf.SortDescending, "sort-desc", "D", false, "Sort in descending order (default: ascending)")
+	rootCmd.PersistentFlags().BoolVarP(&conf.SortDescending, "sort-desc", "D", false,
-	rootCmd.PersistentFlags().BoolVarP(&sortmode.Numeric, "sort-numeric", "i", false, "sort according to string numerical value")
+		"Sort in descending order (default: ascending)")
-	rootCmd.PersistentFlags().BoolVarP(&sortmode.Time, "sort-time", "t", false, "sort according to time string")
+	rootCmd.PersistentFlags().BoolVarP(&sortmode.Numeric, "sort-numeric", "i", false,
-	rootCmd.PersistentFlags().BoolVarP(&sortmode.Age, "sort-age", "a", false, "sort according to age (duration) string")
+		"sort according to string numerical value")
-	rootCmd.MarkFlagsMutuallyExclusive("sort-numeric", "sort-time", "sort-age")
+	rootCmd.PersistentFlags().BoolVarP(&sortmode.Time, "sort-time", "t", false,
+		"sort according to time string")
+	rootCmd.PersistentFlags().BoolVarP(&sortmode.Age, "sort-age", "a", false,
+		"sort according to age (duration) string")
+	rootCmd.MarkFlagsMutuallyExclusive("sort-numeric", "sort-time",
+		"sort-age")
 
 	// output flags, only 1 allowed
-	rootCmd.PersistentFlags().BoolVarP(&modeflag.X, "extended", "X", false, "Enable extended output")
+	rootCmd.PersistentFlags().BoolVarP(&modeflag.X, "extended", "X", false,
-	rootCmd.PersistentFlags().BoolVarP(&modeflag.M, "markdown", "M", false, "Enable markdown table output")
+		"Enable extended output")
-	rootCmd.PersistentFlags().BoolVarP(&modeflag.O, "orgtbl", "O", false, "Enable org-mode table output")
+	rootCmd.PersistentFlags().BoolVarP(&modeflag.M, "markdown", "M", false,
-	rootCmd.PersistentFlags().BoolVarP(&modeflag.S, "shell", "S", false, "Enable shell mode output")
+		"Enable markdown table output")
-	rootCmd.PersistentFlags().BoolVarP(&modeflag.Y, "yaml", "Y", false, "Enable yaml output")
+	rootCmd.PersistentFlags().BoolVarP(&modeflag.O, "orgtbl", "O", false,
-	rootCmd.PersistentFlags().BoolVarP(&modeflag.C, "csv", "C", false, "Enable CSV output")
+		"Enable org-mode table output")
-	rootCmd.PersistentFlags().BoolVarP(&modeflag.A, "ascii", "A", false, "Enable ASCII output (default)")
+	rootCmd.PersistentFlags().BoolVarP(&modeflag.S, "shell", "S", false,
-	rootCmd.MarkFlagsMutuallyExclusive("extended", "markdown", "orgtbl", "shell", "yaml", "csv")
+		"Enable shell mode output")
+	rootCmd.PersistentFlags().BoolVarP(&modeflag.Y, "yaml", "Y", false,
+		"Enable yaml output")
+	rootCmd.PersistentFlags().BoolVarP(&modeflag.C, "csv", "C", false,
+		"Enable CSV output")
+	rootCmd.PersistentFlags().BoolVarP(&modeflag.A, "ascii", "A", false,
+		"Enable ASCII output (default)")
+	rootCmd.MarkFlagsMutuallyExclusive("extended", "markdown", "orgtbl",
+		"shell", "yaml", "csv")
+
+	// lisp options
+	rootCmd.PersistentFlags().StringVarP(&conf.LispLoadPath, "load-path", "l", cfg.DefaultLoadPath,
+		"Load path for lisp plugins (expects *.zy files)")
+
+	// config file
+	rootCmd.PersistentFlags().StringVarP(&conf.Configfile, "config", "f", cfg.DefaultConfigfile,
+		"config file (default: ~/.config/tablizer/config)")
+
+	// filters
+	rootCmd.PersistentFlags().StringArrayVarP(&conf.Rawfilters, "filter", "F", nil, "Filter by field (field=regexp)")
 
 	rootCmd.SetUsageTemplate(strings.TrimSpace(usage) + "\n")
 

cmd/tablizer.go

@@ -13,18 +13,21 @@ SYNOPSIS
           -v, --invert-match       select non-matching rows
           -n, --no-numbering       Disable header numbering
           -N, --no-color           Disable pattern highlighting
-              --no-headers         Disable headers display
+          -H, --no-headers         Disable headers display
           -s, --separator string   Custom field separator
           -k, --sort-by int        Sort by column (default: 1)
+          -z, --fuzzy              Use fuzzy search [experimental]
+          -F, --filter field=reg   Filter given field with regex, can be used multiple times
 
         Output Flags (mutually exclusive):
           -X, --extended           Enable extended output
           -M, --markdown           Enable markdown table output
           -O, --orgtbl             Enable org-mode table output
-          -S, --shell              Enable shell evaluable ouput
+          -S, --shell              Enable shell evaluable output
           -Y, --yaml               Enable yaml output
           -C, --csv                Enable CSV output
           -A, --ascii              Default output mode, ascii tabular
+          -L, --hightlight-lines   Use alternating background colors for tables
 
         Sort Mode Flags (mutually exclusive):
           -a, --sort-age           sort according to age (duration) string
@@ -34,10 +37,11 @@ SYNOPSIS
 
         Other Flags:
               --completion <shell> Generate the autocompletion script for <shell>
+          -f, --config <file>      Configuration file (default: ~/.config/tablizer/config)
           -d, --debug              Enable debugging
           -h, --help               help for tablizer
           -m, --man                Display manual page
-          -v, --version            Print program version
+          -V, --version            Print program version
 
 DESCRIPTION
     Many programs generate tabular output. But sometimes you need to
@@ -89,9 +93,9 @@ DESCRIPTION
     The numbering can be suppressed by using the -n option.
 
     By default tablizer shows a header containing the names of each column.
-    This can be disabled using the --no-headers option. Be aware that this
+    This can be disabled using the -H option. Be aware that this only
-    only affects tabular output modes. Shell, Extended, Yaml and CSV output
+    affects tabular output modes. Shell, Extended, Yaml and CSV output modes
-    modes always use the column names.
+    always use the column names.
 
     By default, if a pattern has been speficied, matches will be
     highlighted. You can disable this behavior with the -N option.
@@ -114,7 +118,7 @@ DESCRIPTION
     Finally the -d option enables debugging output which is mostly useful
     for the developer.
 
-  PATTERNS
+  PATTERNS AND FILTERING
     You can reduce the rows being displayed by using a regular expression
     pattern. The regexp is PCRE compatible, refer to the syntax cheat sheet
     here: <https://github.com/google/re2/wiki/Syntax>. If you want to read a
@@ -138,6 +142,23 @@ DESCRIPTION
 
         kubectl get pods -A | tablizer "(?i)account"
 
+    You can use the experimental fuzzy search feature by providing the
+    option -z, in which case the pattern is regarded as a fuzzy search term,
+    not a regexp.
+
+    Sometimes you want to filter by one or more columns. You can do that
+    using the -F option. The option can be specified multiple times and has
+    the following format:
+
+        fieldname=regexp
+
+    Fieldnames (== columns headers) are case insensitive.
+
+    If you specify more than one filter, both filters have to match (AND
+    operation).
+
+    If the option -v is specified, the filtering is inverted.
+
   COLUMNS
     The parameter -c can be used to specify, which columns to display. By
     default tablizer numerizes the header names and these numbers can be
@@ -252,6 +273,36 @@ DESCRIPTION
 
         and source this file from your PowerShell profile.
 
+CONFIGURATION AND COLORS
+    YOu can put certain configuration values into a configuration file in
+    HCL format. By default tablizer looks for
+    "$HOME/.config/tablizer/config", but you can provide one using the
+    parameter "-f".
+
+    In the configuration the following variables can be defined:
+
+        BG             = "lightGreen"
+        FG             = "white"
+        HighlightBG    = "lightGreen"
+        HighlightFG    = "white"
+        NoHighlightBG  = "white"
+        NoHighlightFG  = "lightGreen"
+        HighlightHdrBG = "red"
+        HighlightHdrFG = "white"
+
+    The following color definitions are available:
+
+    black, blue, cyan, darkGray, default, green, lightBlue, lightCyan,
+    lightGreen, lightMagenta, lightRed, lightWhite, lightYellow, magenta,
+    red, white, yellow
+
+    The Variables FG and BG are being used to highlight matches. The other
+    *FG and *BG variables are for colored table output (enabled with the
+    "-L" parameter).
+
+    Colorization can be turned off completely either by setting the
+    parameter "-N" or the environment variable NO_COLOR to a true value.
+
 BUGS
     In order to report a bug, unexpected behavior, feature requests or to
     submit a patch, please open an issue on github:
@@ -261,7 +312,7 @@ LICENSE
     This software is licensed under the GNU GENERAL PUBLIC LICENSE version
     3.
 
-    Copyright (c) 2023 by Thomas von Dein
+    Copyright (c) 2022-2024 by Thomas von Dein
 
     This software uses the following GO modules:
 
@@ -299,18 +350,21 @@ Operational Flags:
   -v, --invert-match       select non-matching rows
   -n, --no-numbering       Disable header numbering
   -N, --no-color           Disable pattern highlighting
-      --no-headers         Disable headers display
+  -H, --no-headers         Disable headers display
   -s, --separator string   Custom field separator
   -k, --sort-by int        Sort by column (default: 1)
+  -z, --fuzzy              Use fuzzy search [experimental]
+  -F, --filter field=reg   Filter given field with regex, can be used multiple times
 
 Output Flags (mutually exclusive):
   -X, --extended           Enable extended output
   -M, --markdown           Enable markdown table output
   -O, --orgtbl             Enable org-mode table output
-  -S, --shell              Enable shell evaluable ouput
+  -S, --shell              Enable shell evaluable output
   -Y, --yaml               Enable yaml output
   -C, --csv                Enable CSV output
   -A, --ascii              Default output mode, ascii tabular
+  -L, --hightlight-lines   Use alternating background colors for tables
 
 Sort Mode Flags (mutually exclusive):
   -a, --sort-age           sort according to age (duration) string
@@ -320,10 +374,11 @@ Sort Mode Flags (mutually exclusive):
 
 Other Flags:
       --completion <shell> Generate the autocompletion script for <shell>
+  -f, --config <file>      Configuration file (default: ~/.config/tablizer/config)
   -d, --debug              Enable debugging
   -h, --help               help for tablizer
   -m, --man                Display manual page
-  -v, --version            Print program version
+  -V, --version            Print program version
 
 
 `

config.hcl

@@ -0,0 +1,12 @@
+# supported  colors:
+# black, blue,  cyan, darkGray, default, green,  lightBlue, lightCyan,
+# lightGreen,   lightMagenta,   lightRed,   lightWhite,   lightYellow,
+# magenta, red, white, yellow
+BG             = "lightGreen"
+FG             = "white"
+HighlightBG    = "lightGreen"
+HighlightFG    = "white"
+NoHighlightBG  = "white"
+NoHighlightFG  = "lightGreen"
+HighlightHdrBG = "red"
+HighlightHdrFG = "white"

go.mod

@@ -5,20 +5,39 @@ go 1.18
 require (
 	github.com/alecthomas/repr v0.1.1
 	github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
+	github.com/glycerine/zygomys v5.1.2+incompatible
 	github.com/gookit/color v1.5.2
+	github.com/hashicorp/hcl/v2 v2.19.1
+	github.com/lithammer/fuzzysearch v1.1.7
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/spf13/cobra v1.6.1
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
+	github.com/agext/levenshtein v1.2.1 // indirect
+	github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
+	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
+	github.com/glycerine/blake2b v0.0.0-20151022103502-3c8c640cd7be // indirect
+	github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 // indirect
+	github.com/glycerine/greenpack v5.1.1+incompatible // indirect
+	github.com/glycerine/liner v0.0.0-20160121172638-72909af234e0 // indirect
+	github.com/google/go-cmp v0.5.6 // indirect
+	github.com/gopherjs/gopherjs v1.17.2 // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
-	github.com/mattn/go-runewidth v0.0.14 // indirect
+	github.com/jtolds/gls v4.20.0+incompatible // indirect
-
+	github.com/mattn/go-runewidth v0.0.10 // indirect
-    // force release. > 0.4. doesnt build everywhere, see:
+	github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 // indirect
-    // https://github.com/TLINDEN/tablizer/actions/runs/3396457307/jobs/5647544615
+	github.com/philhofer/fwd v1.1.2 // indirect
-    github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/rivo/uniseg v0.1.0 // indirect
+	github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636 // indirect
+	github.com/shurcooL/go-goon v1.0.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
+	github.com/tinylib/msgp v1.1.9 // indirect
-	golang.org/x/sys v0.1.0 // indirect
+	github.com/ugorji/go/codec v1.2.11 // indirect
+	github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
+	github.com/zclconf/go-cty v1.13.0 // indirect
+	golang.org/x/sys v0.13.0 // indirect
+	golang.org/x/text v0.11.0 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 )

go.sum

@@ -1,28 +1,65 @@
+github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8=
+github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
 github.com/alecthomas/repr v0.1.1 h1:87P60cSmareLAxMc4Hro0r2RBY4ROm0dYwkJNpS4pPs=
 github.com/alecthomas/repr v0.1.1/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
+github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw=
+github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
+github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY=
+github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4=
 github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA=
 github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/glycerine/blake2b v0.0.0-20151022103502-3c8c640cd7be h1:XBJdPGgA3qqhW+p9CANCAVdF7ZIXdu3pZAkypMkKAjE=
+github.com/glycerine/blake2b v0.0.0-20151022103502-3c8c640cd7be/go.mod h1:OSCrScrFAjcBObrulk6BEQlytA462OkG1UGB5NYj9kE=
+github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy3PbkQ1AERPfmLMMagS60DKF78eWwLn8=
+github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
+github.com/glycerine/greenpack v5.1.1+incompatible h1:fDr9i6MkSGZmAy4VXPfJhW+SyK2/LNnzIp5nHyDiaIM=
+github.com/glycerine/greenpack v5.1.1+incompatible/go.mod h1:us0jVISAESGjsEuLlAfCd5nkZm6W6WQF18HPuOecIg4=
+github.com/glycerine/liner v0.0.0-20160121172638-72909af234e0 h1:4ZegphJXBTc4uFQ08UVoWYmQXorGa+ipXetUj83sMBc=
+github.com/glycerine/liner v0.0.0-20160121172638-72909af234e0/go.mod h1:AqJLs6UeoC65dnHxyCQ6MO31P5STpjcmgaANAU+No8Q=
+github.com/glycerine/zygomys v5.1.2+incompatible h1:jmcdmA3XPxgfOunAXFpipE9LQoUL6eX6d2mhYyjV4GE=
+github.com/glycerine/zygomys v5.1.2+incompatible/go.mod h1:i3SPKZpmy9dwF/3iWrXJ/ZLyzZucegwypwOmqRkUUaQ=
+github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/gookit/color v1.5.2 h1:uLnfXcaFjlrDnQDT+NCBcfhrXqYTx/rcCa6xn01Y8yI=
 github.com/gookit/color v1.5.2/go.mod h1:w8h4bGiHeeBpvQVePTutdbERIUf3oJE5lZ8HM0UgXyg=
+github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g=
+github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k=
+github.com/hashicorp/hcl/v2 v2.19.1 h1://i05Jqznmb2EXqa39Nsvyan2o5XyMowW5fnCKW5RPI=
+github.com/hashicorp/hcl/v2 v2.19.1/go.mod h1:ThLC89FV4p9MPW804KVbe/cEXoQ8NZEh+JtMeeGErHE=
 github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
 github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
+github.com/lithammer/fuzzysearch v1.1.7 h1:q8rZNmBIUkqxsxb/IlwsXVbCoPIH/0juxjFHY0UIwhU=
+github.com/lithammer/fuzzysearch v1.1.7/go.mod h1:ZhIlfRGxnD8qa9car/yplC6GmnM14CS07BYAKJJBK2I=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRRpdGg=
 github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
-github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
+github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 h1:DpOJ2HYzCv8LZP15IdmG+YdwD2luVPHITV96TkirNBM=
-github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
 github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
 github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
+github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw=
+github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rivo/uniseg v0.1.0 h1:+2KBaVoUmb9XzDsrx/Ct0W/EYOSFf/nWTauy++DprtY=
 github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
-github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg=
+github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
+github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636 h1:aSISeOcal5irEhJd1M+IrApc0PdcN7e7Aj4yuEnOrfQ=
+github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
+github.com/shurcooL/go-goon v1.0.0 h1:BCQPvxGkHHJ4WpBO4m/9FXbITVIsvAm/T66cCcCGI7E=
+github.com/shurcooL/go-goon v1.0.0/go.mod h1:2wTHMsGo7qnpmqA8ADYZtP4I1DD94JpXGQ3Dxq2YQ5w=
 github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
 github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
@@ -33,13 +70,53 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/tinylib/msgp v1.1.9 h1:SHf3yoO2sGA0veCJeCBYLHuttAVFHGm2RHgNodW7wQU=
+github.com/tinylib/msgp v1.1.9/go.mod h1:BCXGB54lDD8qUEPmiG0cQQUANC4IUQyB2ItS2UDlO/k=
+github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
+github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
+github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8=
 github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs=
-github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
+github.com/zclconf/go-cty v1.13.0 h1:It5dfKTTZHe9aeppbNOda3mN7Ag7sg6QkBNm6TkyFa0=
-golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561 h1:MDc5xs78ZrZr3HMQugiXOAkSZtfTpbJLDr/lwfgO53E=
+github.com/zclconf/go-cty v1.13.0/go.mod h1:YKQzy/7pZ7iq2jNFzy5go57xdxdWoLLpaEp4u238AE0=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
+golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

lib/common.go

@@ -1,5 +1,5 @@
 /*
-Copyright © 2022 Thomas von Dein
+Copyright © 2022-2024 Thomas von Dein
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -24,3 +24,13 @@ type Tabdata struct {
 	headers        []string // [ "ID", "NAME", ...]
 	entries        [][]string
 }
+
+func (data *Tabdata) CloneEmpty() Tabdata {
+	newdata := Tabdata{
+		maxwidthHeader: data.maxwidthHeader,
+		columns:        data.columns,
+		headers:        data.headers,
+	}
+
+	return newdata
+}

lib/filter.go

@@ -0,0 +1,130 @@
+/*
+Copyright © 2022-2024 Thomas von Dein
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package lib
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/lithammer/fuzzysearch/fuzzy"
+	"github.com/tlinden/tablizer/cfg"
+)
+
+/*
+ * [!]Match a  line, use fuzzy  search for normal pattern  strings and
+ * regexp otherwise.
+ */
+func matchPattern(conf cfg.Config, line string) bool {
+	if conf.UseFuzzySearch {
+		return fuzzy.MatchFold(conf.Pattern, line)
+	}
+
+	return conf.PatternR.MatchString(line)
+}
+
+/*
+ * Filter parsed data by fields. The  filter is positive, so if one or
+ * more filters match on a row, it  will be kept, otherwise it will be
+ * excluded.
+ */
+func FilterByFields(conf cfg.Config, data Tabdata) (Tabdata, bool, error) {
+	if len(conf.Filters) == 0 {
+		// no filters, no checking
+		return Tabdata{}, false, nil
+	}
+
+	newdata := data.CloneEmpty()
+
+	for _, row := range data.entries {
+		keep := true
+
+		for idx, header := range data.headers {
+			if !Exists(conf.Filters, strings.ToLower(header)) {
+				// do not filter by unspecified field
+				continue
+			}
+
+			if !conf.Filters[strings.ToLower(header)].MatchString(row[idx]) {
+				// there IS a filter, but it doesn't match
+				keep = false
+
+				break
+			}
+		}
+
+		if keep == !conf.InvertMatch {
+			// also apply -v
+			newdata.entries = append(newdata.entries, row)
+		}
+	}
+
+	return newdata, true, nil
+}
+
+/* generic map.Exists(key) */
+func Exists[K comparable, V any](m map[K]V, v K) bool {
+	if _, ok := m[v]; ok {
+		return true
+	}
+
+	return false
+}
+
+func FilterByPattern(conf cfg.Config, input io.Reader) (io.Reader, error) {
+	if conf.Pattern == "" {
+		return input, nil
+	}
+
+	scanner := bufio.NewScanner(input)
+	lines := []string{}
+	hadFirst := false
+
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if hadFirst {
+			// don't match 1st line, it's the header
+			if conf.Pattern != "" && matchPattern(conf, line) == conf.InvertMatch {
+				// by default  -v is false, so if a  line does NOT
+				// match the pattern, we will ignore it. However,
+				// if the user specified -v, the matching is inverted,
+				// so we ignore all lines, which DO match.
+				continue
+			}
+
+			// apply user defined lisp filters, if any
+			accept, err := RunFilterHooks(conf, line)
+			if err != nil {
+				return input, fmt.Errorf("failed to apply filter hook: %w", err)
+			}
+
+			if !accept {
+				//  IF there  are filter  hook[s] and  IF one  of them
+				// returns false on the current line, reject it
+				continue
+			}
+		}
+
+		lines = append(lines, line)
+
+		hadFirst = true
+	}
+
+	return strings.NewReader(strings.Join(lines, "\n")), nil
+}

lib/filter_test.go

@@ -0,0 +1,162 @@
+/*
+Copyright © 2024 Thomas von Dein
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package lib
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+
+	"github.com/tlinden/tablizer/cfg"
+)
+
+func TestMatchPattern(t *testing.T) {
+	var input = []struct {
+		name    string
+		fuzzy   bool
+		pattern string
+		line    string
+	}{
+		{
+			name:    "normal",
+			pattern: "haus",
+			line:    "hausparty",
+		},
+		{
+			name:    "fuzzy",
+			pattern: "hpt",
+			line:    "haus-party-termin",
+			fuzzy:   true,
+		},
+	}
+
+	for _, inputdata := range input {
+		testname := fmt.Sprintf("match-pattern-%s", inputdata.name)
+
+		t.Run(testname, func(t *testing.T) {
+			conf := cfg.Config{}
+
+			if inputdata.fuzzy {
+				conf.UseFuzzySearch = true
+			}
+
+			err := conf.PreparePattern(inputdata.pattern)
+			if err != nil {
+				t.Errorf("PreparePattern returned error: %s", err)
+			}
+
+			if !matchPattern(conf, inputdata.line) {
+				t.Errorf("matchPattern() did not match\nExp: true\nGot: false\n")
+			}
+		})
+	}
+}
+
+func TestFilterByFields(t *testing.T) {
+	data := Tabdata{
+		headers: []string{
+			"ONE", "TWO", "THREE",
+		},
+		entries: [][]string{
+			{"asd", "igig", "cxxxncnc"},
+			{"19191", "EDD 1", "x"},
+			{"8d8", "AN 1", "y"},
+		},
+	}
+
+	var input = []struct {
+		name   string
+		filter []string
+		expect Tabdata
+		invert bool
+	}{
+		{
+			name:   "one-field",
+			filter: []string{"one=19"},
+			expect: Tabdata{
+				headers: []string{
+					"ONE", "TWO", "THREE",
+				},
+				entries: [][]string{
+					{"19191", "EDD 1", "x"},
+				},
+			},
+		},
+
+		{
+			name:   "one-field-inverted",
+			filter: []string{"one=19"},
+			invert: true,
+			expect: Tabdata{
+				headers: []string{
+					"ONE", "TWO", "THREE",
+				},
+				entries: [][]string{
+					{"asd", "igig", "cxxxncnc"},
+					{"8d8", "AN 1", "y"},
+				},
+			},
+		},
+
+		{
+			name:   "many-fields",
+			filter: []string{"one=19", "two=DD"},
+			expect: Tabdata{
+				headers: []string{
+					"ONE", "TWO", "THREE",
+				},
+				entries: [][]string{
+					{"19191", "EDD 1", "x"},
+				},
+			},
+		},
+
+		{
+			name:   "many-fields-inverted",
+			filter: []string{"one=19", "two=DD"},
+			invert: true,
+			expect: Tabdata{
+				headers: []string{
+					"ONE", "TWO", "THREE",
+				},
+				entries: [][]string{
+					{"asd", "igig", "cxxxncnc"},
+					{"8d8", "AN 1", "y"},
+				},
+			},
+		},
+	}
+
+	for _, inputdata := range input {
+		testname := fmt.Sprintf("filter-by-fields-%s", inputdata.name)
+
+		t.Run(testname, func(t *testing.T) {
+			conf := cfg.Config{Rawfilters: inputdata.filter, InvertMatch: inputdata.invert}
+
+			err := conf.PrepareFilters()
+			if err != nil {
+				t.Errorf("PrepareFilters returned error: %s", err)
+			}
+
+			data, _, _ := FilterByFields(conf, data)
+			if !reflect.DeepEqual(data, inputdata.expect) {
+				t.Errorf("Filtered data does not match expected data:\ngot: %+v\nexp: %+v", data, inputdata.expect)
+			}
+		})
+	}
+}

lib/helpers.go

@@ -20,13 +20,14 @@ package lib
 import (
 	"errors"
 	"fmt"
-	"github.com/gookit/color"
-	"github.com/tlinden/tablizer/cfg"
 	"os"
 	"regexp"
 	"sort"
 	"strconv"
 	"strings"
+
+	"github.com/gookit/color"
+	"github.com/tlinden/tablizer/cfg"
 )
 
 func contains(s []int, e int) bool {
@@ -35,79 +36,87 @@ func contains(s []int, e int) bool {
 			return true
 		}
 	}
+
 	return false
 }
 
 // parse columns list given  with -c, modifies config.UseColumns based
 // on eventually given regex
-func PrepareColumns(c *cfg.Config, data *Tabdata) error {
+func PrepareColumns(conf *cfg.Config, data *Tabdata) error {
-	if len(c.Columns) > 0 {
+	if conf.Columns == "" {
-		for _, use := range strings.Split(c.Columns, ",") {
+		return nil
-			if len(use) == 0 {
+	}
-				msg := fmt.Sprintf("Could not parse columns list %s: empty column", c.Columns)
+
+	for _, use := range strings.Split(conf.Columns, ",") {
+		if len(use) == 0 {
+			return fmt.Errorf("could not parse columns list %s: empty column", conf.Columns)
+		}
+
+		usenum, err := strconv.Atoi(use)
+		if err != nil {
+			// might be a regexp
+			colPattern, err := regexp.Compile(use)
+			if err != nil {
+				msg := fmt.Sprintf("Could not parse columns list %s: %v", conf.Columns, err)
+
 				return errors.New(msg)
 			}
 
-			usenum, err := strconv.Atoi(use)
+			// find matching header fields
-			if err != nil {
+			for i, head := range data.headers {
-				// might be a regexp
+				if colPattern.MatchString(head) {
-				colPattern, err := regexp.Compile(use)
+					conf.UseColumns = append(conf.UseColumns, i+1)
-				if err != nil {
-					msg := fmt.Sprintf("Could not parse columns list %s: %v", c.Columns, err)
-					return errors.New(msg)
 				}
-
-				// find matching header fields
-				for i, head := range data.headers {
-					if colPattern.MatchString(head) {
-						c.UseColumns = append(c.UseColumns, i+1)
-					}
-
-				}
-			} else {
-				// we digress from go  best practises here, because if
-				// a colum spec is not a number, we process them above
-				// inside the err handler  for atoi(). so only add the
-				// number, if it's really just a number.
-				c.UseColumns = append(c.UseColumns, usenum)
 			}
+		} else {
+			// we digress from go  best practises here, because if
+			// a colum spec is not a number, we process them above
+			// inside the err handler  for atoi(). so only add the
+			// number, if it's really just a number.
+			conf.UseColumns = append(conf.UseColumns, usenum)
 		}
-
-		// deduplicate: put all values into a map (value gets map key)
-		// thereby  removing duplicates,  extract keys into  new slice
-		// and sort it
-		imap := make(map[int]int, len(c.UseColumns))
-		for _, i := range c.UseColumns {
-			imap[i] = 0
-		}
-		c.UseColumns = nil
-		for k := range imap {
-			c.UseColumns = append(c.UseColumns, k)
-		}
-		sort.Ints(c.UseColumns)
 	}
+
+	// deduplicate: put all values into a map (value gets map key)
+	// thereby  removing duplicates,  extract keys into  new slice
+	// and sort it
+	imap := make(map[int]int, len(conf.UseColumns))
+	for _, i := range conf.UseColumns {
+		imap[i] = 0
+	}
+
+	conf.UseColumns = nil
+
+	for k := range imap {
+		conf.UseColumns = append(conf.UseColumns, k)
+	}
+
+	sort.Ints(conf.UseColumns)
+
 	return nil
 }
 
 // prepare headers: add numbers to headers
-func numberizeAndReduceHeaders(c cfg.Config, data *Tabdata) {
+func numberizeAndReduceHeaders(conf cfg.Config, data *Tabdata) {
 	numberedHeaders := []string{}
 	maxwidth := 0 // start from scratch, so we only look at displayed column widths
 
-	for i, head := range data.headers {
+	for idx, head := range data.headers {
-		headlen := 0
+		var headlen int
-		if len(c.Columns) > 0 {
+
+		if len(conf.Columns) > 0 {
 			// -c specified
-			if !contains(c.UseColumns, i+1) {
+			if !contains(conf.UseColumns, idx+1) {
 				// ignore this one
 				continue
 			}
 		}
-		if c.NoNumbering {
+
+		if conf.NoNumbering {
 			numberedHeaders = append(numberedHeaders, head)
 			headlen = len(head)
 		} else {
-			numhead := fmt.Sprintf("%s(%d)", head, i+1)
+			numhead := fmt.Sprintf("%s(%d)", head, idx+1)
 			headlen = len(numhead)
 			numberedHeaders = append(numberedHeaders, numhead)
 		}
@@ -116,49 +125,94 @@ func numberizeAndReduceHeaders(c cfg.Config, data *Tabdata) {
 			maxwidth = headlen
 		}
 	}
+
 	data.headers = numberedHeaders
+
 	if data.maxwidthHeader != maxwidth && maxwidth > 0 {
 		data.maxwidthHeader = maxwidth
 	}
 }
 
 // exclude columns, if any
-func reduceColumns(c cfg.Config, data *Tabdata) {
+func reduceColumns(conf cfg.Config, data *Tabdata) {
-	if len(c.Columns) > 0 {
+	if len(conf.Columns) > 0 {
 		reducedEntries := [][]string{}
+
 		var reducedEntry []string
+
 		for _, entry := range data.entries {
 			reducedEntry = nil
+
 			for i, value := range entry {
-				if !contains(c.UseColumns, i+1) {
+				if !contains(conf.UseColumns, i+1) {
 					continue
 				}
 
 				reducedEntry = append(reducedEntry, value)
 			}
+
 			reducedEntries = append(reducedEntries, reducedEntry)
 		}
+
 		data.entries = reducedEntries
 	}
 }
 
+// FIXME: remove this when we only use Tablewriter and strip in ParseFile()!
 func trimRow(row []string) []string {
-	// FIXME: remove this when we only use Tablewriter and strip in ParseFile()!
+	var fixedrow = make([]string, len(row))
-	var fixedrow []string
+
-	for _, cell := range row {
+	for idx, cell := range row {
-		fixedrow = append(fixedrow, strings.TrimSpace(cell))
+		fixedrow[idx] = strings.TrimSpace(cell)
 	}
 
 	return fixedrow
 }
 
-func colorizeData(c cfg.Config, output string) string {
+// FIXME: refactor this beast!
-	if len(c.Pattern) > 0 && !c.NoColor && color.IsConsole(os.Stdout) {
+func colorizeData(conf cfg.Config, output string) string {
-		r := regexp.MustCompile("(" + c.Pattern + ")")
+	switch {
+	case conf.UseHighlight && color.IsConsole(os.Stdout):
+		highlight := true
+		colorized := ""
+		first := true
+
+		for _, line := range strings.Split(output, "\n") {
+			if highlight {
+				if first {
+					// we  need to add  two spaces to the  header line
+					//  because tablewriter omits them for some reason
+					//  in pprint mode. This doesn't matter as long as
+					//  we don't use colorization. But with colors the
+					// missing spaces can be seen.
+					if conf.OutputMode == cfg.ASCII {
+						line += "  "
+					}
+
+					line = conf.HighlightHdrStyle.Sprint(line)
+					first = false
+				} else {
+					line = conf.HighlightStyle.Sprint(line)
+				}
+			} else {
+				line = conf.NoHighlightStyle.Sprint(line)
+			}
+
+			highlight = !highlight
+
+			colorized += line + "\n"
+		}
+
+		return colorized
+
+	case len(conf.Pattern) > 0 && !conf.NoColor && color.IsConsole(os.Stdout):
+		r := regexp.MustCompile("(" + conf.Pattern + ")")
+
 		return r.ReplaceAllStringFunc(output, func(in string) string {
-			return c.ColorStyle.Sprint(in)
+			return conf.ColorStyle.Sprint(in)
 		})
-	} else {
+
+	default:
 		return output
 	}
 }

lib/helpers_test.go

@@ -19,9 +19,10 @@ package lib
 
 import (
 	"fmt"
-	"github.com/tlinden/tablizer/cfg"
 	"reflect"
 	"testing"
+
+	"github.com/tlinden/tablizer/cfg"
 )
 
 func TestContains(t *testing.T) {
@@ -71,18 +72,18 @@ func TestPrepareColumns(t *testing.T) {
 		{"[a-z,4,5", []int{4, 5}, true}, // invalid regexp
 	}
 
-	for _, tt := range tests {
+	for _, testdata := range tests {
-		testname := fmt.Sprintf("PrepareColumns-%s-%t", tt.input, tt.wanterror)
+		testname := fmt.Sprintf("PrepareColumns-%s-%t", testdata.input, testdata.wanterror)
 		t.Run(testname, func(t *testing.T) {
-			c := cfg.Config{Columns: tt.input}
+			conf := cfg.Config{Columns: testdata.input}
-			err := PrepareColumns(&c, &data)
+			err := PrepareColumns(&conf, &data)
 			if err != nil {
-				if !tt.wanterror {
+				if !testdata.wanterror {
 					t.Errorf("got error: %v", err)
 				}
 			} else {
-				if !reflect.DeepEqual(c.UseColumns, tt.exp) {
+				if !reflect.DeepEqual(conf.UseColumns, testdata.exp) {
-					t.Errorf("got: %v, expected: %v", c.UseColumns, tt.exp)
+					t.Errorf("got: %v, expected: %v", conf.UseColumns, testdata.exp)
 				}
 			}
 		})
@@ -114,14 +115,16 @@ func TestReduceColumns(t *testing.T) {
 
 	input := [][]string{{"a", "b", "c"}}
 
-	for _, tt := range tests {
+	for _, testdata := range tests {
-		testname := fmt.Sprintf("reduce-columns-by-%+v", tt.columns)
+		testname := fmt.Sprintf("reduce-columns-by-%+v", testdata.columns)
+
 		t.Run(testname, func(t *testing.T) {
-			c := cfg.Config{Columns: "x", UseColumns: tt.columns}
+			c := cfg.Config{Columns: "x", UseColumns: testdata.columns}
 			data := Tabdata{entries: input}
 			reduceColumns(c, &data)
-			if !reflect.DeepEqual(data.entries, tt.expect) {
+			if !reflect.DeepEqual(data.entries, testdata.expect) {
-				t.Errorf("reduceColumns returned invalid data:\ngot: %+v\nexp: %+v", data.entries, tt.expect)
+				t.Errorf("reduceColumns returned invalid data:\ngot: %+v\nexp: %+v",
+					data.entries, testdata.expect)
 			}
 		})
 	}
@@ -142,15 +145,17 @@ func TestNumberizeHeaders(t *testing.T) {
 		{[]string{"ONE", "TWO"}, []int{1, 2}, true},
 	}
 
-	for _, tt := range tests {
+	for _, testdata := range tests {
-		testname := fmt.Sprintf("numberize-headers-columns-%+v-nonum-%t", tt.columns, tt.nonum)
+		testname := fmt.Sprintf("numberize-headers-columns-%+v-nonum-%t",
+			testdata.columns, testdata.nonum)
+
 		t.Run(testname, func(t *testing.T) {
-			c := cfg.Config{Columns: "x", UseColumns: tt.columns, NoNumbering: tt.nonum}
+			conf := cfg.Config{Columns: "x", UseColumns: testdata.columns, NoNumbering: testdata.nonum}
 			usedata := data
-			numberizeAndReduceHeaders(c, &usedata)
+			numberizeAndReduceHeaders(conf, &usedata)
-			if !reflect.DeepEqual(usedata.headers, tt.expect) {
+			if !reflect.DeepEqual(usedata.headers, testdata.expect) {
 				t.Errorf("numberizeAndReduceHeaders returned invalid data:\ngot: %+v\nexp: %+v",
-					usedata.headers, tt.expect)
+					usedata.headers, testdata.expect)
 			}
 		})
 	}

lib/io.go

@@ -19,91 +19,96 @@ package lib
 
 import (
 	"errors"
-	"github.com/tlinden/tablizer/cfg"
+	"fmt"
 	"io"
 	"os"
+
+	"github.com/tlinden/tablizer/cfg"
 )
 
-func ProcessFiles(c *cfg.Config, args []string) error {
+const RWRR = 0755
-	fds, pattern, err := determineIO(c, args)
+
+func ProcessFiles(conf *cfg.Config, args []string) error {
+	fds, pattern, err := determineIO(conf, args)
 
 	if err != nil {
 		return err
 	}
 
-	if err := c.PreparePattern(pattern); err != nil {
+	if err := conf.PreparePattern(pattern); err != nil {
 		return err
 	}
 
 	for _, fd := range fds {
-		data, err := Parse(*c, fd)
+		data, err := Parse(*conf, fd)
 		if err != nil {
 			return err
 		}
 
-		err = PrepareColumns(c, &data)
+		err = PrepareColumns(conf, &data)
 		if err != nil {
 			return err
 		}
 
-		printData(os.Stdout, *c, &data)
+		printData(os.Stdout, *conf, &data)
 	}
 
 	return nil
 }
 
-func determineIO(c *cfg.Config, args []string) ([]io.Reader, string, error) {
+func determineIO(conf *cfg.Config, args []string) ([]io.Reader, string, error) {
+	var filehandles []io.Reader
+
 	var pattern string
-	var fds []io.Reader
+
 	var haveio bool
 
 	stat, _ := os.Stdin.Stat()
 	if (stat.Mode() & os.ModeCharDevice) == 0 {
 		// we're reading from STDIN, which takes precedence over file args
-		fds = append(fds, os.Stdin)
+		filehandles = append(filehandles, os.Stdin)
+
 		if len(args) > 0 {
 			// ignore any args > 1
 			pattern = args[0]
-			c.Pattern = args[0] // used for colorization by printData()
+			conf.Pattern = args[0] // used for colorization by printData()
 		}
+
 		haveio = true
-	} else {
+	} else if len(args) > 0 {
-		if len(args) > 0 {
+		// threre were args left, take a look
-			// threre were args left, take a look
+		if args[0] == "-" {
-			if args[0] == "-" {
+			// in traditional unix programs a dash denotes STDIN (forced)
-				// in traditional unix programs a dash denotes STDIN (forced)
+			filehandles = append(filehandles, os.Stdin)
-				fds = append(fds, os.Stdin)
+			haveio = true
-				haveio = true
+		} else {
-			} else {
+			if _, err := os.Stat(args[0]); err != nil {
-				if _, err := os.Stat(args[0]); err != nil {
+				// first  one is  not a  file, consider  it as  regexp and
-					// first  one is  not a  file, consider  it as  regexp and
+				// shift arg list
-					// shift arg list
+				pattern = args[0]
-					pattern = args[0]
+				conf.Pattern = args[0] // used for colorization by printData()
-					c.Pattern = args[0] // used for colorization by printData()
+				args = args[1:]
-					args = args[1:]
+			}
-				}
 
-				if len(args) > 0 {
+			if len(args) > 0 {
-					// consider any other args as files
+				// consider any other args as files
-					for _, file := range args {
+				for _, file := range args {
+					filehandle, err := os.OpenFile(file, os.O_RDONLY, RWRR)
 
-						fd, err := os.OpenFile(file, os.O_RDONLY, 0755)
+					if err != nil {
-
+						return nil, "", fmt.Errorf("failed to read input file %s: %w", file, err)
-						if err != nil {
-							return nil, "", err
-						}
-
-						fds = append(fds, fd)
-						haveio = true
 					}
+
+					filehandles = append(filehandles, filehandle)
+					haveio = true
 				}
 			}
 		}
 	}
 
 	if !haveio {
-		return nil, "", errors.New("No file specified and nothing to read on stdin!")
+		return nil, "", errors.New("no file specified and nothing to read on stdin")
 	}
 
-	return fds, pattern, nil
+	return filehandles, pattern, nil
 }

lib/lisp.go

@@ -0,0 +1,313 @@
+/*
+Copyright © 2023 Thomas von Dein
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package lib
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+
+	"github.com/glycerine/zygomys/zygo"
+	"github.com/tlinden/tablizer/cfg"
+)
+
+/*
+needs to be global because we can't feed an cfg object to AddHook()
+which is being called from user lisp code
+*/
+var Hooks map[string][]*zygo.SexpSymbol
+
+/*
+AddHook() (called addhook from lisp code)  can be used by the user to
+add a function to one of the available hooks provided by tablizer.
+*/
+func AddHook(env *zygo.Zlisp, name string, args []zygo.Sexp) (zygo.Sexp, error) {
+	var hookname string
+
+	if len(args) < 2 {
+		return zygo.SexpNull, errors.New("argument of %add-hook should be: %hook-name %your-function")
+	}
+
+	switch sexptype := args[0].(type) {
+	case *zygo.SexpSymbol:
+		if !HookExists(sexptype.Name()) {
+			return zygo.SexpNull, errors.New("Unknown hook " + sexptype.Name())
+		}
+
+		hookname = sexptype.Name()
+
+	default:
+		return zygo.SexpNull, errors.New("hook name must be a symbol ")
+	}
+
+	switch sexptype := args[1].(type) {
+	case *zygo.SexpSymbol:
+		_, exists := Hooks[hookname]
+		if !exists {
+			Hooks[hookname] = []*zygo.SexpSymbol{sexptype}
+		} else {
+			Hooks[hookname] = append(Hooks[hookname], sexptype)
+		}
+
+	default:
+		return zygo.SexpNull, errors.New("hook function must be a symbol ")
+	}
+
+	return zygo.SexpNull, nil
+}
+
+/*
+Check if a hook exists
+*/
+func HookExists(key string) bool {
+	for _, hook := range cfg.ValidHooks {
+		if hook == key {
+			return true
+		}
+	}
+
+	return false
+}
+
+/*
+ * Basic sanity checks and load lisp file
+ */
+func LoadAndEvalFile(env *zygo.Zlisp, path string) error {
+	if strings.HasSuffix(path, `.zy`) {
+		code, err := os.ReadFile(path)
+		if err != nil {
+			return fmt.Errorf("failed to read lisp file %s: %w", path, err)
+		}
+
+		// FIXME: check what res (_ here) could be and mean
+		_, err = env.EvalString(string(code))
+		if err != nil {
+			log.Fatal(env.GetStackTrace(err))
+		}
+	}
+
+	return nil
+}
+
+/*
+ * Setup lisp interpreter environment
+ */
+func SetupLisp(conf *cfg.Config) error {
+	// iterate over load-path and evaluate all *.zy files there, if any
+	// we ignore if load-path does not exist, which is the default anyway
+	path, err := os.Stat(conf.LispLoadPath)
+	if os.IsNotExist(err) {
+		return nil
+	}
+
+	// init global hooks
+	Hooks = make(map[string][]*zygo.SexpSymbol)
+
+	// init sandbox
+	env := zygo.NewZlispSandbox()
+	env.AddFunction("addhook", AddHook)
+
+	if !path.IsDir() {
+		// load single lisp file
+		err = LoadAndEvalFile(env, conf.LispLoadPath)
+		if err != nil {
+			return err
+		}
+	} else {
+		// load all lisp file in load dir
+		dir, err := os.ReadDir(conf.LispLoadPath)
+		if err != nil {
+			return fmt.Errorf("failed to read lisp dir %s: %w",
+				conf.LispLoadPath, err)
+		}
+
+		for _, entry := range dir {
+			if !entry.IsDir() {
+				err := LoadAndEvalFile(env, conf.LispLoadPath+"/"+entry.Name())
+				if err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	RegisterLib(env)
+
+	conf.Lisp = env
+
+	return nil
+}
+
+/*
+Execute every user lisp function registered as filter hook.
+
+Each function is given the current line as argument and is expected to
+return a boolean.   True indicates to keep the line,  false to skip
+it.
+
+If there  are multiple such  functions registered, then the  first one
+returning false wins,  that is if each function returns  true the line
+will  be kept,  if at  least one  of them  returns false,  it will  be
+skipped.
+*/
+func RunFilterHooks(conf cfg.Config, line string) (bool, error) {
+	for _, hook := range Hooks["filter"] {
+		var result bool
+
+		conf.Lisp.Clear()
+
+		res, err := conf.Lisp.EvalString(fmt.Sprintf("(%s `%s`)", hook.Name(), line))
+		if err != nil {
+			return false, fmt.Errorf("failed to evaluate hook loader: %w", err)
+		}
+
+		switch sexptype := res.(type) {
+		case *zygo.SexpBool:
+			result = sexptype.Val
+		default:
+			return false, fmt.Errorf("filter hook shall return bool")
+		}
+
+		if !result {
+			// the first hook which returns false leads to complete false
+			return result, nil
+		}
+	}
+
+	// if no hook returned false, we succeed and accept the given line
+	return true, nil
+}
+
+/*
+These hooks get the data (Tabdata) readily processed by tablizer as
+argument. They are expected to  return a SexpPair containing a boolean
+denoting  if  the data  has  been  modified  and the  actual  modified
+data. Columns must  be the same, rows may differ.  Cells may also have
+been modified.
+
+Replaces the  internal data structure  Tabdata with the  user supplied
+version.
+
+Only one process hook function is supported.
+
+The somewhat complicated code is being  caused by the fact, that we
+need to convert our internal structure  to a lisp variable and vice
+versa afterwards.
+*/
+func RunProcessHooks(conf cfg.Config, data Tabdata) (Tabdata, bool, error) {
+	var userdata Tabdata
+
+	lisplist := []zygo.Sexp{}
+
+	if len(Hooks["process"]) == 0 {
+		return userdata, false, nil
+	}
+
+	if len(Hooks["process"]) > 1 {
+		fmt.Println("Warning: only one process hook is allowed!")
+	}
+
+	// there are hook[s] installed, convert the go data structure 'data to lisp
+	for _, row := range data.entries {
+		var entry zygo.SexpHash
+
+		for idx, cell := range row {
+			err := entry.HashSet(&zygo.SexpStr{S: data.headers[idx]}, &zygo.SexpStr{S: cell})
+			if err != nil {
+				return userdata, false, fmt.Errorf("failed to convert to lisp data: %w", err)
+			}
+		}
+
+		lisplist = append(lisplist, &entry)
+	}
+
+	// we need to add it to the env so that the function can use the struct directly
+	conf.Lisp.AddGlobal("data", &zygo.SexpArray{Val: lisplist, Env: conf.Lisp})
+
+	// execute the actual hook
+	hook := Hooks["process"][0]
+
+	conf.Lisp.Clear()
+
+	var result bool
+
+	res, err := conf.Lisp.EvalString(fmt.Sprintf("(%s data)", hook.Name()))
+	if err != nil {
+		return userdata, false, fmt.Errorf("failed to eval lisp loader: %w", err)
+	}
+
+	// we expect (bool, array(hash)) as return from the function
+	switch sexptype := res.(type) {
+	case *zygo.SexpPair:
+		switch th := sexptype.Head.(type) {
+		case *zygo.SexpBool:
+			result = th.Val
+		default:
+			return userdata, false, errors.New("xpect (bool, array(hash)) as return value")
+		}
+
+		switch sexptailtype := sexptype.Tail.(type) {
+		case *zygo.SexpArray:
+			lisplist = sexptailtype.Val
+		default:
+			return userdata, false, errors.New("expect (bool, array(hash)) as return value ")
+		}
+	default:
+		return userdata, false, errors.New("filter hook shall return array of hashes ")
+	}
+
+	if !result {
+		// no further processing required
+		return userdata, result, nil
+	}
+
+	// finally convert lispdata back to Tabdata
+	for _, item := range lisplist {
+		row := []string{}
+
+		switch hash := item.(type) {
+		case *zygo.SexpHash:
+			for _, header := range data.headers {
+				entry, err := hash.HashGetDefault(
+					conf.Lisp,
+					&zygo.SexpStr{S: header},
+					&zygo.SexpStr{S: ""})
+				if err != nil {
+					return userdata, false, fmt.Errorf("failed to get lisp hash entry: %w", err)
+				}
+
+				switch sexptype := entry.(type) {
+				case *zygo.SexpStr:
+					row = append(row, sexptype.S)
+				default:
+					return userdata, false, errors.New("hsh values should be string ")
+				}
+			}
+		default:
+			return userdata, false, errors.New("rturned array should contain hashes ")
+		}
+
+		userdata.entries = append(userdata.entries, row)
+	}
+
+	userdata.headers = data.headers
+
+	return userdata, result, nil
+}

lib/lisplib.go

@@ -0,0 +1,88 @@
+/*
+Copyright © 2023 Thomas von Dein
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package lib
+
+import (
+	"errors"
+	"fmt"
+	"regexp"
+	"strconv"
+
+	"github.com/glycerine/zygomys/zygo"
+)
+
+func Splice2SexpList(list []string) zygo.Sexp {
+	slist := []zygo.Sexp{}
+
+	for _, item := range list {
+		slist = append(slist, &zygo.SexpStr{S: item})
+	}
+
+	return zygo.MakeList(slist)
+}
+
+func StringReSplit(env *zygo.Zlisp, name string, args []zygo.Sexp) (zygo.Sexp, error) {
+	if len(args) < 2 {
+		return zygo.SexpNull, errors.New("expecting 2 arguments")
+	}
+
+	var separator, input string
+
+	switch t := args[0].(type) {
+	case *zygo.SexpStr:
+		input = t.S
+	default:
+		return zygo.SexpNull, errors.New("second argument must be a string")
+	}
+
+	switch t := args[1].(type) {
+	case *zygo.SexpStr:
+		separator = t.S
+	default:
+		return zygo.SexpNull, errors.New("first argument must be a string")
+	}
+
+	sep := regexp.MustCompile(separator)
+
+	return Splice2SexpList(sep.Split(input, -1)), nil
+}
+
+func String2Int(env *zygo.Zlisp, name string, args []zygo.Sexp) (zygo.Sexp, error) {
+	var number int
+
+	switch t := args[0].(type) {
+	case *zygo.SexpStr:
+		num, err := strconv.Atoi(t.S)
+
+		if err != nil {
+			return zygo.SexpNull, fmt.Errorf("failed to convert string to number: %w", err)
+		}
+
+		number = num
+
+	default:
+		return zygo.SexpNull, errors.New("argument must be a string")
+	}
+
+	return &zygo.SexpInt{Val: int64(number)}, nil
+}
+
+func RegisterLib(env *zygo.Zlisp) {
+	env.AddFunction("resplit", StringReSplit)
+	env.AddFunction("atoi", String2Int)
+}

lib/parser.go

@@ -1,5 +1,5 @@
 /*
-Copyright © 2022 Thomas von Dein
+Copyright © 2022-2024 Thomas von Dein
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -20,61 +20,44 @@ package lib
 import (
 	"bufio"
 	"encoding/csv"
-	"errors"
 	"fmt"
-	"github.com/alecthomas/repr"
-	"github.com/tlinden/tablizer/cfg"
 	"io"
 	"regexp"
 	"strings"
+
+	"github.com/alecthomas/repr"
+	"github.com/tlinden/tablizer/cfg"
 )
 
 /*
-   Parser switch
+Parser switch
 */
-func Parse(c cfg.Config, input io.Reader) (Tabdata, error) {
+func Parse(conf cfg.Config, input io.Reader) (Tabdata, error) {
-	if len(c.Separator) == 1 {
+	if len(conf.Separator) == 1 {
-		return parseCSV(c, input)
+		return parseCSV(conf, input)
 	}
 
-	return parseTabular(c, input)
+	return parseTabular(conf, input)
 }
 
 /*
-   Parse CSV input.
+Parse CSV input.
 */
-func parseCSV(c cfg.Config, input io.Reader) (Tabdata, error) {
+func parseCSV(conf cfg.Config, input io.Reader) (Tabdata, error) {
-	var content io.Reader = input
 	data := Tabdata{}
 
-	if len(c.Pattern) > 0 {
+	// apply pattern, if any
-		scanner := bufio.NewScanner(input)
+	content, err := FilterByPattern(conf, input)
-		lines := []string{}
+	if err != nil {
-		hadFirst := false
+		return data, err
-		for scanner.Scan() {
-			line := strings.TrimSpace(scanner.Text())
-			if hadFirst {
-				// don't match 1st line, it's the header
-				if c.PatternR.MatchString(line) == c.InvertMatch {
-					// by default  -v is false, so if a  line does NOT
-					// match the pattern, we will ignore it. However,
-					// if the user specified -v, the matching is inverted,
-					// so we ignore all lines, which DO match.
-					continue
-				}
-			}
-			lines = append(lines, line)
-			hadFirst = true
-		}
-		content = strings.NewReader(strings.Join(lines, "\n"))
 	}
 
 	csvreader := csv.NewReader(content)
-	csvreader.Comma = rune(c.Separator[0])
+	csvreader.Comma = rune(conf.Separator[0])
 
 	records, err := csvreader.ReadAll()
 	if err != nil {
-		return data, errors.Unwrap(fmt.Errorf("Could not parse CSV input: %w", err))
+		return data, fmt.Errorf("could not parse CSV input: %w", err)
 	}
 
 	if len(records) >= 1 {
@@ -94,19 +77,29 @@ func parseCSV(c cfg.Config, input io.Reader) (Tabdata, error) {
 		}
 	}
 
+	// apply user defined lisp process hooks, if any
+	userdata, changed, err := RunProcessHooks(conf, data)
+	if err != nil {
+		return data, fmt.Errorf("failed to apply filter hook: %w", err)
+	}
+
+	if changed {
+		data = userdata
+	}
+
 	return data, nil
 }
 
 /*
-   Parse tabular input.
+Parse tabular input.
 */
-func parseTabular(c cfg.Config, input io.Reader) (Tabdata, error) {
+func parseTabular(conf cfg.Config, input io.Reader) (Tabdata, error) {
 	data := Tabdata{}
 
 	var scanner *bufio.Scanner
 
 	hadFirst := false
-	separate := regexp.MustCompile(c.Separator)
+	separate := regexp.MustCompile(conf.Separator)
 
 	scanner = bufio.NewScanner(input)
 
@@ -123,10 +116,6 @@ func parseTabular(c cfg.Config, input io.Reader) (Tabdata, error) {
 
 			// process all header fields
 			for _, part := range parts {
-				// if Debug {
-				// 	fmt.Printf("Part: <%s>\n", string(line[beg:part[0]]))
-				//}
-
 				// register widest header field
 				headerlen := len(part)
 				if headerlen > data.maxwidthHeader {
@@ -141,14 +130,24 @@ func parseTabular(c cfg.Config, input io.Reader) (Tabdata, error) {
 			}
 		} else {
 			// data processing
-			if len(c.Pattern) > 0 {
+			if conf.Pattern != "" && matchPattern(conf, line) == conf.InvertMatch {
-				if c.PatternR.MatchString(line) == c.InvertMatch {
+				// by default  -v is false, so if a  line does NOT
-					// by default  -v is false, so if a  line does NOT
+				// match the pattern, we will ignore it. However,
-					// match the pattern, we will ignore it. However,
+				// if the user specified -v, the matching is inverted,
-					// if the user specified -v, the matching is inverted,
+				// so we ignore all lines, which DO match.
-					// so we ignore all lines, which DO match.
+				continue
-					continue
+			}
-				}
+
+			// apply user defined lisp filters, if any
+			accept, err := RunFilterHooks(conf, line)
+			if err != nil {
+				return data, fmt.Errorf("failed to apply filter hook: %w", err)
+			}
+
+			if !accept {
+				//  IF there  are filter  hook[s] and  IF one  of them
+				// returns false on the current line, reject it
+				continue
 			}
 
 			idx := 0 // we cannot use the header index, because we could exclude columns
@@ -172,10 +171,30 @@ func parseTabular(c cfg.Config, input io.Reader) (Tabdata, error) {
 	}
 
 	if scanner.Err() != nil {
-		return data, errors.Unwrap(fmt.Errorf("Failed to read from io.Reader: %w", scanner.Err()))
+		return data, fmt.Errorf("failed to read from io.Reader: %w", scanner.Err())
 	}
 
-	if c.Debug {
+	// filter by field filters, if any
+	filtereddata, changed, err := FilterByFields(conf, data)
+	if err != nil {
+		return data, fmt.Errorf("failed to filter fields: %w", err)
+	}
+
+	if changed {
+		data = filtereddata
+	}
+
+	// apply user defined lisp process hooks, if any
+	userdata, changed, err := RunProcessHooks(conf, data)
+	if err != nil {
+		return data, fmt.Errorf("failed to apply filter hook: %w", err)
+	}
+
+	if changed {
+		data = userdata
+	}
+
+	if conf.Debug {
 		repr.Print(data)
 	}
 

lib/parser_test.go

@@ -19,10 +19,11 @@ package lib
 
 import (
 	"fmt"
-	"github.com/tlinden/tablizer/cfg"
 	"reflect"
 	"strings"
 	"testing"
+
+	"github.com/tlinden/tablizer/cfg"
 )
 
 var input = []struct {
@@ -61,12 +62,12 @@ func TestParser(t *testing.T) {
 		},
 	}
 
-	for _, in := range input {
+	for _, testdata := range input {
-		testname := fmt.Sprintf("parse-%s", in.name)
+		testname := fmt.Sprintf("parse-%s", testdata.name)
 		t.Run(testname, func(t *testing.T) {
-			readFd := strings.NewReader(strings.TrimSpace(in.text))
+			readFd := strings.NewReader(strings.TrimSpace(testdata.text))
-			c := cfg.Config{Separator: in.separator}
+			conf := cfg.Config{Separator: testdata.separator}
-			gotdata, err := Parse(c, readFd)
+			gotdata, err := Parse(conf, readFd)
 
 			if err != nil {
 				t.Errorf("Parser returned error: %s\nData processed so far: %+v", err, gotdata)
@@ -103,28 +104,28 @@ func TestParserPatternmatching(t *testing.T) {
 		},
 	}
 
-	for _, in := range input {
+	for _, inputdata := range input {
-		for _, tt := range tests {
+		for _, testdata := range tests {
 			testname := fmt.Sprintf("parse-%s-with-pattern-%s-inverted-%t",
-				in.name, tt.pattern, tt.invert)
+				inputdata.name, testdata.pattern, testdata.invert)
 			t.Run(testname, func(t *testing.T) {
-				c := cfg.Config{InvertMatch: tt.invert, Pattern: tt.pattern,
+				conf := cfg.Config{InvertMatch: testdata.invert, Pattern: testdata.pattern,
-					Separator: in.separator}
+					Separator: inputdata.separator}
 
-				_ = c.PreparePattern(tt.pattern)
+				_ = conf.PreparePattern(testdata.pattern)
 
-				readFd := strings.NewReader(strings.TrimSpace(in.text))
+				readFd := strings.NewReader(strings.TrimSpace(inputdata.text))
-				gotdata, err := Parse(c, readFd)
+				gotdata, err := Parse(conf, readFd)
 
 				if err != nil {
-					if !tt.want {
+					if !testdata.want {
 						t.Errorf("Parser returned error: %s\nData processed so far: %+v",
 							err, gotdata)
 					}
 				} else {
-					if !reflect.DeepEqual(tt.entries, gotdata.entries) {
+					if !reflect.DeepEqual(testdata.entries, gotdata.entries) {
 						t.Errorf("Parser returned invalid data (pattern: %s, invert: %t)\nExp: %+v\nGot: %+v\n",
-							tt.pattern, tt.invert, tt.entries, gotdata.entries)
+							testdata.pattern, testdata.invert, testdata.entries, gotdata.entries)
 					}
 				}
 			})
@@ -151,8 +152,8 @@ asd    igig
 19191  EDD 1  X`
 
 	readFd := strings.NewReader(strings.TrimSpace(table))
-	c := cfg.Config{Separator: cfg.DefaultSeparator}
+	conf := cfg.Config{Separator: cfg.DefaultSeparator}
-	gotdata, err := Parse(c, readFd)
+	gotdata, err := Parse(conf, readFd)
 
 	if err != nil {
 		t.Errorf("Parser returned error: %s\nData processed so far: %+v", err, gotdata)
@@ -160,6 +161,6 @@ asd    igig
 
 	if !reflect.DeepEqual(data, gotdata) {
 		t.Errorf("Parser returned invalid data, Regex: %s\nExp: %+v\nGot: %+v\n",
-			c.Separator, data, gotdata)
+			conf.Separator, data, gotdata)
 	}
 }

lib/printer.go

@@ -20,62 +20,60 @@ package lib
 import (
 	"encoding/csv"
 	"fmt"
-	"github.com/gookit/color"
-	"github.com/olekukonko/tablewriter"
-	"github.com/tlinden/tablizer/cfg"
-	"gopkg.in/yaml.v3"
 	"io"
 	"log"
 	"regexp"
 	"strconv"
 	"strings"
+
+	"github.com/gookit/color"
+	"github.com/olekukonko/tablewriter"
+	"github.com/tlinden/tablizer/cfg"
+	"gopkg.in/yaml.v3"
 )
 
-func printData(w io.Writer, c cfg.Config, data *Tabdata) {
+func printData(writer io.Writer, conf cfg.Config, data *Tabdata) {
-	// some output preparations:
-
 	// add numbers to headers and remove this we're not interested in
-	numberizeAndReduceHeaders(c, data)
+	numberizeAndReduceHeaders(conf, data)
 
 	// remove unwanted columns, if any
-	reduceColumns(c, data)
+	reduceColumns(conf, data)
 
 	// sort the data
-	sortTable(c, data)
+	sortTable(conf, data)
 
-	switch c.OutputMode {
+	switch conf.OutputMode {
 	case cfg.Extended:
-		printExtendedData(w, c, data)
+		printExtendedData(writer, conf, data)
-	case cfg.Ascii:
+	case cfg.ASCII:
-		printAsciiData(w, c, data)
+		printASCIIData(writer, conf, data)
 	case cfg.Orgtbl:
-		printOrgmodeData(w, c, data)
+		printOrgmodeData(writer, conf, data)
 	case cfg.Markdown:
-		printMarkdownData(w, c, data)
+		printMarkdownData(writer, conf, data)
 	case cfg.Shell:
-		printShellData(w, c, data)
+		printShellData(writer, data)
 	case cfg.Yaml:
-		printYamlData(w, c, data)
+		printYamlData(writer, data)
 	case cfg.CSV:
-		printCSVData(w, c, data)
+		printCSVData(writer, data)
 	default:
-		printAsciiData(w, c, data)
+		printASCIIData(writer, conf, data)
 	}
-
 }
 
-func output(w io.Writer, str string) {
+func output(writer io.Writer, str string) {
-	fmt.Fprint(w, str)
+	fmt.Fprint(writer, str)
 }
 
 /*
 Emacs org-mode compatible table (also orgtbl-mode)
 */
-func printOrgmodeData(w io.Writer, c cfg.Config, data *Tabdata) {
+func printOrgmodeData(writer io.Writer, conf cfg.Config, data *Tabdata) {
 	tableString := &strings.Builder{}
 	table := tablewriter.NewWriter(tableString)
 
-	if !c.NoHeaders {
+	if !conf.NoHeaders {
 		table.SetHeader(data.headers)
 	}
 
@@ -99,8 +97,8 @@ func printOrgmodeData(w io.Writer, c cfg.Config, data *Tabdata) {
 	leftR := regexp.MustCompile(`(?m)^\\+`)
 	rightR := regexp.MustCompile(`\\+(?m)$`)
 
-	output(w, color.Sprint(
+	output(writer, color.Sprint(
-		colorizeData(c,
+		colorizeData(conf,
 			rightR.ReplaceAllString(
 				leftR.ReplaceAllString(tableString.String(), "|"), "|"))))
 }
@@ -108,11 +106,11 @@ func printOrgmodeData(w io.Writer, c cfg.Config, data *Tabdata) {
 /*
 Markdown table
 */
-func printMarkdownData(w io.Writer, c cfg.Config, data *Tabdata) {
+func printMarkdownData(writer io.Writer, conf cfg.Config, data *Tabdata) {
 	tableString := &strings.Builder{}
 	table := tablewriter.NewWriter(tableString)
 
-	if !c.NoHeaders {
+	if !conf.NoHeaders {
 		table.SetHeader(data.headers)
 	}
 
@@ -124,19 +122,20 @@ func printMarkdownData(w io.Writer, c cfg.Config, data *Tabdata) {
 	table.SetCenterSeparator("|")
 
 	table.Render()
-	output(w, color.Sprint(colorizeData(c, tableString.String())))
+	output(writer, color.Sprint(colorizeData(conf, tableString.String())))
 }
 
 /*
 Simple ASCII table without any borders etc, just like the input we expect
 */
-func printAsciiData(w io.Writer, c cfg.Config, data *Tabdata) {
+func printASCIIData(writer io.Writer, conf cfg.Config, data *Tabdata) {
 	tableString := &strings.Builder{}
 	table := tablewriter.NewWriter(tableString)
 
-	if !c.NoHeaders {
+	if !conf.NoHeaders {
 		table.SetHeader(data.headers)
 	}
+
 	table.AppendBulk(data.entries)
 
 	table.SetAutoWrapText(false)
@@ -148,20 +147,27 @@ func printAsciiData(w io.Writer, c cfg.Config, data *Tabdata) {
 	table.SetRowSeparator("")
 	table.SetHeaderLine(false)
 	table.SetBorder(false)
-	table.SetTablePadding("\t") // pad with tabs
 	table.SetNoWhiteSpace(true)
 
+	if !conf.UseHighlight {
+		// the tabs destroy the highlighting
+		table.SetTablePadding("\t") // pad with tabs
+	} else {
+		table.SetTablePadding("   ")
+	}
+
 	table.Render()
-	output(w, color.Sprint(colorizeData(c, tableString.String())))
+	output(writer, color.Sprint(colorizeData(conf, tableString.String())))
 }
 
 /*
 We simulate the \x command of psql (the PostgreSQL client)
 */
-func printExtendedData(w io.Writer, c cfg.Config, data *Tabdata) {
+func printExtendedData(writer io.Writer, conf cfg.Config, data *Tabdata) {
 	// needed for data output
 	format := fmt.Sprintf("%%%ds: %%s\n", data.maxwidthHeader)
 	out := ""
+
 	if len(data.entries) > 0 {
 		for _, entry := range data.entries {
 			for i, value := range entry {
@@ -172,67 +178,71 @@ func printExtendedData(w io.Writer, c cfg.Config, data *Tabdata) {
 		}
 	}
 
-	output(w, colorizeData(c, out))
+	output(writer, colorizeData(conf, out))
 }
 
 /*
 Shell output, ready to be eval'd. Just like FreeBSD stat(1)
 */
-func printShellData(w io.Writer, c cfg.Config, data *Tabdata) {
+func printShellData(writer io.Writer, data *Tabdata) {
 	out := ""
+
 	if len(data.entries) > 0 {
 		for _, entry := range data.entries {
 			shentries := []string{}
-			for i, value := range entry {
+
+			for idx, value := range entry {
 				shentries = append(shentries, fmt.Sprintf("%s=\"%s\"",
-					data.headers[i], value))
+					data.headers[idx], value))
 			}
-			out += fmt.Sprint(strings.Join(shentries, " ")) + "\n"
+
+			out += strings.Join(shentries, " ") + "\n"
 		}
 	}
 
 	// no colorization here
-	output(w, out)
+	output(writer, out)
 }
 
-func printYamlData(w io.Writer, c cfg.Config, data *Tabdata) {
+func printYamlData(writer io.Writer, data *Tabdata) {
-	type D struct {
+	type Data struct {
 		Entries []map[string]interface{} `yaml:"entries"`
 	}
 
-	d := D{}
+	yamlout := Data{}
 
 	for _, entry := range data.entries {
-		ml := map[string]interface{}{}
+		yamldata := map[string]interface{}{}
 
-		for i, entry := range entry {
+		for idx, entry := range entry {
 			style := yaml.TaggedStyle
+
 			_, err := strconv.Atoi(entry)
 			if err != nil {
 				style = yaml.DoubleQuotedStyle
 			}
 
-			ml[strings.ToLower(data.headers[i])] =
+			yamldata[strings.ToLower(data.headers[idx])] =
 				&yaml.Node{
 					Kind:  yaml.ScalarNode,
 					Style: style,
 					Value: entry}
 		}
 
-		d.Entries = append(d.Entries, ml)
+		yamlout.Entries = append(yamlout.Entries, yamldata)
 	}
 
-	yamlstr, err := yaml.Marshal(&d)
+	yamlstr, err := yaml.Marshal(&yamlout)
 
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	output(w, string(yamlstr))
+	output(writer, string(yamlstr))
 }
 
-func printCSVData(w io.Writer, c cfg.Config, data *Tabdata) {
+func printCSVData(writer io.Writer, data *Tabdata) {
-	csvout := csv.NewWriter(w)
+	csvout := csv.NewWriter(writer)
 
 	if err := csvout.Write(data.headers); err != nil {
 		log.Fatalln("error writing record to csv:", err)

lib/printer_test.go

@@ -20,10 +20,10 @@ package lib
 import (
 	"bytes"
 	"fmt"
-	//"github.com/alecthomas/repr"
-	"github.com/tlinden/tablizer/cfg"
 	"strings"
 	"testing"
+
+	"github.com/tlinden/tablizer/cfg"
 )
 
 func newData() Tabdata {
@@ -73,7 +73,7 @@ var tests = []struct {
 }{
 	// --------------------- Default settings mode tests ``
 	{
-		mode: cfg.Ascii,
+		mode: cfg.ASCII,
 		name: "default",
 		expect: `
 NAME(1)	DURATION(2)	COUNT(3)	WHEN(4)                    
@@ -250,39 +250,39 @@ DURATION(2)	WHEN(4)
 }
 
 func TestPrinter(t *testing.T) {
-	for _, tt := range tests {
+	for _, testdata := range tests {
 		testname := fmt.Sprintf("print-sortcol-%d-desc-%t-sortby-%s-mode-%d-usecolumns-%s",
-			tt.column, tt.desc, tt.sortby, tt.mode, tt.usecolstr)
+			testdata.column, testdata.desc, testdata.sortby, testdata.mode, testdata.usecolstr)
 		t.Run(testname, func(t *testing.T) {
 			// replaces os.Stdout, but we ignore it
-			var w bytes.Buffer
+			var writer bytes.Buffer
 
 			// cmd flags
-			c := cfg.Config{
+			conf := cfg.Config{
-				SortByColumn:   tt.column,
+				SortByColumn:   testdata.column,
-				SortDescending: tt.desc,
+				SortDescending: testdata.desc,
-				SortMode:       tt.sortby,
+				SortMode:       testdata.sortby,
-				OutputMode:     tt.mode,
+				OutputMode:     testdata.mode,
-				NoNumbering:    tt.nonum,
+				NoNumbering:    testdata.nonum,
-				UseColumns:     tt.usecol,
+				UseColumns:     testdata.usecol,
 				NoColor:        true,
 			}
 
-			c.ApplyDefaults()
+			conf.ApplyDefaults()
 
 			// the test checks the len!
-			if len(tt.usecol) > 0 {
+			if len(testdata.usecol) > 0 {
-				c.Columns = "yes"
+				conf.Columns = "yes"
 			} else {
-				c.Columns = ""
+				conf.Columns = ""
 			}
 
-			testdata := newData()
+			data := newData()
-			exp := strings.TrimSpace(tt.expect)
+			exp := strings.TrimSpace(testdata.expect)
 
-			printData(&w, c, &testdata)
+			printData(&writer, conf, &data)
 
-			got := strings.TrimSpace(w.String())
+			got := strings.TrimSpace(writer.String())
 
 			if got != exp {
 				t.Errorf("not rendered correctly:\n+++ got:\n%s\n+++ want:\n%s",

lib/sort.go

@@ -18,21 +18,22 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
 package lib
 
 import (
-	"github.com/araddon/dateparse"
-	"github.com/tlinden/tablizer/cfg"
 	"regexp"
 	"sort"
 	"strconv"
+
+	"github.com/araddon/dateparse"
+	"github.com/tlinden/tablizer/cfg"
 )
 
-func sortTable(c cfg.Config, data *Tabdata) {
+func sortTable(conf cfg.Config, data *Tabdata) {
-	if c.SortByColumn <= 0 {
+	if conf.SortByColumn <= 0 {
 		// no sorting wanted
 		return
 	}
 
 	// slightly modified here to match internal array indicies
-	col := c.SortByColumn
+	col := conf.SortByColumn
 
 	col-- // ui starts counting by 1, but use 0 internally
 
@@ -48,38 +49,42 @@ func sortTable(c cfg.Config, data *Tabdata) {
 
 	// actual sorting
 	sort.SliceStable(data.entries, func(i, j int) bool {
-		return compare(&c, data.entries[i][col], data.entries[j][col])
+		return compare(&conf, data.entries[i][col], data.entries[j][col])
 	})
 }
 
 // config is not modified here, but it would be inefficient to copy it every loop
-func compare(c *cfg.Config, a string, b string) bool {
+func compare(conf *cfg.Config, left string, right string) bool {
 	var comp bool
 
-	switch c.SortMode {
+	switch conf.SortMode {
 	case "numeric":
-		left, err := strconv.Atoi(a)
+		left, err := strconv.Atoi(left)
 		if err != nil {
 			left = 0
 		}
-		right, err := strconv.Atoi(b)
+
+		right, err := strconv.Atoi(right)
 		if err != nil {
 			right = 0
 		}
+
 		comp = left < right
 	case "duration":
-		left := duration2int(a)
+		left := duration2int(left)
-		right := duration2int(b)
+		right := duration2int(right)
+
 		comp = left < right
 	case "time":
-		left, _ := dateparse.ParseAny(a)
+		left, _ := dateparse.ParseAny(left)
-		right, _ := dateparse.ParseAny(b)
+		right, _ := dateparse.ParseAny(right)
+
 		comp = left.Unix() < right.Unix()
 	default:
-		comp = a < b
+		comp = left < right
 	}
 
-	if c.SortDescending {
+	if conf.SortDescending {
 		comp = !comp
 	}
 
@@ -87,15 +92,15 @@ func compare(c *cfg.Config, a string, b string) bool {
 }
 
 /*
-   We could use time.ParseDuration(), but this doesn't support days.
+We could use time.ParseDuration(), but this doesn't support days.
 
-   We  could also  use github.com/xhit/go-str2duration/v2,  which does
+We  could also  use github.com/xhit/go-str2duration/v2,  which does
-   the job,  but it's  just another dependency,  just for  this little
+the job,  but it's  just another dependency,  just for  this little
-   gem. And  we don't need a  time.Time value. And int  is good enough
+gem. And  we don't need a  time.Time value. And int  is good enough
-   for duration comparision.
+for duration comparison.
 
-   Convert a  durartion into  an integer.  Valid  time units  are "s",
+Convert a  duration into  an integer.  Valid  time units  are "s",
-   "m", "h" and "d".
+"m", "h" and "d".
 */
 func duration2int(duration string) int {
 	re := regexp.MustCompile(`(\d+)([dhms])`)
@@ -103,16 +108,17 @@ func duration2int(duration string) int {
 
 	for _, match := range re.FindAllStringSubmatch(duration, -1) {
 		if len(match) == 3 {
-			v, _ := strconv.Atoi(match[1])
+			durationvalue, _ := strconv.Atoi(match[1])
+
 			switch match[2][0] {
 			case 'd':
-				seconds += v * 86400
+				seconds += durationvalue * 86400
 			case 'h':
-				seconds += v * 3600
+				seconds += durationvalue * 3600
 			case 'm':
-				seconds += v * 60
+				seconds += durationvalue * 60
 			case 's':
-				seconds += v
+				seconds += durationvalue
 			}
 		}
 	}

lib/sort_test.go

@@ -19,8 +19,9 @@ package lib
 
 import (
 	"fmt"
-	"github.com/tlinden/tablizer/cfg"
 	"testing"
+
+	"github.com/tlinden/tablizer/cfg"
 )
 
 func TestDuration2Seconds(t *testing.T) {
@@ -36,12 +37,12 @@ func TestDuration2Seconds(t *testing.T) {
 		{"19t77X what?4s", 4},
 	}
 
-	for _, tt := range tests {
+	for _, testdata := range tests {
-		testname := fmt.Sprintf("duration-%s", tt.dur)
+		testname := fmt.Sprintf("duration-%s", testdata.dur)
 		t.Run(testname, func(t *testing.T) {
-			seconds := duration2int(tt.dur)
+			seconds := duration2int(testdata.dur)
-			if seconds != tt.expect {
+			if seconds != testdata.expect {
-				t.Errorf("got %d, want %d", seconds, tt.expect)
+				t.Errorf("got %d, want %d", seconds, testdata.expect)
 			}
 		})
 	}
@@ -66,13 +67,15 @@ func TestCompare(t *testing.T) {
 		{"time", "12/24/2022", "1/1/1970", true, true},
 	}
 
-	for _, tt := range tests {
+	for _, testdata := range tests {
-		testname := fmt.Sprintf("compare-mode-%s-a-%s-b-%s-desc-%t", tt.mode, tt.a, tt.b, tt.desc)
+		testname := fmt.Sprintf("compare-mode-%s-a-%s-b-%s-desc-%t",
+			testdata.mode, testdata.a, testdata.b, testdata.desc)
+
 		t.Run(testname, func(t *testing.T) {
-			c := cfg.Config{SortMode: tt.mode, SortDescending: tt.desc}
+			c := cfg.Config{SortMode: testdata.mode, SortDescending: testdata.desc}
-			got := compare(&c, tt.a, tt.b)
+			got := compare(&c, testdata.a, testdata.b)
-			if got != tt.want {
+			if got != testdata.want {
-				t.Errorf("got %t, want %t", got, tt.want)
+				t.Errorf("got %t, want %t", got, testdata.want)
 			}
 		})
 	}

t/plugintest.zy

@@ -0,0 +1,10 @@
+/*
+Simple filter hook function. Splits the argument by whitespace,
+fetches the 2nd element, converts it to an int and returns true
+if it s larger than 5, false otherwise.
+*/
+(defn uselarge [line]
+  (cond (> (atoi (second (resplit line `\s+`))) 5) true false))
+
+/* Register the filter hook */
+(addhook %filter %uselarge)

t/test.sh

@@ -30,14 +30,14 @@ cd $(dirname $0)
 echo "Executing commandline tests ..."
 
 # io pattern tests
-ex io-pattern-and-file $t bk7 testtable.kube
+ex io-pattern-and-file $t bk7 testtable
-cat testtable.kube | ex io-pattern-and-stdin $t bk7
+cat testtable | ex io-pattern-and-stdin $t bk7
-cat testtable.kube | ex io-pattern-and-stdin-dash $t bk7 -
+cat testtable | ex io-pattern-and-stdin-dash $t bk7 -
 
 # same w/o pattern
-ex io-just-file $t testtable.kube
+ex io-just-file $t testtable
-cat testtable.kube | ex io-just-stdin $t
+cat testtable | ex io-just-stdin $t
-cat testtable.kube | ex io-just-stdin-dash $t -
+cat testtable | ex io-just-stdin-dash $t -
 
 if test $fail -ne 0; then
     echo "!!! Some tests failed !!!"

t/testtable2

@@ -0,0 +1,6 @@
+NAME  DURATION
+x     10
+a     100
+z     0
+u     4
+k     6

tablizer.1

@@ -133,7 +133,7 @@
 .\" ========================================================================
 .\"
 .IX Title "TABLIZER 1"
-.TH TABLIZER 1 "2023-04-21" "1" "User Commands"
+.TH TABLIZER 1 "2024-05-07" "1" "User Commands"
 .\" For nroff, turn off justification.  Always turn off hyphenation; it makes
 .\" way too many mistakes in technical documents.
 .if n .ad l
@@ -151,18 +151,21 @@ tablizer \- Manipulate tabular output of other programs
 \&      \-v, \-\-invert\-match       select non\-matching rows
 \&      \-n, \-\-no\-numbering       Disable header numbering
 \&      \-N, \-\-no\-color           Disable pattern highlighting
-\&          \-\-no\-headers         Disable headers display
+\&      \-H, \-\-no\-headers         Disable headers display
 \&      \-s, \-\-separator string   Custom field separator
 \&      \-k, \-\-sort\-by int        Sort by column (default: 1)
+\&      \-z, \-\-fuzzy              Use fuzzy search [experimental]
+\&      \-F, \-\-filter field=reg   Filter given field with regex, can be used multiple times
 \&
 \&    Output Flags (mutually exclusive):
 \&      \-X, \-\-extended           Enable extended output
 \&      \-M, \-\-markdown           Enable markdown table output
 \&      \-O, \-\-orgtbl             Enable org\-mode table output
-\&      \-S, \-\-shell              Enable shell evaluable ouput
+\&      \-S, \-\-shell              Enable shell evaluable output
 \&      \-Y, \-\-yaml               Enable yaml output
 \&      \-C, \-\-csv                Enable CSV output
 \&      \-A, \-\-ascii              Default output mode, ascii tabular
+\&      \-L, \-\-hightlight\-lines   Use alternating background colors for tables
 \&
 \&    Sort Mode Flags (mutually exclusive):
 \&      \-a, \-\-sort\-age           sort according to age (duration) string
@@ -172,10 +175,11 @@ tablizer \- Manipulate tabular output of other programs
 \&
 \&    Other Flags:
 \&          \-\-completion <shell> Generate the autocompletion script for <shell>
+\&      \-f, \-\-config <file>      Configuration file (default: ~/.config/tablizer/config)
 \&      \-d, \-\-debug              Enable debugging
 \&      \-h, \-\-help               help for tablizer
 \&      \-m, \-\-man                Display manual page
-\&      \-v, \-\-version            Print program version
+\&      \-V, \-\-version            Print program version
 .Ve
 .SH "DESCRIPTION"
 .IX Header "DESCRIPTION"
@@ -236,9 +240,9 @@ the original order.
 The numbering can be suppressed by using the \fB\-n\fR option.
 .PP
 By  default tablizer  shows  a  header containing  the  names of  each
-column.  This  can be disabled  using the \fB\-\-no\-headers\fR  option.  Be
+column.  This can  be disabled using the \fB\-H\fR option.   Be aware that
-aware that  this only affects  tabular output modes.  Shell, Extended,
+this only affects tabular output modes.  Shell, Extended, Yaml and \s-1CSV\s0
-Yaml and \s-1CSV\s0 output modes always use the column names.
+output modes always use the column names.
 .PP
 By  default, if  a  \fBpattern\fR  has been  speficied,  matches will  be
 highlighted. You can disable this behavior with the \fB\-N\fR option.
@@ -261,8 +265,8 @@ Sorts timestamps.
 .PP
 Finally the  \fB\-d\fR option  enables debugging  output which  is mostly
 useful for the developer.
-.SS "\s-1PATTERNS\s0"
+.SS "\s-1PATTERNS AND FILTERING\s0"
-.IX Subsection "PATTERNS"
+.IX Subsection "PATTERNS AND FILTERING"
 You can reduce the rows being  displayed by using a regular expression
 pattern.  The  regexp is  \s-1PCRE\s0 compatible, refer  to the  syntax cheat
 sheet here: <https://github.com/google/re2/wiki/Syntax>.  If you want
@@ -293,6 +297,25 @@ Example for a case insensitive search:
 .Vb 1
 \&    kubectl get pods \-A | tablizer "(?i)account"
 .Ve
+.PP
+You  can use  the experimental  fuzzy search  feature by  providing the
+option \fB\-z\fR, in which case the  pattern is regarded as a fuzzy search
+term, not a regexp.
+.PP
+Sometimes you want to  filter by one or more columns.  You can do that
+using the \fB\-F\fR option. The option can be specified multiple times and
+has the following format:
+.PP
+.Vb 1
+\&    fieldname=regexp
+.Ve
+.PP
+Fieldnames (== columns headers) are case insensitive.
+.PP
+If you specify more than one filter, both filters have to match (\s-1AND\s0
+operation).
+.PP
+If the option \fB\-v\fR is specified, the filtering is inverted.
 .SS "\s-1COLUMNS\s0"
 .IX Subsection "COLUMNS"
 The  parameter  \fB\-c\fR  can  be  used  to  specify,  which  columns  to
@@ -439,6 +462,38 @@ To load completions for every new session, run:
 .Ve
 .Sp
 and source this file from your PowerShell profile.
+.SH "CONFIGURATION AND COLORS"
+.IX Header "CONFIGURATION AND COLORS"
+YOu can put certain configuration values into a configuration file in
+\&\s-1HCL\s0 format. By default tablizer looks for
+\&\f(CW\*(C`$HOME/.config/tablizer/config\*(C'\fR, but you can provide one using the
+parameter \f(CW\*(C`\-f\*(C'\fR.
+.PP
+In the configuration the following variables can be defined:
+.PP
+.Vb 8
+\&    BG             = "lightGreen"
+\&    FG             = "white"
+\&    HighlightBG    = "lightGreen"
+\&    HighlightFG    = "white"
+\&    NoHighlightBG  = "white"
+\&    NoHighlightFG  = "lightGreen"
+\&    HighlightHdrBG = "red"
+\&    HighlightHdrFG = "white"
+.Ve
+.PP
+The following color definitions are available:
+.PP
+black, blue,  cyan, darkGray, default, green,  lightBlue, lightCyan,
+lightGreen,   lightMagenta,   lightRed,   lightWhite,   lightYellow,
+magenta, red, white, yellow
+.PP
+The Variables \fB\s-1FG\s0\fR and \fB\s-1BG\s0\fR are being used to highlight matches. The
+other *FG and *BG variables are for colored table output (enabled with
+the \f(CW\*(C`\-L\*(C'\fR parameter).
+.PP
+Colorization can be turned off completely either by setting the
+parameter \f(CW\*(C`\-N\*(C'\fR or the environment variable \fB\s-1NO_COLOR\s0\fR to a true value.
 .SH "BUGS"
 .IX Header "BUGS"
 In order to report a bug, unexpected behavior, feature requests
@@ -448,7 +503,7 @@ or to submit a patch, please open an issue on github:
 .IX Header "LICENSE"
 This software is licensed under the \s-1GNU GENERAL PUBLIC LICENSE\s0 version 3.
 .PP
-Copyright (c) 2023 by Thomas von Dein
+Copyright (c) 2022\-2024 by Thomas von Dein
 .PP
 This software uses the following \s-1GO\s0 modules:
 .IP "repr (https://github.com/alecthomas/repr)" 4

tablizer.pod

@@ -12,18 +12,21 @@ tablizer - Manipulate tabular output of other programs
       -v, --invert-match       select non-matching rows
       -n, --no-numbering       Disable header numbering
       -N, --no-color           Disable pattern highlighting
-          --no-headers         Disable headers display
+      -H, --no-headers         Disable headers display
       -s, --separator string   Custom field separator
       -k, --sort-by int        Sort by column (default: 1)
+      -z, --fuzzy              Use fuzzy search [experimental]
+      -F, --filter field=reg   Filter given field with regex, can be used multiple times
 
     Output Flags (mutually exclusive):
       -X, --extended           Enable extended output
       -M, --markdown           Enable markdown table output
       -O, --orgtbl             Enable org-mode table output
-      -S, --shell              Enable shell evaluable ouput
+      -S, --shell              Enable shell evaluable output
       -Y, --yaml               Enable yaml output
       -C, --csv                Enable CSV output
       -A, --ascii              Default output mode, ascii tabular
+      -L, --hightlight-lines   Use alternating background colors for tables
 
     Sort Mode Flags (mutually exclusive):
       -a, --sort-age           sort according to age (duration) string
@@ -33,10 +36,11 @@ tablizer - Manipulate tabular output of other programs
 
     Other Flags:
           --completion <shell> Generate the autocompletion script for <shell>
+      -f, --config <file>      Configuration file (default: ~/.config/tablizer/config)
       -d, --debug              Enable debugging
       -h, --help               help for tablizer
       -m, --man                Display manual page
-      -v, --version            Print program version
+      -V, --version            Print program version
 
 
 =head1 DESCRIPTION
@@ -92,9 +96,9 @@ the original order.
 The numbering can be suppressed by using the B<-n> option.
 
 By  default tablizer  shows  a  header containing  the  names of  each
-column.  This  can be disabled  using the B<--no-headers>  option.  Be
+column.  This can  be disabled using the B<-H> option.   Be aware that
-aware that  this only affects  tabular output modes.  Shell, Extended,
+this only affects tabular output modes.  Shell, Extended, Yaml and CSV
-Yaml and CSV output modes always use the column names.
+output modes always use the column names.
 
 By  default, if  a  B<pattern>  has been  speficied,  matches will  be
 highlighted. You can disable this behavior with the B<-N> option.
@@ -125,7 +129,7 @@ Sorts timestamps.
 Finally the  B<-d> option  enables debugging  output which  is mostly
 useful for the developer.
 
-=head2 PATTERNS
+=head2 PATTERNS AND FILTERING
 
 You can reduce the rows being  displayed by using a regular expression
 pattern.  The  regexp is  PCRE compatible, refer  to the  syntax cheat
@@ -152,6 +156,23 @@ Example for a case insensitive search:
 
     kubectl get pods -A | tablizer "(?i)account"
 
+You  can use  the experimental  fuzzy search  feature by  providing the
+option B<-z>, in which case the  pattern is regarded as a fuzzy search
+term, not a regexp.
+
+Sometimes you want to  filter by one or more columns.  You can do that
+using the B<-F> option. The option can be specified multiple times and
+has the following format:
+
+    fieldname=regexp
+
+Fieldnames (== columns headers) are case insensitive.
+
+If you specify more than one filter, both filters have to match (AND
+operation).
+
+If the option B<-v> is specified, the filtering is inverted.
+
 
 =head2 COLUMNS
 
@@ -289,6 +310,37 @@ and source this file from your PowerShell profile.
 
 =back
 
+=head1 CONFIGURATION AND COLORS
+
+YOu can put certain configuration values into a configuration file in
+HCL format. By default tablizer looks for
+C<$HOME/.config/tablizer/config>, but you can provide one using the
+parameter C<-f>.
+
+In the configuration the following variables can be defined:
+
+    BG             = "lightGreen"
+    FG             = "white"
+    HighlightBG    = "lightGreen"
+    HighlightFG    = "white"
+    NoHighlightBG  = "white"
+    NoHighlightFG  = "lightGreen"
+    HighlightHdrBG = "red"
+    HighlightHdrFG = "white"
+
+The following color definitions are available:
+
+black, blue,  cyan, darkGray, default, green,  lightBlue, lightCyan,
+lightGreen,   lightMagenta,   lightRed,   lightWhite,   lightYellow,
+magenta, red, white, yellow
+
+The Variables B<FG> and B<BG> are being used to highlight matches. The
+other *FG and *BG variables are for colored table output (enabled with
+the C<-L> parameter).
+
+Colorization can be turned off completely either by setting the
+parameter C<-N> or the environment variable B<NO_COLOR> to a true value.
+
 =head1 BUGS
 
 In order to report a bug, unexpected behavior, feature requests
@@ -299,7 +351,7 @@ L<https://github.com/TLINDEN/tablizer/issues>.
 
 This software is licensed under the GNU GENERAL PUBLIC LICENSE version 3.
 
-Copyright (c) 2023 by Thomas von Dein
+Copyright (c) 2022-2024 by Thomas von Dein
 
 This software uses the following GO modules: