mirror of
https://codeberg.org/scip/tablizer.git
synced 2025-12-17 04:30:56 +01:00
Add JSON input support (#74)
* added basic json input support * add coverage to make test * enhanced unit tests, switch to testify/assert * reduce ci runs
This commit is contained in:
2
.github/workflows/ci.yaml
vendored
2
.github/workflows/ci.yaml
vendored
@@ -1,5 +1,5 @@
|
|||||||
name: build-and-test-tablizer
|
name: build-and-test-tablizer
|
||||||
on: [push, pull_request]
|
on: [push]
|
||||||
jobs:
|
jobs:
|
||||||
build:
|
build:
|
||||||
strategy:
|
strategy:
|
||||||
|
|||||||
2
Makefile
2
Makefile
@@ -65,7 +65,7 @@ clean:
|
|||||||
rm -rf $(tool) releases coverage.out
|
rm -rf $(tool) releases coverage.out
|
||||||
|
|
||||||
test: clean
|
test: clean
|
||||||
go test ./... $(OPTS)
|
go test -cover ./... $(OPTS)
|
||||||
|
|
||||||
singletest:
|
singletest:
|
||||||
@echo "Call like this: 'make singletest TEST=TestPrepareColumns MOD=lib'"
|
@echo "Call like this: 'make singletest TEST=TestPrepareColumns MOD=lib'"
|
||||||
|
|||||||
@@ -34,6 +34,7 @@ Operational Flags:
|
|||||||
-F, --filter <field[!]=reg> Filter given field with regex, can be used multiple times
|
-F, --filter <field[!]=reg> Filter given field with regex, can be used multiple times
|
||||||
-T, --transpose-columns string Transpose the speficied columns (separated by ,)
|
-T, --transpose-columns string Transpose the speficied columns (separated by ,)
|
||||||
-R, --regex-transposer </from/to/> Apply /search/replace/ regexp to fields given in -T
|
-R, --regex-transposer </from/to/> Apply /search/replace/ regexp to fields given in -T
|
||||||
|
-j, --json Read JSON input (must be array of hashes)
|
||||||
-I, --interactive Interactively filter and select rows
|
-I, --interactive Interactively filter and select rows
|
||||||
|
|
||||||
Output Flags (mutually exclusive):
|
Output Flags (mutually exclusive):
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const DefaultSeparator string = `(\s\s+|\t)`
|
const DefaultSeparator string = `(\s\s+|\t)`
|
||||||
const Version string = "v1.5.5"
|
const Version string = "v1.5.6"
|
||||||
const MAXPARTS = 2
|
const MAXPARTS = 2
|
||||||
|
|
||||||
var DefaultConfigfile = os.Getenv("HOME") + "/.config/tablizer/config"
|
var DefaultConfigfile = os.Getenv("HOME") + "/.config/tablizer/config"
|
||||||
@@ -79,6 +79,7 @@ type Config struct {
|
|||||||
UseFuzzySearch bool
|
UseFuzzySearch bool
|
||||||
UseHighlight bool
|
UseHighlight bool
|
||||||
Interactive bool
|
Interactive bool
|
||||||
|
InputJSON bool
|
||||||
|
|
||||||
SortMode string
|
SortMode string
|
||||||
SortDescending bool
|
SortDescending bool
|
||||||
|
|||||||
@@ -21,6 +21,8 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
// "reflect"
|
// "reflect"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestPrepareModeFlags(t *testing.T) {
|
func TestPrepareModeFlags(t *testing.T) {
|
||||||
@@ -44,9 +46,8 @@ func TestPrepareModeFlags(t *testing.T) {
|
|||||||
conf := Config{}
|
conf := Config{}
|
||||||
|
|
||||||
conf.PrepareModeFlags(testdata.flag)
|
conf.PrepareModeFlags(testdata.flag)
|
||||||
if conf.OutputMode != testdata.expect {
|
|
||||||
t.Errorf("got: %d, expect: %d", conf.OutputMode, testdata.expect)
|
assert.EqualValues(t, testdata.expect, conf.OutputMode)
|
||||||
}
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -70,9 +71,7 @@ func TestPrepareSortFlags(t *testing.T) {
|
|||||||
|
|
||||||
conf.PrepareSortFlags(testdata.flag)
|
conf.PrepareSortFlags(testdata.flag)
|
||||||
|
|
||||||
if conf.SortMode != testdata.expect {
|
assert.EqualValues(t, testdata.expect, conf.SortMode)
|
||||||
t.Errorf("got: %s, expect: %s", conf.SortMode, testdata.expect)
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -81,7 +80,7 @@ func TestPreparePattern(t *testing.T) {
|
|||||||
var tests = []struct {
|
var tests = []struct {
|
||||||
patterns []*Pattern
|
patterns []*Pattern
|
||||||
name string
|
name string
|
||||||
wanterr bool
|
wanterror bool
|
||||||
wanticase bool
|
wanticase bool
|
||||||
wantneg bool
|
wantneg bool
|
||||||
}{
|
}{
|
||||||
@@ -123,16 +122,16 @@ func TestPreparePattern(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, testdata := range tests {
|
for _, testdata := range tests {
|
||||||
testname := fmt.Sprintf("PreparePattern-pattern-%s-wanterr-%t", testdata.name, testdata.wanterr)
|
testname := fmt.Sprintf("PreparePattern-pattern-%s-wanterr-%t", testdata.name, testdata.wanterror)
|
||||||
t.Run(testname, func(t *testing.T) {
|
t.Run(testname, func(t *testing.T) {
|
||||||
conf := Config{}
|
conf := Config{}
|
||||||
|
|
||||||
err := conf.PreparePattern(testdata.patterns)
|
err := conf.PreparePattern(testdata.patterns)
|
||||||
|
|
||||||
if err != nil {
|
if testdata.wanterror {
|
||||||
if !testdata.wanterr {
|
assert.Error(t, err)
|
||||||
t.Errorf("PreparePattern returned error: %s", err)
|
} else {
|
||||||
}
|
assert.NoError(t, err)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -131,9 +131,11 @@ func Execute() {
|
|||||||
rootCmd.PersistentFlags().StringVarP(&conf.TransposeColumns, "transpose-columns", "T", "",
|
rootCmd.PersistentFlags().StringVarP(&conf.TransposeColumns, "transpose-columns", "T", "",
|
||||||
"Transpose the speficied columns (separated by ,)")
|
"Transpose the speficied columns (separated by ,)")
|
||||||
rootCmd.PersistentFlags().BoolVarP(&conf.Interactive, "interactive", "I", false,
|
rootCmd.PersistentFlags().BoolVarP(&conf.Interactive, "interactive", "I", false,
|
||||||
"interactive mode (experimental)")
|
"interactive mode")
|
||||||
rootCmd.PersistentFlags().StringVarP(&conf.OFS, "ofs", "", "",
|
rootCmd.PersistentFlags().StringVarP(&conf.OFS, "ofs", "", "",
|
||||||
"Output field separator (' ' for ascii table, ',' for CSV)")
|
"Output field separator (' ' for ascii table, ',' for CSV)")
|
||||||
|
rootCmd.PersistentFlags().BoolVarP(&conf.InputJSON, "json", "j", false,
|
||||||
|
"JSON input mode")
|
||||||
|
|
||||||
// sort options
|
// sort options
|
||||||
rootCmd.PersistentFlags().StringVarP(&conf.SortByColumn, "sort-by", "k", "",
|
rootCmd.PersistentFlags().StringVarP(&conf.SortByColumn, "sort-by", "k", "",
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ SYNOPSIS
|
|||||||
-F, --filter <field[!]=reg> Filter given field with regex, can be used multiple times
|
-F, --filter <field[!]=reg> Filter given field with regex, can be used multiple times
|
||||||
-T, --transpose-columns string Transpose the speficied columns (separated by ,)
|
-T, --transpose-columns string Transpose the speficied columns (separated by ,)
|
||||||
-R, --regex-transposer </from/to/> Apply /search/replace/ regexp to fields given in -T
|
-R, --regex-transposer </from/to/> Apply /search/replace/ regexp to fields given in -T
|
||||||
|
-j, --json Read JSON input (must be array of hashes)
|
||||||
-I, --interactive Interactively filter and select rows
|
-I, --interactive Interactively filter and select rows
|
||||||
|
|
||||||
Output Flags (mutually exclusive):
|
Output Flags (mutually exclusive):
|
||||||
@@ -463,6 +464,7 @@ Operational Flags:
|
|||||||
-F, --filter <field[!]=reg> Filter given field with regex, can be used multiple times
|
-F, --filter <field[!]=reg> Filter given field with regex, can be used multiple times
|
||||||
-T, --transpose-columns string Transpose the speficied columns (separated by ,)
|
-T, --transpose-columns string Transpose the speficied columns (separated by ,)
|
||||||
-R, --regex-transposer </from/to/> Apply /search/replace/ regexp to fields given in -T
|
-R, --regex-transposer </from/to/> Apply /search/replace/ regexp to fields given in -T
|
||||||
|
-j, --json Read JSON input (must be array of hashes)
|
||||||
-I, --interactive Interactively filter and select rows
|
-I, --interactive Interactively filter and select rows
|
||||||
|
|
||||||
Output Flags (mutually exclusive):
|
Output Flags (mutually exclusive):
|
||||||
|
|||||||
3
go.mod
3
go.mod
@@ -30,6 +30,7 @@ require (
|
|||||||
github.com/charmbracelet/x/ansi v0.9.3 // indirect
|
github.com/charmbracelet/x/ansi v0.9.3 // indirect
|
||||||
github.com/charmbracelet/x/cellbuf v0.0.13 // indirect
|
github.com/charmbracelet/x/cellbuf v0.0.13 // indirect
|
||||||
github.com/charmbracelet/x/term v0.2.1 // indirect
|
github.com/charmbracelet/x/term v0.2.1 // indirect
|
||||||
|
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
|
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
|
||||||
github.com/fatih/color v1.18.0 // indirect
|
github.com/fatih/color v1.18.0 // indirect
|
||||||
github.com/google/go-cmp v0.6.0 // indirect
|
github.com/google/go-cmp v0.6.0 // indirect
|
||||||
@@ -47,8 +48,10 @@ require (
|
|||||||
github.com/olekukonko/errors v1.1.0 // indirect
|
github.com/olekukonko/errors v1.1.0 // indirect
|
||||||
github.com/olekukonko/ll v0.0.9 // indirect
|
github.com/olekukonko/ll v0.0.9 // indirect
|
||||||
github.com/pkg/errors v0.9.1 // indirect
|
github.com/pkg/errors v0.9.1 // indirect
|
||||||
|
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||||
github.com/rivo/uniseg v0.4.7 // indirect
|
github.com/rivo/uniseg v0.4.7 // indirect
|
||||||
github.com/spf13/pflag v1.0.6 // indirect
|
github.com/spf13/pflag v1.0.6 // indirect
|
||||||
|
github.com/stretchr/testify v1.11.1 // indirect
|
||||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||||
github.com/zclconf/go-cty v1.16.3 // indirect
|
github.com/zclconf/go-cty v1.16.3 // indirect
|
||||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
|
golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
|
||||||
|
|||||||
2
go.sum
2
go.sum
@@ -96,6 +96,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
|
|||||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||||
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
||||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||||
|
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||||
|
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||||
github.com/tiagomelo/go-clipboard v0.1.2 h1:Ph2icR0vZRIj3v5ExvsGweBwsbbDUTlS6HoF40MkQD8=
|
github.com/tiagomelo/go-clipboard v0.1.2 h1:Ph2icR0vZRIj3v5ExvsGweBwsbbDUTlS6HoF40MkQD8=
|
||||||
github.com/tiagomelo/go-clipboard v0.1.2/go.mod h1:kXtjJBIMimZaGbxmcKZ8+JqK+acSNf5tAJiChlZBOr8=
|
github.com/tiagomelo/go-clipboard v0.1.2/go.mod h1:kXtjJBIMimZaGbxmcKZ8+JqK+acSNf5tAJiChlZBOr8=
|
||||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
|
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
|
||||||
|
|||||||
@@ -19,9 +19,9 @@ package lib
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"reflect"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/tlinden/tablizer/cfg"
|
"github.com/tlinden/tablizer/cfg"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -56,13 +56,11 @@ func TestMatchPattern(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
err := conf.PreparePattern(inputdata.patterns)
|
err := conf.PreparePattern(inputdata.patterns)
|
||||||
if err != nil {
|
|
||||||
t.Errorf("PreparePattern returned error: %s", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !matchPattern(conf, inputdata.line) {
|
assert.NoError(t, err)
|
||||||
t.Errorf("matchPattern() did not match\nExp: true\nGot: false\n")
|
|
||||||
}
|
res := matchPattern(conf, inputdata.line)
|
||||||
|
assert.EqualValues(t, true, res)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -163,14 +161,12 @@ func TestFilterByFields(t *testing.T) {
|
|||||||
conf := cfg.Config{Rawfilters: inputdata.filter, InvertMatch: inputdata.invert}
|
conf := cfg.Config{Rawfilters: inputdata.filter, InvertMatch: inputdata.invert}
|
||||||
|
|
||||||
err := conf.PrepareFilters()
|
err := conf.PrepareFilters()
|
||||||
if err != nil {
|
|
||||||
t.Errorf("PrepareFilters returned error: %s", err)
|
assert.NoError(t, err)
|
||||||
}
|
|
||||||
|
|
||||||
data, _, _ := FilterByFields(conf, &data)
|
data, _, _ := FilterByFields(conf, &data)
|
||||||
if !reflect.DeepEqual(*data, inputdata.expect) {
|
|
||||||
t.Errorf("Filtered data does not match expected data:\ngot: %+v\nexp: %+v", data, inputdata.expect)
|
assert.EqualValues(t, inputdata.expect, *data)
|
||||||
}
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,9 +19,9 @@ package lib
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"reflect"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/tlinden/tablizer/cfg"
|
"github.com/tlinden/tablizer/cfg"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -39,9 +39,8 @@ func TestContains(t *testing.T) {
|
|||||||
testname := fmt.Sprintf("contains-%d,%d,%t", tt.list, tt.search, tt.want)
|
testname := fmt.Sprintf("contains-%d,%d,%t", tt.list, tt.search, tt.want)
|
||||||
t.Run(testname, func(t *testing.T) {
|
t.Run(testname, func(t *testing.T) {
|
||||||
answer := contains(tt.list, tt.search)
|
answer := contains(tt.list, tt.search)
|
||||||
if answer != tt.want {
|
|
||||||
t.Errorf("got %t, want %t", answer, tt.want)
|
assert.EqualValues(t, tt.want, answer)
|
||||||
}
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -77,14 +76,12 @@ func TestPrepareColumns(t *testing.T) {
|
|||||||
t.Run(testname, func(t *testing.T) {
|
t.Run(testname, func(t *testing.T) {
|
||||||
conf := cfg.Config{Columns: testdata.input}
|
conf := cfg.Config{Columns: testdata.input}
|
||||||
err := PrepareColumns(&conf, &data)
|
err := PrepareColumns(&conf, &data)
|
||||||
if err != nil {
|
|
||||||
if !testdata.wanterror {
|
if testdata.wanterror {
|
||||||
t.Errorf("got error: %v", err)
|
assert.Error(t, err)
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
if !reflect.DeepEqual(conf.UseColumns, testdata.exp) {
|
assert.NoError(t, err)
|
||||||
t.Errorf("got: %v, expected: %v", conf.UseColumns, testdata.exp)
|
assert.EqualValues(t, testdata.exp, conf.UseColumns)
|
||||||
}
|
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -153,18 +150,13 @@ func TestPrepareTransposerColumns(t *testing.T) {
|
|||||||
t.Run(testname, func(t *testing.T) {
|
t.Run(testname, func(t *testing.T) {
|
||||||
conf := cfg.Config{TransposeColumns: testdata.input, Transposers: testdata.transp}
|
conf := cfg.Config{TransposeColumns: testdata.input, Transposers: testdata.transp}
|
||||||
err := PrepareTransposerColumns(&conf, &data)
|
err := PrepareTransposerColumns(&conf, &data)
|
||||||
if err != nil {
|
|
||||||
if !testdata.wanterror {
|
|
||||||
t.Errorf("got error: %v", err)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if len(conf.UseTransposeColumns) != testdata.exp {
|
|
||||||
t.Errorf("got %d, want %d", conf.UseTransposeColumns, testdata.exp)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(conf.Transposers) != len(conf.UseTransposeColumns) {
|
if testdata.wanterror {
|
||||||
t.Errorf("got %d, want %d", conf.UseTransposeColumns, testdata.exp)
|
assert.Error(t, err)
|
||||||
}
|
} else {
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.EqualValues(t, testdata.exp, len(conf.UseTransposeColumns))
|
||||||
|
assert.EqualValues(t, len(conf.UseTransposeColumns), len(conf.Transposers))
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -202,10 +194,8 @@ func TestReduceColumns(t *testing.T) {
|
|||||||
c := cfg.Config{Columns: "x", UseColumns: testdata.columns}
|
c := cfg.Config{Columns: "x", UseColumns: testdata.columns}
|
||||||
data := Tabdata{entries: input}
|
data := Tabdata{entries: input}
|
||||||
reduceColumns(c, &data)
|
reduceColumns(c, &data)
|
||||||
if !reflect.DeepEqual(data.entries, testdata.expect) {
|
|
||||||
t.Errorf("reduceColumns returned invalid data:\ngot: %+v\nexp: %+v",
|
assert.EqualValues(t, testdata.expect, data.entries)
|
||||||
data.entries, testdata.expect)
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -233,10 +223,8 @@ func TestNumberizeHeaders(t *testing.T) {
|
|||||||
conf := cfg.Config{Columns: "x", UseColumns: testdata.columns, Numbering: testdata.numberize}
|
conf := cfg.Config{Columns: "x", UseColumns: testdata.columns, Numbering: testdata.numberize}
|
||||||
usedata := data
|
usedata := data
|
||||||
numberizeAndReduceHeaders(conf, &usedata)
|
numberizeAndReduceHeaders(conf, &usedata)
|
||||||
if !reflect.DeepEqual(usedata.headers, testdata.expect) {
|
|
||||||
t.Errorf("numberizeAndReduceHeaders returned invalid data:\ngot: %+v\nexp: %+v",
|
assert.EqualValues(t, testdata.expect, usedata.headers)
|
||||||
usedata.headers, testdata.expect)
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
110
lib/parser.go
110
lib/parser.go
@@ -1,5 +1,5 @@
|
|||||||
/*
|
/*
|
||||||
Copyright © 2022-2024 Thomas von Dein
|
Copyright © 2022-2025 Thomas von Dein
|
||||||
|
|
||||||
This program is free software: you can redistribute it and/or modify
|
This program is free software: you can redistribute it and/or modify
|
||||||
it under the terms of the GNU General Public License as published by
|
it under the terms of the GNU General Public License as published by
|
||||||
@@ -20,8 +20,11 @@ package lib
|
|||||||
import (
|
import (
|
||||||
"bufio"
|
"bufio"
|
||||||
"encoding/csv"
|
"encoding/csv"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"log"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
@@ -39,6 +42,8 @@ func Parse(conf cfg.Config, input io.Reader) (Tabdata, error) {
|
|||||||
// first step, parse the data
|
// first step, parse the data
|
||||||
if len(conf.Separator) == 1 {
|
if len(conf.Separator) == 1 {
|
||||||
data, err = parseCSV(conf, input)
|
data, err = parseCSV(conf, input)
|
||||||
|
} else if conf.InputJSON {
|
||||||
|
data, err = parseJSON(conf, input)
|
||||||
} else {
|
} else {
|
||||||
data, err = parseTabular(conf, input)
|
data, err = parseTabular(conf, input)
|
||||||
}
|
}
|
||||||
@@ -172,6 +177,109 @@ func parseTabular(conf cfg.Config, input io.Reader) (Tabdata, error) {
|
|||||||
return data, nil
|
return data, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
Parse JSON input. We only support an array of maps.
|
||||||
|
*/
|
||||||
|
func parseRawJSON(conf cfg.Config, input io.Reader) (Tabdata, error) {
|
||||||
|
dec := json.NewDecoder(input)
|
||||||
|
headers := []string{}
|
||||||
|
idxmap := map[string]int{}
|
||||||
|
data := [][]string{}
|
||||||
|
row := []string{}
|
||||||
|
iskey := true
|
||||||
|
haveheaders := false
|
||||||
|
var currentfield string
|
||||||
|
var idx int
|
||||||
|
var isjson bool
|
||||||
|
|
||||||
|
for {
|
||||||
|
t, err := dec.Token()
|
||||||
|
if err == io.EOF {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
switch val := t.(type) {
|
||||||
|
case string:
|
||||||
|
if iskey {
|
||||||
|
if !haveheaders {
|
||||||
|
// consider only the keys of the first item as headers
|
||||||
|
headers = append(headers, val)
|
||||||
|
}
|
||||||
|
currentfield = val
|
||||||
|
} else {
|
||||||
|
if !haveheaders {
|
||||||
|
// the first row uses the order as it comes in
|
||||||
|
row = append(row, val)
|
||||||
|
} else {
|
||||||
|
// use the pre-determined order, that way items
|
||||||
|
// can be in any order as long as they contain all
|
||||||
|
// neccessary fields. They may also contain less
|
||||||
|
// fields than the first item, these will contain
|
||||||
|
// the empty string
|
||||||
|
row[idxmap[currentfield]] = val
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case json.Delim:
|
||||||
|
if val.String() == "}" {
|
||||||
|
data = append(data, row)
|
||||||
|
row = make([]string, len(headers))
|
||||||
|
idx++
|
||||||
|
|
||||||
|
if !haveheaders {
|
||||||
|
// remember the array position of header fields,
|
||||||
|
// which we use to assign elements to the correct
|
||||||
|
// row index
|
||||||
|
for i, header := range headers {
|
||||||
|
idxmap[header] = i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
haveheaders = true
|
||||||
|
}
|
||||||
|
isjson = true
|
||||||
|
}
|
||||||
|
|
||||||
|
iskey = !iskey
|
||||||
|
}
|
||||||
|
|
||||||
|
if isjson && (len(headers) == 0 || len(data) == 0) {
|
||||||
|
return Tabdata{}, errors.New("failed to parse JSON, input did not contain array of hashes")
|
||||||
|
}
|
||||||
|
|
||||||
|
return Tabdata{headers: headers, entries: data, columns: len(headers)}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseJSON(conf cfg.Config, input io.Reader) (Tabdata, error) {
|
||||||
|
// parse raw json
|
||||||
|
data, err := parseRawJSON(conf, input)
|
||||||
|
if err != nil {
|
||||||
|
return data, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// apply filter, if any
|
||||||
|
filtered := [][]string{}
|
||||||
|
var line string
|
||||||
|
|
||||||
|
for _, row := range data.entries {
|
||||||
|
line = strings.Join(row, " ")
|
||||||
|
|
||||||
|
if matchPattern(conf, line) == conf.InvertMatch {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
filtered = append(filtered, row)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(filtered) != len(data.entries) {
|
||||||
|
data.entries = filtered
|
||||||
|
}
|
||||||
|
|
||||||
|
return data, nil
|
||||||
|
}
|
||||||
|
|
||||||
func PostProcess(conf cfg.Config, data *Tabdata) (*Tabdata, bool, error) {
|
func PostProcess(conf cfg.Config, data *Tabdata) (*Tabdata, bool, error) {
|
||||||
var modified bool
|
var modified bool
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
/*
|
/*
|
||||||
Copyright © 2022 Thomas von Dein
|
Copyright © 2022-2025 Thomas von Dein
|
||||||
|
|
||||||
This program is free software: you can redistribute it and/or modify
|
This program is free software: you can redistribute it and/or modify
|
||||||
it under the terms of the GNU General Public License as published by
|
it under the terms of the GNU General Public License as published by
|
||||||
@@ -19,10 +19,11 @@ package lib
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"reflect"
|
"io"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/tlinden/tablizer/cfg"
|
"github.com/tlinden/tablizer/cfg"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -67,16 +68,10 @@ func TestParser(t *testing.T) {
|
|||||||
t.Run(testname, func(t *testing.T) {
|
t.Run(testname, func(t *testing.T) {
|
||||||
readFd := strings.NewReader(strings.TrimSpace(testdata.text))
|
readFd := strings.NewReader(strings.TrimSpace(testdata.text))
|
||||||
conf := cfg.Config{Separator: testdata.separator}
|
conf := cfg.Config{Separator: testdata.separator}
|
||||||
gotdata, err := Parse(conf, readFd)
|
gotdata, err := wrapValidateParser(conf, readFd)
|
||||||
|
|
||||||
if err != nil {
|
assert.NoError(t, err)
|
||||||
t.Errorf("Parser returned error: %s\nData processed so far: %+v", err, gotdata)
|
assert.EqualValues(t, data, gotdata)
|
||||||
}
|
|
||||||
|
|
||||||
if !reflect.DeepEqual(data, gotdata) {
|
|
||||||
t.Errorf("Parser returned invalid data\nExp: %+v\nGot: %+v\n",
|
|
||||||
data, gotdata)
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -87,7 +82,7 @@ func TestParserPatternmatching(t *testing.T) {
|
|||||||
entries [][]string
|
entries [][]string
|
||||||
patterns []*cfg.Pattern
|
patterns []*cfg.Pattern
|
||||||
invert bool
|
invert bool
|
||||||
want bool
|
wanterror bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "match",
|
name: "match",
|
||||||
@@ -121,18 +116,13 @@ func TestParserPatternmatching(t *testing.T) {
|
|||||||
_ = conf.PreparePattern(testdata.patterns)
|
_ = conf.PreparePattern(testdata.patterns)
|
||||||
|
|
||||||
readFd := strings.NewReader(strings.TrimSpace(inputdata.text))
|
readFd := strings.NewReader(strings.TrimSpace(inputdata.text))
|
||||||
gotdata, err := Parse(conf, readFd)
|
data, err := wrapValidateParser(conf, readFd)
|
||||||
|
|
||||||
if err != nil {
|
if testdata.wanterror {
|
||||||
if !testdata.want {
|
assert.Error(t, err)
|
||||||
t.Errorf("Parser returned error: %s\nData processed so far: %+v",
|
|
||||||
err, gotdata)
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
if !reflect.DeepEqual(testdata.entries, gotdata.entries) {
|
assert.NoError(t, err)
|
||||||
t.Errorf("Parser returned invalid data (pattern: %s, invert: %t)\nExp: %+v\nGot: %+v\n",
|
assert.EqualValues(t, testdata.entries, data.entries)
|
||||||
testdata.name, testdata.invert, testdata.entries, gotdata.entries)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -159,14 +149,147 @@ asd igig
|
|||||||
|
|
||||||
readFd := strings.NewReader(strings.TrimSpace(table))
|
readFd := strings.NewReader(strings.TrimSpace(table))
|
||||||
conf := cfg.Config{Separator: cfg.DefaultSeparator}
|
conf := cfg.Config{Separator: cfg.DefaultSeparator}
|
||||||
gotdata, err := Parse(conf, readFd)
|
gotdata, err := wrapValidateParser(conf, readFd)
|
||||||
|
|
||||||
if err != nil {
|
assert.NoError(t, err)
|
||||||
t.Errorf("Parser returned error: %s\nData processed so far: %+v", err, gotdata)
|
assert.EqualValues(t, data, gotdata)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParserJSONInput(t *testing.T) {
|
||||||
|
var tests = []struct {
|
||||||
|
name string
|
||||||
|
input string
|
||||||
|
expect Tabdata
|
||||||
|
wanterror bool // true: expect fail, false: expect success
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
// too deep nesting
|
||||||
|
name: "invalidjson",
|
||||||
|
wanterror: true,
|
||||||
|
input: `[
|
||||||
|
{
|
||||||
|
"item": {
|
||||||
|
"NAME": "postgres-operator-7f4c7c8485-ntlns",
|
||||||
|
"READY": "1/1",
|
||||||
|
"STATUS": "Running",
|
||||||
|
"RESTARTS": "0",
|
||||||
|
"AGE": "24h"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
`,
|
||||||
|
expect: Tabdata{},
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
// one field missing + different order
|
||||||
|
// but shall not fail
|
||||||
|
name: "kgpfail",
|
||||||
|
wanterror: false,
|
||||||
|
input: `[
|
||||||
|
{
|
||||||
|
"NAME": "postgres-operator-7f4c7c8485-ntlns",
|
||||||
|
"READY": "1/1",
|
||||||
|
"STATUS": "Running",
|
||||||
|
"RESTARTS": "0",
|
||||||
|
"AGE": "24h"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"NAME": "wal-g-exporter-778dcd95f5-wcjzn",
|
||||||
|
"RESTARTS": "0",
|
||||||
|
"READY": "1/1",
|
||||||
|
"AGE": "24h"
|
||||||
|
}
|
||||||
|
]`,
|
||||||
|
expect: Tabdata{
|
||||||
|
columns: 5,
|
||||||
|
headers: []string{"NAME", "READY", "STATUS", "RESTARTS", "AGE"},
|
||||||
|
entries: [][]string{
|
||||||
|
[]string{
|
||||||
|
"postgres-operator-7f4c7c8485-ntlns",
|
||||||
|
"1/1",
|
||||||
|
"Running",
|
||||||
|
"0",
|
||||||
|
"24h",
|
||||||
|
},
|
||||||
|
[]string{
|
||||||
|
"wal-g-exporter-778dcd95f5-wcjzn",
|
||||||
|
"1/1",
|
||||||
|
"",
|
||||||
|
"0",
|
||||||
|
"24h",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
name: "kgp",
|
||||||
|
wanterror: false,
|
||||||
|
input: `[
|
||||||
|
{
|
||||||
|
"NAME": "postgres-operator-7f4c7c8485-ntlns",
|
||||||
|
"READY": "1/1",
|
||||||
|
"STATUS": "Running",
|
||||||
|
"RESTARTS": "0",
|
||||||
|
"AGE": "24h"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"NAME": "wal-g-exporter-778dcd95f5-wcjzn",
|
||||||
|
"STATUS": "Running",
|
||||||
|
"READY": "1/1",
|
||||||
|
"RESTARTS": "0",
|
||||||
|
"AGE": "24h"
|
||||||
|
}
|
||||||
|
]`,
|
||||||
|
expect: Tabdata{
|
||||||
|
columns: 5,
|
||||||
|
headers: []string{"NAME", "READY", "STATUS", "RESTARTS", "AGE"},
|
||||||
|
entries: [][]string{
|
||||||
|
[]string{
|
||||||
|
"postgres-operator-7f4c7c8485-ntlns",
|
||||||
|
"1/1",
|
||||||
|
"Running",
|
||||||
|
"0",
|
||||||
|
"24h",
|
||||||
|
},
|
||||||
|
[]string{
|
||||||
|
"wal-g-exporter-778dcd95f5-wcjzn",
|
||||||
|
"1/1",
|
||||||
|
"Running",
|
||||||
|
"0",
|
||||||
|
"24h",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
if !reflect.DeepEqual(data, gotdata) {
|
for _, testdata := range tests {
|
||||||
t.Errorf("Parser returned invalid data, Regex: %s\nExp: %+v\nGot: %+v\n",
|
testname := fmt.Sprintf("parse-json-%s", testdata.name)
|
||||||
conf.Separator, data, gotdata)
|
t.Run(testname, func(t *testing.T) {
|
||||||
|
conf := cfg.Config{InputJSON: true}
|
||||||
|
|
||||||
|
readFd := strings.NewReader(strings.TrimSpace(testdata.input))
|
||||||
|
data, err := wrapValidateParser(conf, readFd)
|
||||||
|
|
||||||
|
if testdata.wanterror {
|
||||||
|
assert.Error(t, err)
|
||||||
|
} else {
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.EqualValues(t, testdata.expect, data)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func wrapValidateParser(conf cfg.Config, input io.Reader) (Tabdata, error) {
|
||||||
|
data, err := Parse(conf, input)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return data, err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = ValidateConsistency(&data)
|
||||||
|
|
||||||
|
return data, err
|
||||||
|
}
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/tlinden/tablizer/cfg"
|
"github.com/tlinden/tablizer/cfg"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -307,13 +308,7 @@ func TestPrinter(t *testing.T) {
|
|||||||
|
|
||||||
got := strings.TrimSpace(writer.String())
|
got := strings.TrimSpace(writer.String())
|
||||||
|
|
||||||
if got != exp {
|
assert.EqualValues(t, exp, got)
|
||||||
t.Errorf("not rendered correctly:\n+++ got:\n%s\n+++ want:\n%s",
|
|
||||||
got, exp,
|
|
||||||
// strings.ReplaceAll(got, " ", "_"),
|
|
||||||
// strings.ReplaceAll(exp, " ", "_")
|
|
||||||
)
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/tlinden/tablizer/cfg"
|
"github.com/tlinden/tablizer/cfg"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -41,9 +42,7 @@ func TestDuration2Seconds(t *testing.T) {
|
|||||||
testname := fmt.Sprintf("duration-%s", testdata.dur)
|
testname := fmt.Sprintf("duration-%s", testdata.dur)
|
||||||
t.Run(testname, func(t *testing.T) {
|
t.Run(testname, func(t *testing.T) {
|
||||||
seconds := duration2int(testdata.dur)
|
seconds := duration2int(testdata.dur)
|
||||||
if seconds != testdata.expect {
|
assert.EqualValues(t, testdata.expect, seconds)
|
||||||
t.Errorf("got %d, want %d", seconds, testdata.expect)
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -74,9 +73,7 @@ func TestCompare(t *testing.T) {
|
|||||||
t.Run(testname, func(t *testing.T) {
|
t.Run(testname, func(t *testing.T) {
|
||||||
c := cfg.Config{SortMode: testdata.mode, SortDescending: testdata.desc}
|
c := cfg.Config{SortMode: testdata.mode, SortDescending: testdata.desc}
|
||||||
got := compare(&c, testdata.a, testdata.b)
|
got := compare(&c, testdata.a, testdata.b)
|
||||||
if got != testdata.want {
|
assert.EqualValues(t, testdata.want, got)
|
||||||
t.Errorf("got %d, want %d", got, testdata.want)
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/tiagomelo/go-clipboard/clipboard"
|
"github.com/tiagomelo/go-clipboard/clipboard"
|
||||||
"github.com/tlinden/tablizer/cfg"
|
"github.com/tlinden/tablizer/cfg"
|
||||||
)
|
)
|
||||||
@@ -59,14 +60,9 @@ func DISABLED_TestYankColumns(t *testing.T) {
|
|||||||
printData(&writer, conf, &data)
|
printData(&writer, conf, &data)
|
||||||
|
|
||||||
got, err := cb.PasteText()
|
got, err := cb.PasteText()
|
||||||
if err != nil {
|
|
||||||
t.Errorf("failed to fetch yanked text from clipboard")
|
|
||||||
}
|
|
||||||
|
|
||||||
if got != testdata.expect {
|
assert.NoError(t, err)
|
||||||
t.Errorf("not yanked correctly:\n+++ got:\n%s\n+++ want:\n%s",
|
assert.EqualValues(t, testdata.expect, got)
|
||||||
got, testdata.expect)
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -133,7 +133,7 @@
|
|||||||
.\" ========================================================================
|
.\" ========================================================================
|
||||||
.\"
|
.\"
|
||||||
.IX Title "TABLIZER 1"
|
.IX Title "TABLIZER 1"
|
||||||
.TH TABLIZER 1 "2025-09-30" "1" "User Commands"
|
.TH TABLIZER 1 "2025-10-01" "1" "User Commands"
|
||||||
.\" For nroff, turn off justification. Always turn off hyphenation; it makes
|
.\" For nroff, turn off justification. Always turn off hyphenation; it makes
|
||||||
.\" way too many mistakes in technical documents.
|
.\" way too many mistakes in technical documents.
|
||||||
.if n .ad l
|
.if n .ad l
|
||||||
@@ -158,6 +158,7 @@ tablizer \- Manipulate tabular output of other programs
|
|||||||
\& \-F, \-\-filter <field[!]=reg> Filter given field with regex, can be used multiple times
|
\& \-F, \-\-filter <field[!]=reg> Filter given field with regex, can be used multiple times
|
||||||
\& \-T, \-\-transpose\-columns string Transpose the speficied columns (separated by ,)
|
\& \-T, \-\-transpose\-columns string Transpose the speficied columns (separated by ,)
|
||||||
\& \-R, \-\-regex\-transposer </from/to/> Apply /search/replace/ regexp to fields given in \-T
|
\& \-R, \-\-regex\-transposer </from/to/> Apply /search/replace/ regexp to fields given in \-T
|
||||||
|
\& \-j, \-\-json Read JSON input (must be array of hashes)
|
||||||
\& \-I, \-\-interactive Interactively filter and select rows
|
\& \-I, \-\-interactive Interactively filter and select rows
|
||||||
\&
|
\&
|
||||||
\& Output Flags (mutually exclusive):
|
\& Output Flags (mutually exclusive):
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ tablizer - Manipulate tabular output of other programs
|
|||||||
-F, --filter <field[!]=reg> Filter given field with regex, can be used multiple times
|
-F, --filter <field[!]=reg> Filter given field with regex, can be used multiple times
|
||||||
-T, --transpose-columns string Transpose the speficied columns (separated by ,)
|
-T, --transpose-columns string Transpose the speficied columns (separated by ,)
|
||||||
-R, --regex-transposer </from/to/> Apply /search/replace/ regexp to fields given in -T
|
-R, --regex-transposer </from/to/> Apply /search/replace/ regexp to fields given in -T
|
||||||
|
-j, --json Read JSON input (must be array of hashes)
|
||||||
-I, --interactive Interactively filter and select rows
|
-I, --interactive Interactively filter and select rows
|
||||||
|
|
||||||
Output Flags (mutually exclusive):
|
Output Flags (mutually exclusive):
|
||||||
|
|||||||
Reference in New Issue
Block a user