Add JSON input support (#74)

* added basic json input support
* add coverage to make test
* enhanced unit tests, switch to testify/assert
* reduce ci runs

lib/filter_test.go

@@ -19,9 +19,9 @@ package lib
 
 import (
 	"fmt"
-	"reflect"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/tlinden/tablizer/cfg"
 )
 
@@ -56,13 +56,11 @@ func TestMatchPattern(t *testing.T) {
 			}
 
 			err := conf.PreparePattern(inputdata.patterns)
-			if err != nil {
-				t.Errorf("PreparePattern returned error: %s", err)
-			}
 
-			if !matchPattern(conf, inputdata.line) {
-				t.Errorf("matchPattern() did not match\nExp: true\nGot: false\n")
-			}
+			assert.NoError(t, err)
+
+			res := matchPattern(conf, inputdata.line)
+			assert.EqualValues(t, true, res)
 		})
 	}
 }
@@ -163,14 +161,12 @@ func TestFilterByFields(t *testing.T) {
 			conf := cfg.Config{Rawfilters: inputdata.filter, InvertMatch: inputdata.invert}
 
 			err := conf.PrepareFilters()
-			if err != nil {
-				t.Errorf("PrepareFilters returned error: %s", err)
-			}
+
+			assert.NoError(t, err)
 
 			data, _, _ := FilterByFields(conf, &data)
-			if !reflect.DeepEqual(*data, inputdata.expect) {
-				t.Errorf("Filtered data does not match expected data:\ngot: %+v\nexp: %+v", data, inputdata.expect)
-			}
+
+			assert.EqualValues(t, inputdata.expect, *data)
 		})
 	}
 }

lib/helpers_test.go

@@ -19,9 +19,9 @@ package lib
 
 import (
 	"fmt"
-	"reflect"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/tlinden/tablizer/cfg"
 )
 
@@ -39,9 +39,8 @@ func TestContains(t *testing.T) {
 		testname := fmt.Sprintf("contains-%d,%d,%t", tt.list, tt.search, tt.want)
 		t.Run(testname, func(t *testing.T) {
 			answer := contains(tt.list, tt.search)
-			if answer != tt.want {
-				t.Errorf("got %t, want %t", answer, tt.want)
-			}
+
+			assert.EqualValues(t, tt.want, answer)
 		})
 	}
 }
@@ -77,14 +76,12 @@ func TestPrepareColumns(t *testing.T) {
 		t.Run(testname, func(t *testing.T) {
 			conf := cfg.Config{Columns: testdata.input}
 			err := PrepareColumns(&conf, &data)
-			if err != nil {
-				if !testdata.wanterror {
-					t.Errorf("got error: %v", err)
-				}
+
+			if testdata.wanterror {
+				assert.Error(t, err)
 			} else {
-				if !reflect.DeepEqual(conf.UseColumns, testdata.exp) {
-					t.Errorf("got: %v, expected: %v", conf.UseColumns, testdata.exp)
-				}
+				assert.NoError(t, err)
+				assert.EqualValues(t, testdata.exp, conf.UseColumns)
 			}
 		})
 	}
@@ -153,18 +150,13 @@ func TestPrepareTransposerColumns(t *testing.T) {
 		t.Run(testname, func(t *testing.T) {
 			conf := cfg.Config{TransposeColumns: testdata.input, Transposers: testdata.transp}
 			err := PrepareTransposerColumns(&conf, &data)
-			if err != nil {
-				if !testdata.wanterror {
-					t.Errorf("got error: %v", err)
-				}
-			} else {
-				if len(conf.UseTransposeColumns) != testdata.exp {
-					t.Errorf("got %d, want %d", conf.UseTransposeColumns, testdata.exp)
-				}
 
-				if len(conf.Transposers) != len(conf.UseTransposeColumns) {
-					t.Errorf("got %d, want %d", conf.UseTransposeColumns, testdata.exp)
-				}
+			if testdata.wanterror {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+				assert.EqualValues(t, testdata.exp, len(conf.UseTransposeColumns))
+				assert.EqualValues(t, len(conf.UseTransposeColumns), len(conf.Transposers))
 			}
 		})
 	}
@@ -202,10 +194,8 @@ func TestReduceColumns(t *testing.T) {
 			c := cfg.Config{Columns: "x", UseColumns: testdata.columns}
 			data := Tabdata{entries: input}
 			reduceColumns(c, &data)
-			if !reflect.DeepEqual(data.entries, testdata.expect) {
-				t.Errorf("reduceColumns returned invalid data:\ngot: %+v\nexp: %+v",
-					data.entries, testdata.expect)
-			}
+
+			assert.EqualValues(t, testdata.expect, data.entries)
 		})
 	}
 }
@@ -233,10 +223,8 @@ func TestNumberizeHeaders(t *testing.T) {
 			conf := cfg.Config{Columns: "x", UseColumns: testdata.columns, Numbering: testdata.numberize}
 			usedata := data
 			numberizeAndReduceHeaders(conf, &usedata)
-			if !reflect.DeepEqual(usedata.headers, testdata.expect) {
-				t.Errorf("numberizeAndReduceHeaders returned invalid data:\ngot: %+v\nexp: %+v",
-					usedata.headers, testdata.expect)
-			}
+
+			assert.EqualValues(t, testdata.expect, usedata.headers)
 		})
 	}
 }

lib/parser.go

@@ -1,5 +1,5 @@
 /*
-Copyright © 2022-2024 Thomas von Dein
+Copyright © 2022-2025 Thomas von Dein
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -20,8 +20,11 @@ package lib
 import (
 	"bufio"
 	"encoding/csv"
+	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
+	"log"
 	"regexp"
 	"strings"
 
@@ -39,6 +42,8 @@ func Parse(conf cfg.Config, input io.Reader) (Tabdata, error) {
 	// first step, parse the data
 	if len(conf.Separator) == 1 {
 		data, err = parseCSV(conf, input)
+	} else if conf.InputJSON {
+		data, err = parseJSON(conf, input)
 	} else {
 		data, err = parseTabular(conf, input)
 	}
@@ -172,6 +177,109 @@ func parseTabular(conf cfg.Config, input io.Reader) (Tabdata, error) {
 	return data, nil
 }
 
+/*
+Parse JSON input.  We only support an array of  maps.
+*/
+func parseRawJSON(conf cfg.Config, input io.Reader) (Tabdata, error) {
+	dec := json.NewDecoder(input)
+	headers := []string{}
+	idxmap := map[string]int{}
+	data := [][]string{}
+	row := []string{}
+	iskey := true
+	haveheaders := false
+	var currentfield string
+	var idx int
+	var isjson bool
+
+	for {
+		t, err := dec.Token()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		switch val := t.(type) {
+		case string:
+			if iskey {
+				if !haveheaders {
+					// consider only the keys of the first item as headers
+					headers = append(headers, val)
+				}
+				currentfield = val
+			} else {
+				if !haveheaders {
+					// the first row uses the order as it comes in
+					row = append(row, val)
+				} else {
+					// use  the pre-determined  order, that  way items
+					// can be in any order as long as they contain all
+					// neccessary  fields. They may also  contain less
+					// fields than the  first item, these will contain
+					// the empty string
+					row[idxmap[currentfield]] = val
+				}
+			}
+		case json.Delim:
+			if val.String() == "}" {
+				data = append(data, row)
+				row = make([]string, len(headers))
+				idx++
+
+				if !haveheaders {
+					// remember  the array position of  header fields,
+					// which we use to  assign elements to the correct
+					// row index
+					for i, header := range headers {
+						idxmap[header] = i
+					}
+				}
+
+				haveheaders = true
+			}
+			isjson = true
+		}
+
+		iskey = !iskey
+	}
+
+	if isjson && (len(headers) == 0 || len(data) == 0) {
+		return Tabdata{}, errors.New("failed to parse JSON, input did not contain array of hashes")
+	}
+
+	return Tabdata{headers: headers, entries: data, columns: len(headers)}, nil
+}
+
+func parseJSON(conf cfg.Config, input io.Reader) (Tabdata, error) {
+	// parse raw json
+	data, err := parseRawJSON(conf, input)
+	if err != nil {
+		return data, err
+	}
+
+	// apply filter, if any
+	filtered := [][]string{}
+	var line string
+
+	for _, row := range data.entries {
+		line = strings.Join(row, " ")
+
+		if matchPattern(conf, line) == conf.InvertMatch {
+			continue
+		}
+
+		filtered = append(filtered, row)
+	}
+
+	if len(filtered) != len(data.entries) {
+		data.entries = filtered
+	}
+
+	return data, nil
+}
+
 func PostProcess(conf cfg.Config, data *Tabdata) (*Tabdata, bool, error) {
 	var modified bool
 

lib/parser_test.go

@@ -1,5 +1,5 @@
 /*
-Copyright © 2022 Thomas von Dein
+Copyright © 2022-2025 Thomas von Dein
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -19,10 +19,11 @@ package lib
 
 import (
 	"fmt"
-	"reflect"
+	"io"
 	"strings"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/tlinden/tablizer/cfg"
 )
 
@@ -67,27 +68,21 @@ func TestParser(t *testing.T) {
 		t.Run(testname, func(t *testing.T) {
 			readFd := strings.NewReader(strings.TrimSpace(testdata.text))
 			conf := cfg.Config{Separator: testdata.separator}
-			gotdata, err := Parse(conf, readFd)
+			gotdata, err := wrapValidateParser(conf, readFd)
 
-			if err != nil {
-				t.Errorf("Parser returned error: %s\nData processed so far: %+v", err, gotdata)
-			}
-
-			if !reflect.DeepEqual(data, gotdata) {
-				t.Errorf("Parser returned invalid data\nExp: %+v\nGot: %+v\n",
-					data, gotdata)
-			}
+			assert.NoError(t, err)
+			assert.EqualValues(t, data, gotdata)
 		})
 	}
 }
 
 func TestParserPatternmatching(t *testing.T) {
 	var tests = []struct {
-		name     string
-		entries  [][]string
-		patterns []*cfg.Pattern
-		invert   bool
-		want     bool
+		name      string
+		entries   [][]string
+		patterns  []*cfg.Pattern
+		invert    bool
+		wanterror bool
 	}{
 		{
 			name: "match",
@@ -121,18 +116,13 @@ func TestParserPatternmatching(t *testing.T) {
 				_ = conf.PreparePattern(testdata.patterns)
 
 				readFd := strings.NewReader(strings.TrimSpace(inputdata.text))
-				gotdata, err := Parse(conf, readFd)
+				data, err := wrapValidateParser(conf, readFd)
 
-				if err != nil {
-					if !testdata.want {
-						t.Errorf("Parser returned error: %s\nData processed so far: %+v",
-							err, gotdata)
-					}
+				if testdata.wanterror {
+					assert.Error(t, err)
 				} else {
-					if !reflect.DeepEqual(testdata.entries, gotdata.entries) {
-						t.Errorf("Parser returned invalid data (pattern: %s, invert: %t)\nExp: %+v\nGot: %+v\n",
-							testdata.name, testdata.invert, testdata.entries, gotdata.entries)
-					}
+					assert.NoError(t, err)
+					assert.EqualValues(t, testdata.entries, data.entries)
 				}
 			})
 		}
@@ -159,14 +149,147 @@ asd    igig
 
 	readFd := strings.NewReader(strings.TrimSpace(table))
 	conf := cfg.Config{Separator: cfg.DefaultSeparator}
-	gotdata, err := Parse(conf, readFd)
+	gotdata, err := wrapValidateParser(conf, readFd)
 
-	if err != nil {
-		t.Errorf("Parser returned error: %s\nData processed so far: %+v", err, gotdata)
+	assert.NoError(t, err)
+	assert.EqualValues(t, data, gotdata)
+}
+
+func TestParserJSONInput(t *testing.T) {
+	var tests = []struct {
+		name      string
+		input     string
+		expect    Tabdata
+		wanterror bool // true: expect fail, false: expect success
+	}{
+		{
+			// too deep nesting
+			name:      "invalidjson",
+			wanterror: true,
+			input: `[
+  {
+    "item": {
+       "NAME": "postgres-operator-7f4c7c8485-ntlns",
+       "READY": "1/1",
+       "STATUS": "Running",
+       "RESTARTS": "0",
+       "AGE": "24h"
+    }
+  }
+`,
+			expect: Tabdata{},
+		},
+
+		{
+			// one field missing + different order
+			// but shall not fail
+			name:      "kgpfail",
+			wanterror: false,
+			input: `[
+  {
+    "NAME": "postgres-operator-7f4c7c8485-ntlns",
+    "READY": "1/1",
+    "STATUS": "Running",
+    "RESTARTS": "0",
+    "AGE": "24h"
+  },
+  {
+    "NAME": "wal-g-exporter-778dcd95f5-wcjzn",
+    "RESTARTS": "0",
+    "READY": "1/1",
+    "AGE": "24h"
+  }
+]`,
+			expect: Tabdata{
+				columns: 5,
+				headers: []string{"NAME", "READY", "STATUS", "RESTARTS", "AGE"},
+				entries: [][]string{
+					[]string{
+						"postgres-operator-7f4c7c8485-ntlns",
+						"1/1",
+						"Running",
+						"0",
+						"24h",
+					},
+					[]string{
+						"wal-g-exporter-778dcd95f5-wcjzn",
+						"1/1",
+						"",
+						"0",
+						"24h",
+					},
+				},
+			},
+		},
+
+		{
+			name:      "kgp",
+			wanterror: false,
+			input: `[
+  {
+    "NAME": "postgres-operator-7f4c7c8485-ntlns",
+    "READY": "1/1",
+    "STATUS": "Running",
+    "RESTARTS": "0",
+    "AGE": "24h"
+  },
+  {
+    "NAME": "wal-g-exporter-778dcd95f5-wcjzn",
+    "STATUS": "Running",
+    "READY": "1/1",
+    "RESTARTS": "0",
+    "AGE": "24h"
+  }
+]`,
+			expect: Tabdata{
+				columns: 5,
+				headers: []string{"NAME", "READY", "STATUS", "RESTARTS", "AGE"},
+				entries: [][]string{
+					[]string{
+						"postgres-operator-7f4c7c8485-ntlns",
+						"1/1",
+						"Running",
+						"0",
+						"24h",
+					},
+					[]string{
+						"wal-g-exporter-778dcd95f5-wcjzn",
+						"1/1",
+						"Running",
+						"0",
+						"24h",
+					},
+				},
+			},
+		},
 	}
 
-	if !reflect.DeepEqual(data, gotdata) {
-		t.Errorf("Parser returned invalid data, Regex: %s\nExp: %+v\nGot: %+v\n",
-			conf.Separator, data, gotdata)
+	for _, testdata := range tests {
+		testname := fmt.Sprintf("parse-json-%s", testdata.name)
+		t.Run(testname, func(t *testing.T) {
+			conf := cfg.Config{InputJSON: true}
+
+			readFd := strings.NewReader(strings.TrimSpace(testdata.input))
+			data, err := wrapValidateParser(conf, readFd)
+
+			if testdata.wanterror {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+				assert.EqualValues(t, testdata.expect, data)
+			}
+		})
 	}
 }
+
+func wrapValidateParser(conf cfg.Config, input io.Reader) (Tabdata, error) {
+	data, err := Parse(conf, input)
+
+	if err != nil {
+		return data, err
+	}
+
+	err = ValidateConsistency(&data)
+
+	return data, err
+}

lib/printer_test.go

@@ -23,6 +23,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/tlinden/tablizer/cfg"
 )
 
@@ -307,13 +308,7 @@ func TestPrinter(t *testing.T) {
 
 			got := strings.TrimSpace(writer.String())
 
-			if got != exp {
-				t.Errorf("not rendered correctly:\n+++ got:\n%s\n+++ want:\n%s",
-					got, exp,
-					// strings.ReplaceAll(got, " ", "_"),
-					// strings.ReplaceAll(exp, " ", "_")
-				)
-			}
+			assert.EqualValues(t, exp, got)
 		})
 	}
 }

lib/sort_test.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/tlinden/tablizer/cfg"
 )
 
@@ -41,9 +42,7 @@ func TestDuration2Seconds(t *testing.T) {
 		testname := fmt.Sprintf("duration-%s", testdata.dur)
 		t.Run(testname, func(t *testing.T) {
 			seconds := duration2int(testdata.dur)
-			if seconds != testdata.expect {
-				t.Errorf("got %d, want %d", seconds, testdata.expect)
-			}
+			assert.EqualValues(t, testdata.expect, seconds)
 		})
 	}
 }
@@ -74,9 +73,7 @@ func TestCompare(t *testing.T) {
 		t.Run(testname, func(t *testing.T) {
 			c := cfg.Config{SortMode: testdata.mode, SortDescending: testdata.desc}
 			got := compare(&c, testdata.a, testdata.b)
-			if got != testdata.want {
-				t.Errorf("got %d, want %d", got, testdata.want)
-			}
+			assert.EqualValues(t, testdata.want, got)
 		})
 	}
 }

lib/yank_test.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/tiagomelo/go-clipboard/clipboard"
 	"github.com/tlinden/tablizer/cfg"
 )
@@ -59,14 +60,9 @@ func DISABLED_TestYankColumns(t *testing.T) {
 			printData(&writer, conf, &data)
 
 			got, err := cb.PasteText()
-			if err != nil {
-				t.Errorf("failed to fetch yanked text from clipboard")
-			}
 
-			if got != testdata.expect {
-				t.Errorf("not yanked correctly:\n+++ got:\n%s\n+++ want:\n%s",
-					got, testdata.expect)
-			}
+			assert.NoError(t, err)
+			assert.EqualValues(t, testdata.expect, got)
 		})
 	}
 }