From 98a577f68afae971a70508e7eb767c2bde5cae1c Mon Sep 17 00:00:00 2001 From: "T. von Dein" Date: Fri, 22 May 2026 13:50:17 +0200 Subject: [PATCH] add doc show, enhance search, add jsonpath to search and doc show --- cmd/doc.go | 64 ++++++++++++++++-- cmd/search.go | 72 +++++++++++++++++--- go.mod | 3 + go.sum | 6 ++ pkg/cfg/config.go | 2 + pkg/es/doc.go | 40 ++++++++++- pkg/es/index.go | 8 +++ pkg/es/search.go | 55 ++++++--------- pkg/es/search_filter.go | 146 ++++++++++++++++++++++++++++++++++++++++ pkg/printer/table.go | 4 +- 10 files changed, 348 insertions(+), 52 deletions(-) create mode 100644 pkg/es/search_filter.go diff --git a/cmd/doc.go b/cmd/doc.go index 3ad3833..b6d2508 100644 --- a/cmd/doc.go +++ b/cmd/doc.go @@ -33,6 +33,7 @@ func Doc(conf *cfg.Config) *cli.Command { Commands: []*cli.Command{ DocAdd(conf), + DocShow(conf), //Delete(conf), }, } @@ -43,16 +44,71 @@ func DocAdd(conf *cfg.Config) *cli.Command { Name: "add", Aliases: []string{"+"}, Usage: "add JSON document index", - UsageText: "add [options] ''", + UsageText: "add [options] -i ''", + + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "index", + Usage: "index to work on", + Sources: cli.EnvVars("ES_INDEX"), + Destination: &conf.Index, + Aliases: []string{"i"}, + }, + }, Action: func(ctx context.Context, cmd *cli.Command) error { args := cmd.Args() - if args.Len() != 2 { - return errors.New("missing arguments: ") + if args.Len() != 1 { + return errors.New("missing arguments: ") } - return es.DocAdd(conf, cmd.Args().Get(0), cmd.Args().Get(1)) + return es.DocAdd(conf, cmd.Args().Get(0)) + }, + } +} + +func DocShow(conf *cfg.Config) *cli.Command { + return &cli.Command{ + Name: "show", + Aliases: []string{"sh"}, + Usage: "show a JSON document", + UsageText: "show [options] -i ", + + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "index", + Usage: "index to work on", + Sources: cli.EnvVars("ES_INDEX"), + Destination: &conf.Index, + Aliases: []string{"i"}, + }, + &cli.StringFlag{ + Name: "path", + Usage: "jsonPath filter (e.g. source.message)", + Destination: &conf.Path, + Aliases: []string{"p"}, + }, + &cli.BoolFlag{ + Name: "help-jsonpath", + Usage: "show jsonPath help", + Destination: &conf.Subhelp, + Aliases: []string{"H"}, + }, + }, + + Action: func(ctx context.Context, cmd *cli.Command) error { + if conf.Subhelp { + return showJsonPathHelp() + } + + args := cmd.Args() + + if args.Len() != 1 { + return errors.New("missing arguments: ") + } + + return es.DocShow(conf, cmd.Args().Get(0)) }, } } diff --git a/cmd/search.go b/cmd/search.go index b193393..5bed9f1 100644 --- a/cmd/search.go +++ b/cmd/search.go @@ -18,7 +18,7 @@ package cmd import ( "context" - "errors" + "fmt" "codeberg.org/scip/esctl/pkg/cfg" "codeberg.org/scip/esctl/pkg/es" @@ -28,10 +28,20 @@ import ( func Search(conf *cfg.Config) *cli.Command { return &cli.Command{ - Name: "search", - Aliases: []string{"/"}, - Usage: "search within an index", - UsageText: "search [options] ...", + Name: "search", + Aliases: []string{"/"}, + Usage: "search within an index", + UsageText: `search [options] [<[field]pattern> ...] + + might be one of: + =: Must match +!=: Must not match + ?: Should match + +You can omit a field spec and thereby search across all fields. + +You can also search multiple fields by separating them with comma, eg: +user,group=root`, Flags: []cli.Flag{ &cli.StringFlag{ @@ -57,19 +67,63 @@ func Search(conf *cfg.Config) *cli.Command { }, &cli.StringSliceFlag{ Name: "filter", - Usage: "additional filters. format: key=value", + Usage: "additional boolean filters. format: key=value", Destination: &conf.Filter, Aliases: []string{"F"}, }, + &cli.StringFlag{ + Name: "jsonpath", + Usage: "jsonPath filter (e.g. source.message)", + Destination: &conf.Path, + Aliases: []string{"p"}, + }, + &cli.BoolFlag{ + Name: "help-jsonpath", + Usage: "show jsonPath help", + Destination: &conf.Subhelp, + Aliases: []string{"H"}, + }, }, Action: func(ctx context.Context, cmd *cli.Command) error { - args := cmd.Args() - if args.Len() == 0 { - return errors.New("at least one query must be specified (format: field=pattern)") + if conf.Subhelp { + return showJsonPathHelp() } + args := cmd.Args() + return es.Search(conf, args.Slice()) }, } } + +func showJsonPathHelp() error { + _, err := fmt.Println(`jsonPath usage: + +name.last >> "Anderson" +age >> 37 +children >> ["Sara","Alex","Jack"] +children.# >> 3 +children.1 >> "Alex" +child*.2 >> "Jack" +c?ildren.0 >> "Sara" +fav\.movie >> "Deer Hunter" +friends.#.first >> ["Dale","Roger","Jane"] +friends.1.last >> "Craig" + +You can also query an array for the first match by using #(...), or +find all matches with #(...)#. Queries support the ==, !=, <, <=, >, +>= comparison operators and the simple pattern matching % (like) and +!% (not like) operators. Eg: + +friends.#(last=="Murphy").first >> "Dale" +friends.#(last=="Murphy")#.first >> ["Dale","Jane"] +friends.#(age>45)#.last >> ["Craig","Murphy"] +friends.#(first%"D*").last >> "Murphy" +friends.#(first!%"D*").last >> "Craig" +friends.#(nets.#(=="fb"))#.first >> ["Dale","Roger"] + +Documentation: https://github.com/tidwall/gjson/blob/master/SYNTAX.md`) + + return err +} diff --git a/go.mod b/go.mod index 70422e9..2471917 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,9 @@ require ( github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect github.com/olekukonko/errors v1.2.0 // indirect github.com/olekukonko/ll v0.1.8 // indirect + github.com/tidwall/gjson v1.19.0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/otel v1.35.0 // indirect go.opentelemetry.io/otel/metric v1.35.0 // indirect diff --git a/go.sum b/go.sum index 9e4f0ca..49c15bf 100644 --- a/go.sum +++ b/go.sum @@ -58,6 +58,12 @@ github.com/olekukonko/ll v0.1.8 h1:ysHCJRGHYKzmBSdz9w5AySztx7lG8SQY+naTGYUbsz8= github.com/olekukonko/ll v0.1.8/go.mod h1:RPRC6UcscfFZgjo1nulkfMH5IM0QAYim0LfnMvUuozw= github.com/olekukonko/tablewriter v1.1.4 h1:ORUMI3dXbMnRlRggJX3+q7OzQFDdvgbN9nVWj1drm6I= github.com/olekukonko/tablewriter v1.1.4/go.mod h1:+kedxuyTtgoZLwif3P1Em4hARJs+mVnzKxmsCL/C5RY= +github.com/tidwall/gjson v1.19.0 h1:xwxm7n691Uf3u5OFjzngavjGTh55KX5q/9w9xHW88JU= +github.com/tidwall/gjson v1.19.0/go.mod h1:V37/opeE/JbLUOfH0QTXiNez2l0RUjYUhpT4szFQAfc= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tlinden/yadu v0.1.3 h1:5cRCUmj+l5yvlM2irtpFBIJwVV2DPEgYSaWvF19FtcY= github.com/tlinden/yadu v0.1.3/go.mod h1:l3bRmHKL9zGAR6pnBHY2HRPxBecf7L74BoBgOOpTcUA= github.com/urfave/cli/v3 v3.8.0 h1:XqKPrm0q4P0q5JpoclYoCAv0/MIvH/jZ2umzuf8pNTI= diff --git a/pkg/cfg/config.go b/pkg/cfg/config.go index 4cc710d..4f95ed1 100644 --- a/pkg/cfg/config.go +++ b/pkg/cfg/config.go @@ -58,6 +58,8 @@ type Config struct { Primary bool // index allocation: -p From, To, MaxItems int // search: flags Filter []string // search: -F + Path string // search+doc sh: -p + Subhelp bool // search+doc sh: -H Exclude string // cluster compare: -e (regexp) All, Verbose bool // cluster status: -a -v Persistent, Transient, Default bool // -p -t -D cluster settings set diff --git a/pkg/es/doc.go b/pkg/es/doc.go index 66905f0..bbe94f8 100644 --- a/pkg/es/doc.go +++ b/pkg/es/doc.go @@ -19,10 +19,12 @@ package es import ( "context" "encoding/json" + "errors" "fmt" "time" "codeberg.org/scip/esctl/pkg/cfg" + "github.com/tidwall/gjson" ) // create index with: @@ -32,7 +34,7 @@ import ( // then add a doc: // // esctl doc add foo2 '{"id":"d8d8d","user":"scip"}' -func DocAdd(conf *cfg.Config, index, jsondoc string) error { +func DocAdd(conf *cfg.Config, jsondoc string) error { data := map[string]any{} err := json.Unmarshal([]byte(jsondoc), &data) @@ -42,14 +44,46 @@ func DocAdd(conf *cfg.Config, index, jsondoc string) error { now := fmt.Sprintf("%d", time.Now().Unix()) - _, err = conf.DefaultCluster.ES.Create(index, now). + res, err := conf.DefaultCluster.ES.Create(conf.Index, now). Document(data). Header("content-type", "application/json"). Header("accept", "application/json"). Do(context.Background()) if err != nil { - return fmt.Errorf("failed to create new doc in index %s: %s", index, err) + return fmt.Errorf("failed to create new doc in index %s: %s", conf.Index, err) } + fmt.Println(res.Id_) + + return nil +} + +func DocShow(conf *cfg.Config, id string) error { + res, err := conf.DefaultCluster.ES.Get(conf.Index, id). + Header("content-type", "application/json"). + Header("accept", "application/json"). + Do(context.Background()) + if err != nil { + return fmt.Errorf("failed to retrieve doc in index %s: %s", conf.Index, err) + } + + if !res.Found { + return errors.New("no document with that id found") + } + + docjson := fmt.Sprintf(`{"id":%s, "index":"%s", "source":%s}`, + res.Id_, + res.Index_, + res.Source_) + + if conf.Path != "" { + value := gjson.Get(docjson, conf.Path) + fmt.Println(value.String()) + } else { + fmt.Println(docjson) + } + + fmt.Println() + return nil } diff --git a/pkg/es/index.go b/pkg/es/index.go index ee4588f..ed5e589 100644 --- a/pkg/es/index.go +++ b/pkg/es/index.go @@ -129,6 +129,13 @@ func IndexShow(conf *cfg.Config, index string) error { slog.Debug("ES result", "index", res) + fields := make([]string, len(res[index].Mappings.Properties)) + idx := 0 + for field := range res[index].Mappings.Properties { + fields[idx] = field + idx++ + } + table := printer.NewTable(conf, 2, 5) table.Addheaders("index property", "value") @@ -145,6 +152,7 @@ func IndexShow(conf *cfg.Config, index string) error { {"shards", *res[index].Settings.Index.NumberOfShards}, {"created", created.Format("2006-01-02 15:04:05")}, {"uuid", *res[index].Settings.Index.Uuid}, + {"fields", strings.Join(fields, ",")}, } if err := table.Print(); err != nil { diff --git a/pkg/es/search.go b/pkg/es/search.go index a417408..61078a1 100644 --- a/pkg/es/search.go +++ b/pkg/es/search.go @@ -20,12 +20,9 @@ import ( "context" "fmt" "log/slog" - "strings" "codeberg.org/scip/esctl/pkg/cfg" - "github.com/elastic/go-elasticsearch/v9/typedapi/core/search" - "github.com/elastic/go-elasticsearch/v9/typedapi/esdsl" - "github.com/elastic/go-elasticsearch/v9/typedapi/types" + "github.com/tidwall/gjson" ) /* @@ -35,40 +32,19 @@ Execute an ES search. additional filters can be given as -F key=value */ func Search(conf *cfg.Config, queries []string) error { - query := esdsl.NewBoolQuery() + search := conf.DefaultCluster.ES.Search(). + Index(conf.Index) - for _, q := range queries { - parts := strings.Split(q, "=") - if len(parts) != 2 { - return fmt.Errorf("search queries must be in the form field=pattern") + if len(queries) > 0 { + req, err := prepareQuery(conf, queries) + if err != nil { + return err } - query.Must(esdsl.NewMatchQuery(parts[0], parts[1])) + search.Request(req) } - filters := make([]types.QueryVariant, len(conf.Filter)) - - for idx, filter := range conf.Filter { - parts := strings.Split(filter, "=") - if len(parts) != 2 { - return fmt.Errorf("invalid filter spec: %s, expecting key=value", filter) - } - - filters[idx] = esdsl.NewTermQuery(parts[0], esdsl.NewFieldValue().String(parts[1])) - } - - if len(filters) > 0 { - query.Filter(filters...) - } - - res, err := conf.DefaultCluster.ES.Search(). - Index(conf.Index). - Request(&search.Request{ - Query: query.QueryCaster(), - From: &conf.From, - Size: &conf.To, - }). - Do(context.Background()) + res, err := search.Do(context.Background()) if err != nil { return fmt.Errorf("failed to run search (esdsl): %s", err) } @@ -76,7 +52,18 @@ func Search(conf *cfg.Config, queries []string) error { slog.Debug("ES result", "search", res) for _, hit := range res.Hits.Hits { - fmt.Printf("%s\n", hit.Source_) + docjson := fmt.Sprintf(`{"id":%s, "score":%0.4f, "index":"%s", "source":%s}`, + *hit.Id_, + *hit.Score_, + hit.Index_, + hit.Source_) + + if conf.Path != "" { + value := gjson.Get(docjson, conf.Path) + fmt.Println(value.String()) + } else { + fmt.Println(docjson) + } } return nil diff --git a/pkg/es/search_filter.go b/pkg/es/search_filter.go new file mode 100644 index 0000000..d1715d1 --- /dev/null +++ b/pkg/es/search_filter.go @@ -0,0 +1,146 @@ +/* +Copyright © 2026 Thomas von Dein + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ +package es + +import ( + "fmt" + "strings" + + "codeberg.org/scip/esctl/pkg/cfg" + "github.com/elastic/go-elasticsearch/v9/typedapi/core/search" + "github.com/elastic/go-elasticsearch/v9/typedapi/esdsl" + "github.com/elastic/go-elasticsearch/v9/typedapi/types" +) + +const ( + Fmust = iota + Fmustnot + Fshould +) + +// hold filter configuration, can be used multiple times through a search +type filter struct { + term, filter string // message=foo + mterm []string // multi_match + criteria int // must(=), mustnot(!=), should(?) + multi bool // message,title=foo => filter:foo, multi: []string{"message","title"} +} + +// build a new filter object +func NewFilter(query string) (*filter, error) { + var separator string + var criteria int // we use the constants on top for this + + switch { + case strings.Contains(query, "!="): + criteria = Fmustnot + separator = "!=" + case strings.Contains(query, "?"): + criteria = Fshould + separator = "?" + default: + criteria = Fmust + separator = "=" + } + + part := strings.Split(query, separator) + if len(part) != 2 { + return nil, fmt.Errorf("search queries must be in the form fieldpattern where must be one of: =, !=, ?") + } + + f := &filter{term: part[0], filter: part[1], criteria: criteria} + + if strings.Contains(part[0], ",") { + // a MultiMatchQuery, match across multiple fields at once + multi := strings.Split(part[0], ",") + f.multi = true + f.mterm = multi + } + + return f, nil +} + +// prepare q user search query and turn it into a proper search request +func prepareQuery(conf *cfg.Config, queries []string) (*search.Request, error) { + if len(queries) == 0 { + // nothing given, just return all docs, if any + return &search.Request{ + Query: esdsl.NewMatchAllQuery().QueryCaster(), + From: &conf.From, + Size: &conf.To, + }, nil + } + + if len(queries) == 1 && !strings.ContainsAny(queries[0], "!=?") { + // a general query w/o fields, search across all fields + return &search.Request{ + Query: esdsl.NewSimpleQueryStringQuery(queries[0]).QueryCaster(), + From: &conf.From, + Size: &conf.To, + }, nil + } + + // complex query, form a proper query struct + query := esdsl.NewBoolQuery() + + for _, q := range queries { + filter, err := NewFilter(q) + if err != nil { + return nil, err + } + + // by default we match on a single field + var match types.QueryVariant = esdsl.NewMatchQuery(filter.term, filter.filter) + + if filter.multi { + // ok, match across multiple given fields + match = esdsl.NewMultiMatchQuery(filter.filter).Fields(filter.mterm...) + } + + // apply logic + switch filter.criteria { + case Fmustnot: + query.MustNot(match) + case Fmust: + query.Must(match) + case Fshould: + query.Should(match) + } + } + + // there might be boolean filters as well + filters := make([]types.QueryVariant, len(conf.Filter)) + + for idx, filter := range conf.Filter { + parts := strings.Split(filter, "=") + if len(parts) != 2 { + return nil, fmt.Errorf("invalid filter spec: %s, expecting key=value", filter) + } + + filters[idx] = esdsl.NewTermQuery(parts[0], esdsl.NewFieldValue().String(parts[1])) + } + + if len(filters) > 0 { + query.Filter(filters...) + } + + return &search.Request{ + Query: query.QueryCaster(), + From: &conf.From, + Size: &conf.To, + }, nil +} diff --git a/pkg/printer/table.go b/pkg/printer/table.go index 7ee17d9..a0eb3dc 100644 --- a/pkg/printer/table.go +++ b/pkg/printer/table.go @@ -285,9 +285,9 @@ func (data *Table) Addheaders(headers ...string) { for idx, header := range headers { switch data.Mode { case "json", "yaml": - data.Headers[idx] = strings.ToLower(header) + data.Headers[idx] = strings.ReplaceAll(strings.ToLower(header), " ", "_") default: - data.Headers[idx] = bold(strings.ToUpper(header)) + data.Headers[idx] = bold(strings.ReplaceAll(strings.ToUpper(header), " ", "-")) } } }