marshall explain output only in debug mode

README.md

@@ -4,6 +4,48 @@
 
 Elasticsearch CLI
 
+## Introduction
+
+This is a handy cli tool which interfaces to an elasticsearch cluster
+(or two of them if you're using cross cluster replication). It is a
+work-in-progress project yet, things might change occasionally. Expect
+a stable release once we reach major version 1.0.0.
+
+Features:
+
+- Configuration of cluster credentials using environment vars or
+  config file. Multiple clusters can be configured. `esctl cluster ls`
+  shows which one is reachable.
+- Shell completion support (bash, zsh and fish). Put this into your
+  rc: `source <(esctl completion bash)`.
+- Cluster settings can be viewed and modified.
+- Search: you can search indices using full text or by fields, select
+  logical condition (OR, AND), use PIT, limit datetime (ES date math
+  can be used), etc. It is however not yet possible to create
+  recursive searches like: `(cond1 AND cond2) OR (cond3 OR cond4)`.
+- Cross cluster replication (ccr): view, pause, resume, delete
+  replication. You can also manage follower configuration.
+- Index management: manage aliases, create, modify, delete indices,
+  display field mappings etc.
+- Node management: only list nodes yet.
+- Shard management: only list shards yet.
+- Snapshot management: only list snapshots yet.
+- Role management: only list roles yet. There's also a `role diff`
+  subcommand, which is for internal use. It can be used to verify if
+  role defs in a CSV match the deployed roles.
+- Repl: this is an interactive REPL (read eval print loop) towards the
+  elasticsearch API. You can run API calls on the current selected
+  cluster w/o the hassle to specify the whole url, credentials etc. It
+  has line editing and history support. If `jq` is installed output
+  JSON will be syntax highlighted.
+- Doc support. You can put, delete and show docs for an index. Very
+  handy if you want to play with it. Just create a new index:
+  `esctl index create foo` and then insert docs into it for search
+  experiments:
+  ```console
+  esctl doc add -i foo '{"title":"curry in a hurry", "message":"australian thai"}'
+  ```
+
 ## Usage
 
 Command tree:
@@ -28,9 +70,13 @@ Command tree:
          list
          set
      status
+ debug
  doc
      add
+     delete
+     show
  help
+ help-jsonpath
  index
      alias
          create
@@ -40,6 +86,7 @@ Command tree:
      close
      create
      delete
+     fields
      list
      modify
      show
@@ -47,6 +94,10 @@ Command tree:
      list
      show
  repl
+ role
+     diff
+     list
+     show
  search
  shard
      list
@@ -84,10 +135,6 @@ can omit `-c ...`.
 If you want to  work on a specific cluster, specify  its name with the
 global `-C` option.
 
-## Introduction
-
-FIXME
-
 ## Installation
 
 The tool does not have any dependencies.  Just download the binary for
@@ -138,6 +185,30 @@ make
 sudo make install
 ```
 
+# Development
+
+## To test completion
+
+Add the flag `--generate-shell-completion` to any command, e.g.:
+
+```console
+./esctl role show --generate-shell-completion
+machine_learning_admin
+rollup_admin
+editor
+reporting_user
+snapshot_user
+fcn_admin
+machine_learning_user
+kibana_system
+beats_admin
+kibana_user
+fcns_space
+transport_client
+transform_user
+[..]
+```
+
 # Report bugs
 
 [Please open an issue](https://codeberg.org/scip/esctl/issues). Thanks!

TODO.md

@@ -1,11 +1,13 @@
 - [Go client docs](https://www.elastic.co/docs/reference/elasticsearch/clients/go/typed-api)
 - [ES API docs](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-get)
 
-- Fix index names custom completion
-  https://github.com/urfave/cli/issues/2332
-  https://github.com/urfave/cli/issues/2333
-  
 - index show: add more details, see screenshots
 
 - add shard explain, aka:
   get /_cluster/allocation/explain {"index":"yourindex", "primary": true, "shard":0}
+
+
+- add datastream support:
+  https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-get-data-stream
+  
+  also exclude data stream backing indices from index ls

cmd/ccr.go

@@ -47,6 +47,7 @@ func CcrStatus(conf *cfg.Config) *cli.Command {
 		Name:      "status",
 		Aliases:   []string{"st"},
 		Usage:     "cross cluster replication status (yaml config with 2 clusters required)",
+		UsageText: "status <leader> <follower>",
 
 		Flags: []cli.Flag{
 			&cli.StringFlag{
@@ -57,6 +58,10 @@ func CcrStatus(conf *cfg.Config) *cli.Command {
 			},
 		},
 
+		ShellComplete: func(ctx context.Context, cmd *cli.Command) {
+			completeCluster(cmd)
+		},
+
 		Action: func(ctx context.Context, cmd *cli.Command) error {
 			leader := cmd.Args().Get(0)
 			follower := cmd.Args().Get(1)
@@ -109,6 +114,10 @@ func CcrRemoteInfo(conf *cfg.Config) *cli.Command {
 		Usage:     "show ccr remote info",
 		UsageText: "info [options] [<index>]",
 
+		ShellComplete: func(ctx context.Context, cmd *cli.Command) {
+			completeIndex(cmd)
+		},
+
 		Action: func(ctx context.Context, cmd *cli.Command) error {
 			return es.CcrRemoteInfo(conf, cmd.Args().Get(0))
 		},

cmd/ccr_follower.go

@@ -59,6 +59,10 @@ func CcrFollowerRenew(conf *cfg.Config) *cli.Command {
 			},
 		},
 
+		ShellComplete: func(ctx context.Context, cmd *cli.Command) {
+			completeIndex(cmd)
+		},
+
 		Action: func(ctx context.Context, cmd *cli.Command) error {
 			args := cmd.Args()
 
@@ -77,6 +81,10 @@ func CcrFollowerResume(conf *cfg.Config) *cli.Command {
 		Usage:     "resume ccr index to follow",
 		UsageText: "resume [options] <index>",
 
+		ShellComplete: func(ctx context.Context, cmd *cli.Command) {
+			completeIndex(cmd)
+		},
+
 		Action: func(ctx context.Context, cmd *cli.Command) error {
 			args := cmd.Args()
 
@@ -95,6 +103,10 @@ func CcrFollowerPause(conf *cfg.Config) *cli.Command {
 		Usage:     "pause ccr index to follow",
 		UsageText: "pause [options] <index>",
 
+		ShellComplete: func(ctx context.Context, cmd *cli.Command) {
+			completeIndex(cmd)
+		},
+
 		Action: func(ctx context.Context, cmd *cli.Command) error {
 			args := cmd.Args()
 
@@ -113,6 +125,10 @@ func CcrFollowerUnfollow(conf *cfg.Config) *cli.Command {
 		Usage:     "unfollow ccr follower index",
 		UsageText: "unfollow [options] <index>",
 
+		ShellComplete: func(ctx context.Context, cmd *cli.Command) {
+			completeIndex(cmd)
+		},
+
 		Action: func(ctx context.Context, cmd *cli.Command) error {
 			args := cmd.Args()
 
@@ -141,6 +157,10 @@ func CcrFollowerAdd(conf *cfg.Config) *cli.Command {
 			},
 		},
 
+		ShellComplete: func(ctx context.Context, cmd *cli.Command) {
+			completeIndex(cmd)
+		},
+
 		Action: func(ctx context.Context, cmd *cli.Command) error {
 			args := cmd.Args()
 
@@ -160,6 +180,10 @@ func CcrFollowerDelete(conf *cfg.Config) *cli.Command {
 		Usage:     "delete ccr follower index",
 		UsageText: "delete <index>",
 
+		ShellComplete: func(ctx context.Context, cmd *cli.Command) {
+			completeIndex(cmd)
+		},
+
 		Action: func(ctx context.Context, cmd *cli.Command) error {
 			args := cmd.Args()
 
@@ -180,6 +204,10 @@ func CcrFollowerShow(conf *cfg.Config) *cli.Command {
 		Usage:     "show ccr follower index details",
 		UsageText: "show <index>",
 
+		ShellComplete: func(ctx context.Context, cmd *cli.Command) {
+			completeIndex(cmd)
+		},
+
 		Action: func(ctx context.Context, cmd *cli.Command) error {
 			args := cmd.Args()
 

cmd/completion.go

@@ -0,0 +1,89 @@
+/*
+Copyright © 2026 Thomas von Dein
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+package cmd
+
+import (
+	"fmt"
+
+	"codeberg.org/scip/esctl/pkg/cfg"
+	"codeberg.org/scip/esctl/pkg/es"
+
+	"github.com/urfave/cli/v3"
+)
+
+func completeIndex(cmd *cli.Command) {
+	if cmd.NArg() > 0 {
+		return
+	}
+
+	// FIXME: config should load from root.Before(), see https://github.com/urfave/cli/issues/2348
+	// workaround: load it directly here
+	conf := cfg.NewConfig()
+	if err := conf.Init(); err != nil {
+		return
+	}
+
+	indices, err := es.IndexNames(conf)
+	if err != nil {
+		return
+	}
+
+	for _, index := range indices {
+		fmt.Println(index)
+	}
+
+}
+
+func completeRole(cmd *cli.Command) {
+	if cmd.NArg() > 0 {
+		return
+	}
+
+	// FIXME: config should load from root.Before(), see https://github.com/urfave/cli/issues/2348
+	// workaround: load it directly here
+	conf := cfg.NewConfig()
+	if err := conf.Init(); err != nil {
+		return
+	}
+
+	roles, err := es.RoleNames(conf)
+	if err != nil {
+		return
+	}
+
+	for _, role := range roles {
+		fmt.Println(role)
+	}
+
+}
+
+func completeCluster(cmd *cli.Command) {
+	if cmd.NArg() > 0 {
+		return
+	}
+
+	// FIXME: config should load from root.Before(), see https://github.com/urfave/cli/issues/2348
+	// workaround: load it directly here
+	conf := cfg.NewConfig()
+	if err := conf.Init(); err != nil {
+		return
+	}
+
+	for cluster := range conf.Clusters {
+		fmt.Println(cluster)
+	}
+}

cmd/doc.go

@@ -34,7 +34,7 @@ func Doc(conf *cfg.Config) *cli.Command {
 		Commands: []*cli.Command{
 			DocAdd(conf),
 			DocShow(conf),
-			//Delete(conf),
+			DocDelete(conf),
 		},
 	}
 }
@@ -89,19 +89,9 @@ func DocShow(conf *cfg.Config) *cli.Command {
 				Destination: &conf.Path,
 				Aliases:     []string{"p"},
 			},
-			&cli.BoolFlag{
-				Name:        "help-jsonpath",
-				Usage:       "show jsonPath help",
-				Destination: &conf.Subhelp,
-				Aliases:     []string{"H"},
-			},
 		},
 
 		Action: func(ctx context.Context, cmd *cli.Command) error {
-			if conf.Subhelp {
-				return showJsonPathHelp()
-			}
-
 			args := cmd.Args()
 
 			if args.Len() != 1 {
@@ -112,3 +102,62 @@ func DocShow(conf *cfg.Config) *cli.Command {
 		},
 	}
 }
+
+func DocDelete(conf *cfg.Config) *cli.Command {
+	return &cli.Command{
+		Name:      "delete",
+		Aliases:   []string{"rm"},
+		Usage:     "delete JSON document[s] from index[es]",
+		UsageText: "delete [options] -i <index> [<[field<sep>]pattern> ...]\n" + SearchUsage,
+
+		Flags: []cli.Flag{
+			&cli.BoolFlag{
+				Name:        "all",
+				Usage:       "delete all docs (dangerous!)",
+				Destination: &conf.All,
+				Aliases:     []string{"a"},
+			},
+			&cli.StringFlag{
+				Name:        "index",
+				Usage:       "index to work on",
+				Sources:     cli.EnvVars("ES_INDEX"),
+				Destination: &conf.Index,
+				Aliases:     []string{"i"},
+			},
+			&cli.StringSliceFlag{
+				Name:        "filter",
+				Usage:       "additional boolean filters. format: key=value",
+				Destination: &conf.Filter,
+				Aliases:     []string{"F"},
+			},
+			&cli.StringFlag{
+				Name:        "timerange",
+				Usage:       "field:<date> to <date> (e.g. @timestamp:2026-05-05 to 2026-05-15)",
+				Destination: &conf.Range,
+				Aliases:     []string{"r"},
+			},
+			&cli.StringFlag{
+				Name:        "timestamp-format",
+				Usage:       "a valid ES builtin timestamp or custom format",
+				Destination: &conf.TimestampFormat,
+				Value:       "strict_date_hour_minute",
+			},
+			&cli.BoolFlag{
+				Name:        "or",
+				Usage:       "logical operator (default: and)",
+				Destination: &conf.Or,
+				Aliases:     []string{"O"},
+			},
+		},
+
+		Action: func(ctx context.Context, cmd *cli.Command) error {
+			args := cmd.Args()
+
+			if args.Len() == 0 && !conf.All {
+				return errors.New("missing arguments: <query>")
+			}
+
+			return es.DocDelete(conf, cmd.Args().Slice())
+		},
+	}
+}

cmd/index.go

@@ -42,6 +42,7 @@ func Index(conf *cfg.Config) *cli.Command {
 			IndexAllocation(conf),
 			IndexModify(conf),
 			IndexAlias(conf),
+			IndexFields(conf),
 		},
 	}
 }
@@ -132,22 +133,8 @@ func IndexShow(conf *cfg.Config) *cli.Command {
 			return es.IndexShow(conf, cmd.Args().Get(0))
 		},
 
-		// FIXME: doesn't work at all
-		// FIXME: also it would ONLY work if the user uses env vars, -C would not be
-		//        there when the completion output is being generated
 		ShellComplete: func(ctx context.Context, cmd *cli.Command) {
-			if cmd.NArg() > 0 {
+			completeIndex(cmd)
-				return
-			}
-
-			indices, err := es.IndexNames(conf)
-			if err != nil {
-				return
-			}
-
-			for _, index := range indices {
-				fmt.Println(index)
-			}
 		},
 	}
 }
@@ -157,6 +144,8 @@ func IndexCreate(conf *cfg.Config) *cli.Command {
 		Name:    "create",
 		Aliases: []string{"+"},
 		Usage:   "create a new index",
+		UsageText: `create <name> <fieldmapping:type>...
+Valid field mapping types: integer, text, date, keyword`,
 
 		Flags: []cli.Flag{
 			&cli.BoolFlag{
@@ -197,6 +186,11 @@ func IndexDelete(conf *cfg.Config) *cli.Command {
 		Name:      "delete",
 		Aliases:   []string{"rm"},
 		Usage:     "delete an index",
+		UsageText: "delete <index>",
+
+		ShellComplete: func(ctx context.Context, cmd *cli.Command) {
+			completeIndex(cmd)
+		},
 
 		Action: func(ctx context.Context, cmd *cli.Command) error {
 			index := cmd.Args().Get(0)
@@ -213,6 +207,11 @@ func IndexClose(conf *cfg.Config) *cli.Command {
 	return &cli.Command{
 		Name:      "close",
 		Usage:     "close an index",
+		UsageText: "close <index>",
+
+		ShellComplete: func(ctx context.Context, cmd *cli.Command) {
+			completeIndex(cmd)
+		},
 
 		Action: func(ctx context.Context, cmd *cli.Command) error {
 			index := cmd.Args().Get(0)
@@ -231,6 +230,10 @@ func IndexModify(conf *cfg.Config) *cli.Command {
 		Usage:     "modify an index",
 		UsageText: "modify <index[,index,...]|_all>",
 
+		ShellComplete: func(ctx context.Context, cmd *cli.Command) {
+			completeIndex(cmd)
+		},
+
 		Flags: []cli.Flag{
 			&cli.IntFlag{
 				Name:        "replicas",
@@ -250,3 +253,45 @@ func IndexModify(conf *cfg.Config) *cli.Command {
 		},
 	}
 }
+
+func IndexFields(conf *cfg.Config) *cli.Command {
+	return &cli.Command{
+		Name:      "fields",
+		Usage:     "show info about field capabilities",
+		UsageText: "index fields <index>",
+
+		Flags: []cli.Flag{
+			&cli.BoolFlag{
+				Name:        "aggretable",
+				Usage:       "include only aggretable fields",
+				Destination: &conf.Aggretable,
+				Aliases:     []string{"a"},
+			},
+			&cli.BoolFlag{
+				Name:        "searchable",
+				Usage:       "include only searchable fields",
+				Destination: &conf.Searchable,
+				Aliases:     []string{"s"},
+			},
+			&cli.StringSliceFlag{
+				Name:        "type",
+				Usage:       "show only fields of this type",
+				Destination: &conf.Filter,
+				Aliases:     []string{"t"},
+			},
+		},
+
+		ShellComplete: func(ctx context.Context, cmd *cli.Command) {
+			completeIndex(cmd)
+		},
+
+		Action: func(ctx context.Context, cmd *cli.Command) error {
+			index := cmd.Args().Get(0)
+			if index == "" {
+				return errors.New("no index specified")
+			}
+
+			return es.IndexFields(conf, index)
+		},
+	}
+}

cmd/index_alias.go

@@ -36,6 +36,7 @@ func IndexAlias(conf *cfg.Config) *cli.Command {
 			IndexAliasCreate(conf),
 			IndexAliasList(conf),
 			IndexAliasDelete(conf),
+			// FIXME: implement IndexAliasShow + IndexAliasAdd
 			//IndexAliasShow(conf),
 			//IndexAliasAdd(conf), // see https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-update-aliases
 		},
@@ -49,6 +50,10 @@ func IndexAliasCreate(conf *cfg.Config) *cli.Command {
 		Usage:     "create an index alias",
 		UsageText: "create <index> <alias>",
 
+		ShellComplete: func(ctx context.Context, cmd *cli.Command) {
+			completeIndex(cmd)
+		},
+
 		Action: func(ctx context.Context, cmd *cli.Command) error {
 			index := cmd.Args().Get(0)
 			alias := cmd.Args().Get(1)
@@ -69,6 +74,10 @@ func IndexAliasDelete(conf *cfg.Config) *cli.Command {
 		Usage:     "delete an index alias",
 		UsageText: "delete <index> <alias>",
 
+		ShellComplete: func(ctx context.Context, cmd *cli.Command) {
+			completeIndex(cmd)
+		},
+
 		Action: func(ctx context.Context, cmd *cli.Command) error {
 			index := cmd.Args().Get(0)
 			alias := cmd.Args().Get(1)

cmd/node.go

@@ -58,6 +58,7 @@ func NodeShow(conf *cfg.Config) *cli.Command {
 		UsageText: "show [options] <node>",
 
 		Action: func(ctx context.Context, cmd *cli.Command) error {
+			// FIXME: implement es.NodeShow()
 			// return es.NodeShow(conf, cmd.Args().Get(0))
 			return nil
 		},

cmd/roles.go

@@ -0,0 +1,141 @@
+/*
+Copyright © 2026 Thomas von Dein
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+package cmd
+
+import (
+	"context"
+	"errors"
+
+	"codeberg.org/scip/esctl/pkg/cfg"
+	"codeberg.org/scip/esctl/pkg/es"
+
+	"github.com/urfave/cli/v3"
+)
+
+func Roles(conf *cfg.Config) *cli.Command {
+	return &cli.Command{
+		Name:  "role",
+		Usage: "manage roles",
+
+		Commands: []*cli.Command{
+			RoleList(conf),
+			RoleShow(conf),
+			RoleDiff(conf),
+		},
+	}
+}
+
+func RoleList(conf *cfg.Config) *cli.Command {
+	return &cli.Command{
+		Name:    "list",
+		Aliases: []string{"ls"},
+		Usage:   "list roles",
+
+		Flags: []cli.Flag{
+			&cli.BoolFlag{
+				Name:        "orphaned",
+				Usage:       "include only orphaned roles",
+				Destination: &conf.Failed,
+				Aliases:     []string{"o"},
+			},
+		},
+
+		Action: func(ctx context.Context, cmd *cli.Command) error {
+			return es.RoleList(conf)
+		},
+	}
+}
+
+func RoleShow(conf *cfg.Config) *cli.Command {
+	return &cli.Command{
+		Name:      "show",
+		Aliases:   []string{"sh"},
+		Usage:     "show details about a role",
+		UsageText: "show [options] <role>",
+
+		ShellComplete: func(ctx context.Context, cmd *cli.Command) {
+			completeRole(cmd)
+		},
+
+		Action: func(ctx context.Context, cmd *cli.Command) error {
+			index := cmd.Args().Get(0)
+			if index == "" {
+				return errors.New("no role specified")
+			}
+
+			return es.RoleShow(conf, cmd.Args().Get(0))
+		},
+	}
+}
+
+func RoleDiff(conf *cfg.Config) *cli.Command {
+	return &cli.Command{
+		Name:  "diff",
+		Usage: "show differences between roles and CSV baseline",
+		UsageText: `diff [options] <file.csv> [<role>]
+CSV format:
+index_name;role;index_privilege;cluster_privilege;ad_group;space;retention;kibana_privilege;field_privilege`,
+
+		MutuallyExclusiveFlags: []cli.MutuallyExclusiveFlags{{
+			Flags: [][]cli.Flag{
+				{
+					&cli.BoolFlag{
+						Name:        "not-deployed",
+						Usage:       "include only not deployed but defined roles",
+						Destination: &conf.NotDeployed,
+						Aliases:     []string{"n"},
+					},
+				},
+				{
+					&cli.BoolFlag{
+						Name:        "undefined",
+						Usage:       "include only deployed but undefined roles",
+						Destination: &conf.Undefined,
+						Aliases:     []string{"u"},
+					},
+				},
+				{
+					&cli.BoolFlag{
+						Name:        "diff",
+						Usage:       "include only differing roles",
+						Destination: &conf.Diff,
+						Aliases:     []string{"D"},
+					},
+				},
+			}},
+		},
+
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:        "separator",
+				Usage:       "CSV field separator",
+				Destination: &conf.Separator,
+				Aliases:     []string{"s"},
+				Value:       ",",
+			},
+		},
+
+		Action: func(ctx context.Context, cmd *cli.Command) error {
+			csvfile := cmd.Args().Get(0)
+			if csvfile == "" {
+				return errors.New("no CSV file specified")
+			}
+
+			return es.RoleDiff(conf, cmd.Args().Get(0), cmd.Args().Get(1))
+		},
+	}
+}

cmd/root.go

@@ -22,6 +22,7 @@ import (
 	"os"
 
 	"codeberg.org/scip/esctl/pkg/cfg"
+	"codeberg.org/scip/esctl/pkg/es"
 	"codeberg.org/scip/esctl/pkg/log"
 
 	"github.com/urfave/cli/v3"
@@ -54,6 +55,12 @@ func Main() int {
 				Sources:     cli.EnvVars("ES_DEBUG"),
 				Destination: &conf.Debug,
 			},
+			&cli.BoolFlag{
+				Name:        "debug-http",
+				Value:       false,
+				Usage:       "enable HTTP debugging",
+				Destination: &conf.DebugHTTP,
+			},
 			&cli.StringFlag{
 				Name:        "config",
 				Aliases:     []string{"c"},
@@ -89,6 +96,9 @@ func Main() int {
 			Doc(conf),
 			Repl(conf),
 			Version(conf),
+			Debug(conf),
+			Roles(conf),
+			HelpJsonPath(conf),
 		},
 
 		Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) {
@@ -110,6 +120,45 @@ func Main() int {
 	return Finish(cmd.Run(context.Background(), os.Args))
 }
 
+func HelpJsonPath(conf *cfg.Config) *cli.Command {
+	msg := `jsonPath usage:
+
+name.last          >> "Anderson"
+age                >> 37
+children           >> ["Sara","Alex","Jack"]
+children.#         >> 3
+children.1         >> "Alex"
+child*.2           >> "Jack"
+c?ildren.0         >> "Sara"
+fav\.movie         >> "Deer Hunter"
+friends.#.first    >> ["Dale","Roger","Jane"]
+friends.1.last     >> "Craig"
+
+You can also  query an array for  the first match by  using #(...), or
+find all matches  with #(...)#. Queries support the ==,  !=, <, <=, >,
+>= comparison operators  and the simple pattern matching  % (like) and
+!% (not like) operators. Eg:
+
+friends.#(last=="Murphy").first    >> "Dale"
+friends.#(last=="Murphy")#.first   >> ["Dale","Jane"]
+friends.#(age>45)#.last            >> ["Craig","Murphy"]
+friends.#(first%"D*").last         >> "Murphy"
+friends.#(first!%"D*").last        >> "Craig"
+friends.#(nets.#(=="fb"))#.first   >> ["Dale","Roger"]
+
+Documentation: https://github.com/tidwall/gjson/blob/master/SYNTAX.md`
+
+	return &cli.Command{
+		Name:  "help-jsonpath",
+		Usage: "show jsonpath help",
+
+		Action: func(ctx context.Context, cmd *cli.Command) error {
+			_, err := fmt.Println(msg)
+			return err
+		},
+	}
+}
+
 func Version(conf *cfg.Config) *cli.Command {
 	return &cli.Command{
 		Name:  "version",
@@ -122,3 +171,24 @@ func Version(conf *cfg.Config) *cli.Command {
 		},
 	}
 }
+
+func Debug(conf *cfg.Config) *cli.Command {
+	return &cli.Command{
+		Name:  "debug",
+		Usage: "developer only",
+
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:        "index",
+				Usage:       "index to search within",
+				Sources:     cli.EnvVars("ES_INDEX"),
+				Destination: &conf.Index,
+				Aliases:     []string{"i"},
+			},
+		},
+
+		Action: func(ctx context.Context, cmd *cli.Command) error {
+			return es.Debug(conf)
+		},
+	}
+}

cmd/search.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"context"
-	"fmt"
 
 	"codeberg.org/scip/esctl/pkg/cfg"
 	"codeberg.org/scip/esctl/pkg/es"
@@ -26,22 +25,33 @@ import (
 	"github.com/urfave/cli/v3"
 )
 
+const SearchUsage = `<sep> might be one of:
+ =: Must match
+!=: Must not match
+
+You can omit a field spec and thereby search across all fields.
+
+By default all queries contribute to matches (logical AND), use
+-O to apply a logical OR operator.
+
+You can also search multiple fields by separating them with comma, eg:
+user,group=root
+
+Use filters to further restrict results, they must match literally.
+
+For datetime range format refer to:
+https://www.elastic.co/docs/reference/elasticsearch/rest-apis/common-options#date-math
+
+For timestamp formats refer to:
+https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/mapping-date-format
+`
+
 func Search(conf *cfg.Config) *cli.Command {
 	return &cli.Command{
 		Name:      "search",
 		Aliases:   []string{"/"},
 		Usage:     "search within an index",
-		UsageText: `search [options] [<[field<sep>]pattern> ...]
+		UsageText: "search [options] [<[field<sep>]pattern> ...]\n" + SearchUsage,
-
-<sep> might be one of:
- =: Must match
-!=: Must not match
- ?: Should match
-
-You can omit a field spec and thereby search across all fields.
-
-You can also search multiple fields by separating them with comma, eg:
-user,group=root`,
 
 		Flags: []cli.Flag{
 			&cli.StringFlag{
@@ -53,17 +63,17 @@ user,group=root`,
 			},
 			&cli.IntFlag{
 				Name:        "from",
-				Usage:       "show results FROM (default 0)",
+				Usage:       "show results starting at <from>",
 				Destination: &conf.From,
 				Value:       0,
 				Aliases:     []string{"f"},
 			},
 			&cli.IntFlag{
-				Name:        "to",
+				Name:        "len",
-				Usage:       "show results to (default 10)",
+				Usage:       "number of results to show (-1: all[max:10k], caution: might be slow)",
 				Destination: &conf.To,
-				Value:       10,
+				Value:       20,
-				Aliases:     []string{"t"},
+				Aliases:     []string{"l"},
 			},
 			&cli.StringSliceFlag{
 				Name:        "filter",
@@ -77,53 +87,65 @@ user,group=root`,
 				Destination: &conf.Path,
 				Aliases:     []string{"p"},
 			},
+			&cli.StringFlag{
+				Name:        "timerange",
+				Usage:       "field:<date> to <date> (e.g. @timestamp:2026-05-05 to 2026-05-15)",
+				Destination: &conf.Range,
+				Aliases:     []string{"r"},
+			},
+			&cli.StringFlag{
+				Name:        "timestamp-format",
+				Usage:       "a valid ES builtin timestamp or custom format",
+				Destination: &conf.TimestampFormat,
+				Value:       "strict_date_hour_minute",
+			},
+			&cli.StringFlag{
+				Name:        "sort-by",
+				Usage:       "sort by a field",
+				Destination: &conf.SortBy,
+				Value:       "@timestamp",
+				Aliases:     []string{"k"},
+			},
 			&cli.BoolFlag{
-				Name:        "help-jsonpath",
+				Name:        "ascending",
-				Usage:       "show jsonPath help",
+				Usage:       "sort in ascending order (default: descending)",
-				Destination: &conf.Subhelp,
+				Destination: &conf.Ascending,
-				Aliases:     []string{"H"},
+				Aliases:     []string{"a"},
+			},
+			&cli.BoolFlag{
+				Name:        "tail",
+				Usage:       "follow search live, like tail -f",
+				Destination: &conf.Tail,
+				Aliases:     []string{"T"},
+			},
+			&cli.BoolFlag{
+				Name:        "or",
+				Usage:       "logical operator (default: and)",
+				Destination: &conf.Or,
+				Aliases:     []string{"O"},
+			},
+			&cli.BoolFlag{
+				Name:        "validate",
+				Usage:       "validate search query",
+				Destination: &conf.Validate,
+				Aliases:     []string{"v"},
+			},
+			&cli.BoolFlag{
+				Name:        "explain",
+				Usage:       "explain search query",
+				Destination: &conf.Explain,
+				Aliases:     []string{"e"},
 			},
 		},
 
 		Action: func(ctx context.Context, cmd *cli.Command) error {
-			if conf.Subhelp {
-				return showJsonPathHelp()
-			}
-
 			args := cmd.Args()
 
+			if conf.To == -1 {
+				conf.To = 10000
+			}
+
 			return es.Search(conf, args.Slice())
 		},
 	}
 }
-
-func showJsonPathHelp() error {
-	_, err := fmt.Println(`jsonPath usage:
-
-name.last          >> "Anderson"
-age                >> 37
-children           >> ["Sara","Alex","Jack"]
-children.#         >> 3
-children.1         >> "Alex"
-child*.2           >> "Jack"
-c?ildren.0         >> "Sara"
-fav\.movie         >> "Deer Hunter"
-friends.#.first    >> ["Dale","Roger","Jane"]
-friends.1.last     >> "Craig"
-
-You can also  query an array for  the first match by  using #(...), or
-find all matches  with #(...)#. Queries support the ==,  !=, <, <=, >,
->= comparison operators  and the simple pattern matching  % (like) and
-!% (not like) operators. Eg:
-
-friends.#(last=="Murphy").first    >> "Dale"
-friends.#(last=="Murphy")#.first   >> ["Dale","Jane"]
-friends.#(age>45)#.last            >> ["Craig","Murphy"]
-friends.#(first%"D*").last         >> "Murphy"
-friends.#(first!%"D*").last        >> "Craig"
-friends.#(nets.#(=="fb"))#.first   >> ["Dale","Roger"]
-
-Documentation: https://github.com/tidwall/gjson/blob/master/SYNTAX.md`)
-
-	return err
-}

go.mod

@@ -26,7 +26,7 @@ require (
 	github.com/mattn/go-isatty v0.0.22
 	github.com/olekukonko/tablewriter v1.1.4
 	github.com/tlinden/yadu v0.1.3
-	github.com/urfave/cli/v3 v3.9.0
+	github.com/urfave/cli/v3 v3.9.1-0.20260524212652-be8b79d0c8de
 	gopkg.in/yaml.v3 v3.0.1
 )
 

go.sum

@@ -70,6 +70,8 @@ github.com/urfave/cli/v3 v3.8.0 h1:XqKPrm0q4P0q5JpoclYoCAv0/MIvH/jZ2umzuf8pNTI=
 github.com/urfave/cli/v3 v3.8.0/go.mod h1:ysVLtOEmg2tOy6PknnYVhDoouyC/6N42TMeoMzskhso=
 github.com/urfave/cli/v3 v3.9.0 h1:AV9lIiPv3ukYnxunaCUsHnEozptYmDN2F0+yWqLMn/c=
 github.com/urfave/cli/v3 v3.9.0/go.mod h1:ysVLtOEmg2tOy6PknnYVhDoouyC/6N42TMeoMzskhso=
+github.com/urfave/cli/v3 v3.9.1-0.20260524212652-be8b79d0c8de h1:ESKPiS7inVoBnv4FmgGNZdjWBI/wmvaragoyD3D9nM4=
+github.com/urfave/cli/v3 v3.9.1-0.20260524212652-be8b79d0c8de/go.mod h1:ysVLtOEmg2tOy6PknnYVhDoouyC/6N42TMeoMzskhso=
 go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
 go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
 go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ=

pkg/cfg/config.go

@@ -17,10 +17,13 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
 package cfg
 
 import (
+	"bytes"
 	"context"
 	"crypto/tls"
+	"encoding/json"
 	"errors"
 	"fmt"
+	"log/slog"
 	"net/http"
 	"os"
 
@@ -31,7 +34,7 @@ import (
 )
 
 const (
-	Version string = `v0.0.13`
+	Version string = `v0.0.18`
 )
 
 var (
@@ -56,15 +59,30 @@ type Config struct {
 	Shards, Replicas               int      // index create+allocation: -s -r
 	Wait                           bool     // index create: -w
 	Primary                        bool     // index allocation: -p
+	Searchable                     bool     // index fields: -s
+	Aggretable                     bool     // index fields: -a
 	From, To, MaxItems             int      // search: flags
 	Filter                         []string // search: -F
 	Path                           string   // search+doc sh: -p
 	Subhelp                        bool     // search+doc sh: -H
+	Tail                           bool     // search: -f [tail]
+	Or                             bool     // search: -O
+	Range                          string   // search: -r
+	TimestampFormat                string   // search: --timestamp-format
+	Explain                        bool     // search: -e
+	Validate                       bool     // search: --validate
+	SortBy                         string   // sort: -k
+	Ascending                      bool     // sort: -a
 	Exclude                        string   // cluster compare: -e (regexp)
 	All, Verbose                   bool     // cluster status: -a -v
 	Persistent, Transient, Default bool     // -p -t -D cluster settings set
 	Force                          bool     // ccr follower renew: -f
 	HaveJQ                         bool     // determined at runtime by ourselfes
+	DebugHTTP                      bool     // root: --debug-http
+	Separator                      string   // role diff: -s
+	NotDeployed                    bool     // role diff: -n
+	Undefined                      bool     // role diff: -u
+	Diff                           bool     // role diff: -D
 }
 
 func NewConfig() *Config {
@@ -200,18 +218,26 @@ func (conf *Config) LoadConfig() error {
 	return nil
 }
 
+func (conf *Config) getTransport() elastictransport.Option {
+	transport := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+
+	if conf.DebugHTTP {
+		return elastictransport.WithTransport(
+			&DebugTransport{Transport: transport},
+		)
+	}
+
+	return elastictransport.WithTransport(transport)
+}
+
 func (conf *Config) SetupES() error {
 	for _, cluster := range conf.Clusters {
 		es, err := elasticsearch.NewTyped(
 			elasticsearch.WithAddresses(cluster.Uri),
 			elasticsearch.WithBasicAuth(cluster.User, cluster.Pass),
-			elasticsearch.WithTransportOptions(
+			elasticsearch.WithTransportOptions(conf.getTransport()),
-				elastictransport.WithTransport(
-					&http.Transport{
-						TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-					},
-				),
-			),
 		)
 
 		if err != nil {
@@ -224,6 +250,39 @@ func (conf *Config) SetupES() error {
 	return nil
 }
 
+// used to print uri, path and body of a request made by the go-client
+type DebugTransport struct {
+	Transport http.RoundTripper
+}
+
+func (t *DebugTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	content := ""
+	contentline := ""
+
+	if req.ContentLength > 0 {
+		buf := new(bytes.Buffer)
+		body, _ := req.GetBody()
+
+		_, err := buf.ReadFrom(body)
+		if err != nil {
+			return nil, err
+		}
+
+		var pretty bytes.Buffer
+		err = json.Indent(&pretty, buf.Bytes(), "", "\t")
+		if err != nil {
+			return nil, fmt.Errorf("json parse error: %s", err)
+		}
+
+		content = pretty.String()
+		contentline = buf.String()
+	}
+
+	slog.Info("req", "host", req.URL.Host, "uri", req.URL.Path, "body", content, "bodyline", contentline)
+
+	return t.Transport.RoundTrip(req)
+}
+
 func fileExists(filename string) bool {
 	info, err := os.Stat(filename)
 

pkg/es/ccr.go

@@ -62,7 +62,7 @@ func CcrStatus(conf *cfg.Config, leader, follower string) error {
 
 		res, err := cat.Do(context.Background())
 		if err != nil {
-			return fmt.Errorf("failed to get indicies on %s: %s", alias, err)
+			return fmt.Errorf("failed to get indicies on %s: %s", alias, esErrorString(err))
 		}
 
 		indices[alias] = make(map[string]*types.IndicesRecord, len(res))
@@ -93,7 +93,7 @@ func CcrRemoteInfo(conf *cfg.Config, index string) error {
 		Header("accept", "application/json").
 		Do(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to retrieve follower info: %s", err)
+		return fmt.Errorf("failed to retrieve follower info: %s", esErrorString(err))
 	}
 
 	slog.Debug("ccr remote info", "info", res)

pkg/es/ccr_follower.go

@@ -31,7 +31,7 @@ func getRemoteName(conf *cfg.Config) (string, error) {
 		Header("accept", "application/json").
 		Do(context.Background())
 	if err != nil {
-		return "", fmt.Errorf("failed to retrieve follower info: %s", err)
+		return "", fmt.Errorf("failed to retrieve follower info: %s", esErrorString(err))
 	}
 
 	remote := ""
@@ -41,7 +41,7 @@ func getRemoteName(conf *cfg.Config) (string, error) {
 	}
 
 	if remote == "" {
-		return "", fmt.Errorf("cluster doesn't have a follower: %s", err)
+		return "", fmt.Errorf("cluster doesn't have a follower")
 	}
 
 	return remote, nil
@@ -96,7 +96,7 @@ func CcrFollowerResume(conf *cfg.Config, index string) error {
 	_, err := create.Do(context.Background())
 
 	if err != nil {
-		return fmt.Errorf("failed to resume ccr following: %s", err)
+		return fmt.Errorf("failed to resume ccr following: %s", esErrorString(err))
 	}
 
 	return nil
@@ -110,7 +110,7 @@ func CcrFollowerPause(conf *cfg.Config, index string) error {
 	_, err := create.Do(context.Background())
 
 	if err != nil {
-		return fmt.Errorf("failed to pause ccr following: %s", err)
+		return fmt.Errorf("failed to pause ccr following: %s", esErrorString(err))
 	}
 
 	return nil
@@ -124,7 +124,7 @@ func CcrFollowerUnfollow(conf *cfg.Config, index string) error {
 	_, err := create.Do(context.Background())
 
 	if err != nil {
-		return fmt.Errorf("failed to unfollow index: %s", err)
+		return fmt.Errorf("failed to unfollow index: %s", esErrorString(err))
 	}
 
 	return nil
@@ -149,7 +149,7 @@ func CcrFollowerAdd(conf *cfg.Config, index string) error {
 	_, err = create.Do(context.Background())
 
 	if err != nil {
-		return fmt.Errorf("failed to create follower index: %s", err)
+		return fmt.Errorf("failed to create follower index: %s", esErrorString(err))
 	}
 
 	return nil
@@ -161,7 +161,7 @@ func CcrFollowerShow(conf *cfg.Config, index string) error {
 		Header("accept", "application/json").
 		Do(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to retrieve follower index info: %s", err)
+		return fmt.Errorf("failed to retrieve follower index info: %s", esErrorString(err))
 	}
 
 	slog.Debug("ES result", "follower stats", res.Indices)

pkg/es/cluster.go

@@ -163,11 +163,12 @@ func ClusterStatus(conf *cfg.Config) error {
 			)
 		}
 
-		table := printer.NewTable(conf, 2, 5)
+		table := printer.NewTable(conf, 2, 7)
 		table.Addheaders(cluster, "status")
 
 		table.Entries = [][]string{
-			{"Cluster Name", printer.Colorize(conf, clusterhealth.Status.Name, clusterhealth.ClusterName)},
+			{"Cluster Name", clusterhealth.ClusterName},
+			{"ES Status", printer.Colorize(conf, clusterhealth.Status.Name, clusterhealth.Status.Name)},
 			{"ES Version", info.Version.Int},
 			{"Active Shards", fmt.Sprintf("%d", clusterhealth.ActiveShards)},
 			{"Active Primary Shards", fmt.Sprintf("%d", clusterhealth.ActivePrimaryShards)},

pkg/es/cluster_settings.go

@@ -63,7 +63,7 @@ func ClusterSettingsList(conf *cfg.Config) error {
 		Header("accept", "application/json").
 		Do(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to get cluster settings: %s", err)
+		return fmt.Errorf("failed to get cluster settings: %s", esErrorString(err))
 	}
 
 	table := printer.NewTable(conf, 2, 0)
@@ -135,7 +135,7 @@ func ClusterSettingsSet(conf *cfg.Config, args cli.Args) error {
 		Do(context.Background())
 
 	if err != nil {
-		return fmt.Errorf("failed to set settings: %s", err)
+		return fmt.Errorf("failed to set settings: %s", esErrorString(err))
 	}
 
 	return nil
@@ -154,7 +154,7 @@ func ClusterSettingsSetSingle(conf *cfg.Config, setting, value string) error {
 		Do(context.Background())
 
 	if err != nil {
-		return fmt.Errorf("failed to set %s: %s", setting, err)
+		return fmt.Errorf("failed to set %s: %s", setting, esErrorString(err))
 	}
 
 	return nil

pkg/es/cluster_util.go

@@ -41,7 +41,7 @@ func checkClusterIsLeader(conf *cfg.Config, leader string) bool {
 		Header("accept", "application/json").
 		Do(context.Background())
 	if err != nil {
-		fmt.Printf("failed to get ccr stats from %s: %s", leader, err)
+		fmt.Printf("failed to get ccr stats from %s: %s", leader, esErrorString(err))
 		return false
 	}
 
@@ -68,21 +68,22 @@ func checkClusterStatus(conf *cfg.Config, leader, follower string) bool {
 			Header("accept", "application/json").
 			Do(context.Background())
 		if err != nil {
-			fmt.Printf("failed to get health from %s: %s", cluster, err)
+			fmt.Printf("failed to get health from %s: %s", cluster, esErrorString(err))
 			return false
 		}
 
 		status[cluster] = st
 	}
 
-	table := printer.NewTable(conf, 3, 5)
+	table := printer.NewTable(conf, 3, 6)
 
 	table.Addheaders("setting", "leader:"+leader, "follower:"+follower)
 
 	table.Entries = [][]string{
-		{"Cluster Name",
+		{"Cluster Name", status[leader].ClusterName, status[follower].ClusterName},
-			printer.Colorize(conf, status[leader].Status.Name, status[leader].ClusterName),
+		{"Cluster Status",
-			printer.Colorize(conf, status[follower].Status.Name, status[follower].ClusterName),
+			printer.Colorize(conf, status[leader].Status.Name, status[leader].Status.Name),
+			printer.Colorize(conf, status[follower].Status.Name, status[follower].Status.Name),
 		},
 		{"Active Shards",
 			fmt.Sprintf("%d", status[leader].ActiveShards),
@@ -107,6 +108,8 @@ func checkClusterStatus(conf *cfg.Config, leader, follower string) bool {
 		return false
 	}
 
+	fmt.Println()
+
 	if status[leader].Status.Name == "green" && status[follower].Status.Name == "green" {
 		return true
 	}
@@ -125,7 +128,7 @@ func findIlmErrors(conf *cfg.Config, leader, follower string) bool {
 			Header("accept", "application/json").
 			Do(context.Background())
 		if err != nil {
-			fmt.Printf("failed to get ilm status from %s: %s", cluster, err)
+			fmt.Printf("failed to get ilm status from %s: %s", cluster, esErrorString(err))
 			return false
 		}
 

pkg/es/doc.go

@@ -21,9 +21,12 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"time"
+	"math/rand/v2"
+	"strings"
 
 	"codeberg.org/scip/esctl/pkg/cfg"
+	"github.com/elastic/go-elasticsearch/v9/typedapi/core/deletebyquery"
+	"github.com/elastic/go-elasticsearch/v9/typedapi/esdsl"
 	"github.com/tidwall/gjson"
 )
 
@@ -42,7 +45,7 @@ func DocAdd(conf *cfg.Config, jsondoc string) error {
 		return fmt.Errorf("supplied document was not valid JSON: %s", err)
 	}
 
-	now := fmt.Sprintf("%d", time.Now().Unix())
+	now := fmt.Sprintf("%d", rand.Int64())
 
 	res, err := conf.DefaultCluster.ES.Create(conf.Index, now).
 		Document(data).
@@ -50,7 +53,7 @@ func DocAdd(conf *cfg.Config, jsondoc string) error {
 		Header("accept", "application/json").
 		Do(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to create new doc in index %s: %s", conf.Index, err)
+		return fmt.Errorf("failed to create new doc in index %s: %s", conf.Index, esErrorString(err))
 	}
 
 	fmt.Println(res.Id_)
@@ -64,7 +67,7 @@ func DocShow(conf *cfg.Config, id string) error {
 		Header("accept", "application/json").
 		Do(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to retrieve doc in index %s: %s", conf.Index, err)
+		return fmt.Errorf("failed to retrieve doc in index %s: %s", conf.Index, esErrorString(err))
 	}
 
 	if !res.Found {
@@ -87,3 +90,43 @@ func DocShow(conf *cfg.Config, id string) error {
 
 	return nil
 }
+
+func DocDelete(conf *cfg.Config, queries []string) error {
+	if len(queries) == 1 && strings.Contains(queries[0], "id=") {
+		id, _ := strings.CutPrefix(queries[0], "id=")
+
+		_, err := conf.DefaultCluster.ES.Delete(conf.Index, id).
+			Header("content-type", "application/json").
+			Header("accept", "application/json").
+			Do(context.Background())
+		if err != nil {
+			return fmt.Errorf("failed to delete doc in index %s: %s", conf.Index, esErrorString(err))
+		}
+
+		return nil
+	}
+
+	req := &deletebyquery.Request{}
+
+	if len(queries) == 0 && conf.All {
+		req.Query = esdsl.NewMatchAllQuery().QueryCaster()
+	} else {
+		queryCaster, err := prepareQuery(conf, queries)
+		if err != nil {
+			return err
+		}
+
+		req.Query = queryCaster
+	}
+
+	_, err := conf.DefaultCluster.ES.DeleteByQuery(conf.Index).
+		Request(req).
+		Header("content-type", "application/json").
+		Header("accept", "application/json").
+		Do(context.Background())
+	if err != nil {
+		return fmt.Errorf("failed to delete docs in index %s: %s", conf.Index, esErrorString(err))
+	}
+
+	return nil
+}

pkg/es/errors.go

@@ -0,0 +1,45 @@
+/*
+Copyright © 2026 Thomas von Dein
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+package es
+
+import (
+	"fmt"
+
+	"github.com/elastic/go-elasticsearch/v9/typedapi/types"
+)
+
+func esErrorString(err error) string {
+	msg := err.Error()
+
+	switch e := err.(type) {
+	case *types.ElasticsearchError:
+		causes := ""
+
+		for _, cause := range e.ErrorCause.RootCause {
+			causes += fmt.Sprintf("%s\n", *cause.Reason)
+		}
+
+		if e.ErrorCause.Reason != nil {
+			msg = *e.ErrorCause.Reason + ": " + causes
+		} else {
+			msg = fmt.Sprintf("http status %d: ", e.Status)
+		}
+
+	}
+
+	return msg
+}

pkg/es/index.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"log/slog"
 	"regexp"
+	"slices"
 	"strconv"
 	"strings"
 	"time"
@@ -40,7 +41,7 @@ func IndexNames(conf *cfg.Config) ([]string, error) {
 		Do(context.Background())
 
 	if err != nil {
-		return nil, fmt.Errorf("failed to get indicies: %s", err)
+		return nil, fmt.Errorf("failed to get indicies: %s", esErrorString(err))
 	}
 
 	indices := make([]string, len(res))
@@ -52,15 +53,24 @@ func IndexNames(conf *cfg.Config) ([]string, error) {
 }
 
 func filterIndices(conf *cfg.Config, list indices.Response) indices.Response {
+	// apply partials filter first
+	selectedlist := indices.Response{}
+	for _, index := range list {
+		if !conf.Partials && strings.HasPrefix(*index.Index, "partial-") {
+			continue
+		}
+		selectedlist = append(selectedlist, index)
+	}
+
 	if len(conf.Filter) == 0 {
-		return list
+		return selectedlist
 	}
 
 	// we support just one filter here, for now
 	filter := *regexp.MustCompile(conf.Filter[0])
 	newlist := indices.Response{}
 
-	for _, index := range list {
+	for _, index := range selectedlist {
 		if filter.MatchString(*index.Index) {
 			newlist = append(newlist, index)
 		}
@@ -81,7 +91,7 @@ func IndexList(conf *cfg.Config) error {
 
 	res, err := cat.Do(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to get indicies: %s", err)
+		return fmt.Errorf("failed to get indicies: %s", esErrorString(err))
 	}
 
 	slog.Debug("ES result", "indicies", res)
@@ -117,21 +127,20 @@ func IndexList(conf *cfg.Config) error {
 	return nil
 }
 
-func IndexShow(conf *cfg.Config, index string) error {
+func IndexShow(conf *cfg.Config, indexpattern string) error {
-	res, err := conf.DefaultCluster.ES.Indices.Get(index).
+	res, err := conf.DefaultCluster.ES.Indices.Get(indexpattern).
 		// we need to add custom request headers, required for older ES instances
 		Header("content-type", "application/json").
 		Header("accept", "application/json").
 		Do(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to get index: %s", err)
+		return fmt.Errorf("failed to get index: %s", esErrorString(err))
 	}
 
-	slog.Debug("ES result", "index", res)
+	for name, index := range res {
-
+		fields := make([]string, len(index.Mappings.Properties))
-	fields := make([]string, len(res[index].Mappings.Properties))
 		idx := 0
-	for field := range res[index].Mappings.Properties {
+		for field := range index.Mappings.Properties {
 			fields[idx] = field
 			idx++
 		}
@@ -139,7 +148,7 @@ func IndexShow(conf *cfg.Config, index string) error {
 		table := printer.NewTable(conf, 2, 5)
 		table.Addheaders("index property", "value")
 
-	ts, err := strconv.ParseInt(res[index].Settings.Index.CreationDate.(string), 10, 64)
+		ts, err := strconv.ParseInt(index.Settings.Index.CreationDate.(string), 10, 64)
 		if err != nil {
 			ts = 0
 		}
@@ -147,11 +156,11 @@ func IndexShow(conf *cfg.Config, index string) error {
 		created := time.Unix(ts/1000, 0)
 
 		table.Entries = [][]string{
-		{"name", index},
+			{"name", name},
-		{"replicas", *res[index].Settings.Index.NumberOfReplicas},
+			{"replicas", *index.Settings.Index.NumberOfReplicas},
-		{"shards", *res[index].Settings.Index.NumberOfShards},
+			{"shards", *index.Settings.Index.NumberOfShards},
 			{"created", created.Format("2006-01-02 15:04:05")},
-		{"uuid", *res[index].Settings.Index.Uuid},
+			{"uuid", *index.Settings.Index.Uuid},
 			{"fields", strings.Join(fields, ",")},
 		}
 
@@ -159,6 +168,9 @@ func IndexShow(conf *cfg.Config, index string) error {
 			return err
 		}
 
+		fmt.Println()
+	}
+
 	return nil
 }
 
@@ -208,7 +220,7 @@ func IndexCreate(conf *cfg.Config, index string, mappings []string) error {
 		Do(context.Background())
 
 	if err != nil {
-		return fmt.Errorf("failed to create index: %s", err)
+		return fmt.Errorf("failed to create index: %s", esErrorString(err))
 	}
 
 	return nil
@@ -220,7 +232,7 @@ func IndexDelete(conf *cfg.Config, index string) error {
 		Header("accept", "application/json").
 		Do(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to delete index: %s", err)
+		return fmt.Errorf("failed to delete index: %s", esErrorString(err))
 	}
 
 	return nil
@@ -234,7 +246,7 @@ func IndexClose(conf *cfg.Config, index string) error {
 	_, err := create.Do(context.Background())
 
 	if err != nil {
-		return fmt.Errorf("failed to close index: %s", err)
+		return fmt.Errorf("failed to close index: %s", esErrorString(err))
 	}
 
 	return nil
@@ -249,7 +261,7 @@ func IndexAllocation(conf *cfg.Config, index string) error {
 		Header("accept", "application/json").
 		Do(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to get index allocation explain: %s", err)
+		return fmt.Errorf("failed to get index allocation explain: %s", esErrorString(err))
 	}
 
 	slog.Debug("ES result", "index", res)
@@ -294,7 +306,53 @@ func IndexModify(conf *cfg.Config, index string) error {
 		Header("accept", "application/json").
 		Do(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to modify index settings: %s", err)
+		return fmt.Errorf("failed to modify index settings: %s", esErrorString(err))
+	}
+
+	return nil
+}
+
+func IndexFields(conf *cfg.Config, index string) error {
+	res, err := conf.DefaultCluster.ES.FieldCaps().
+		Index(index).
+		Fields("*").
+		Header("content-type", "application/json").
+		Header("accept", "application/json").
+		Do(context.Background())
+	if err != nil {
+		return fmt.Errorf("failed to retrieve field capabilties: %s", esErrorString(err))
+	}
+
+	table := printer.NewTable(conf, 5, 0)
+	table.Addheaders("field", "type", "searchable", "aggretable", "metadata")
+
+	idx := 0
+	for name, field := range res.Fields {
+		for fieldtype, caps := range field {
+			// fields only have 1 type, so this one is it
+			switch {
+			case conf.Searchable && !caps.Searchable:
+				continue
+			case conf.Aggretable && !caps.Aggregatable:
+				continue
+			case len(conf.Filter) > 0 && !slices.Contains(conf.Filter, fieldtype):
+				continue
+			}
+
+			table.Entries = append(table.Entries, []string{name, fieldtype,
+				fmt.Sprintf("%t", caps.Searchable),
+				fmt.Sprintf("%t", caps.Aggregatable),
+				fmt.Sprintf("%t", *caps.MetadataField)})
+			break
+		}
+
+		idx++
+	}
+
+	table.Sort()
+
+	if err := table.Print(); err != nil {
+		return err
 	}
 
 	return nil

pkg/es/index_alias.go

@@ -37,7 +37,7 @@ func IndexAliasCreate(conf *cfg.Config, index, alias string) error {
 	slog.Debug("create alias", "result", res)
 
 	if err != nil {
-		return fmt.Errorf("failed to create index alias: %s", err)
+		return fmt.Errorf("failed to create index alias: %s", esErrorString(err))
 	}
 
 	return nil
@@ -58,7 +58,7 @@ func IndexAliasList(conf *cfg.Config) error {
 		Do(context.Background())
 
 	if err != nil {
-		return fmt.Errorf("failed to list index aliases: %s", err)
+		return fmt.Errorf("failed to list index aliases: %s", esErrorString(err))
 	}
 
 	slog.Debug("aliases list", "result", res)
@@ -108,7 +108,7 @@ func IndexAliasDelete(conf *cfg.Config, index, alias string) error {
 	slog.Debug("delete alias", "result", res)
 
 	if err != nil {
-		return fmt.Errorf("failed to delete index alias: %s", err)
+		return fmt.Errorf("failed to delete index alias: %s", esErrorString(err))
 	}
 
 	return nil

pkg/es/node.go

@@ -29,7 +29,7 @@ func NodeList(conf *cfg.Config) error {
 	// get nodes
 	nodes, err := conf.DefaultCluster.ES.Cat.Nodes().Do(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to get nodes: %s", err)
+		return fmt.Errorf("failed to get nodes: %s", esErrorString(err))
 	}
 
 	slog.Debug("ES result", "nodes", nodes)

pkg/es/repl.go

@@ -17,6 +17,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
 package es
 
 import (
+	"bufio"
 	"bytes"
 	"context"
 	"crypto/tls"
@@ -34,19 +35,98 @@ import (
 	"github.com/chzyer/readline"
 )
 
+const intro = `Input format: verb path [data]"
+
+Example:
+
+post /yourindex/_ccr/pause_follow
+put /yourindex/_settings {"number_of_replicas": 1}
+
+You can also put multiline JSON after the path like:
+
+put /yourindex/_settings
+{
+  "number_of_replicas": 1
+}
+
+If you do NOT  supply a JSON in the first line, you  need to hit ENTER
+twice to complete.`
+
+func Repl(conf *cfg.Config) error {
+	verbs := []string{"post", "get", "put", "delete"}
+
+	fmt.Println(intro)
+	fmt.Println()
+
+	reader, err := readline.NewEx(&readline.Config{
+		Prompt:            "> ",
+		HistoryFile:       os.Getenv("HOME") + "/.config/esctl/history",
+		HistoryLimit:      500,
+		InterruptPrompt:   "^C",
+		EOFPrompt:         "exit",
+		HistorySearchFold: true,
+	})
+
+	if err != nil {
+		return fmt.Errorf("failed to initialize readline lib: %s", err)
+	}
+
+	for {
+		text, err := reader.Readline()
+		if err != nil {
+			break
+		}
+
+		text = strings.TrimSpace(text)
+
+		if text == "" {
+			continue
+		}
+
+		parts := strings.SplitN(strings.TrimSpace(text), " ", 3)
+		if len(parts) < 2 {
+			fmt.Println("error: you need to input a verb, uri [and post data]")
+			continue
+		}
+
+		if !slices.Contains(verbs, strings.ToLower(parts[0])) {
+			fmt.Println("error: verb must be one of " + strings.Join(verbs, ","))
+			continue
+		}
+
+		if !strings.HasPrefix(parts[1], "/") {
+			parts[1] = "/" + parts[1]
+		}
+
+		json := ""
+		if len(parts) == 3 {
+			// put /uri {json}
+			json = parts[2]
+		}
+
+		// put /uri/<Ret> [json]
+		data, err := readJSON(json)
+		if err != nil {
+			fmt.Println(err)
+		}
+
+		err = CallAPI(conf, parts[0], parts[1], data)
+		if err != nil {
+			fmt.Printf("failed to call API: %s\n", esErrorString(err))
+		}
+
+		reader.SetPrompt("> ")
+	}
+
+	return nil
+}
+
 func encodeAuth(username, password string) string {
 	return base64.StdEncoding.EncodeToString([]byte(username + ":" + password))
 }
 
-func CallAPI(conf *cfg.Config, input []string) error {
+func CallAPI(conf *cfg.Config, verb, path, data string) error {
-	var data string
+	verb = strings.ToUpper(verb)
-
-	verb := strings.ToUpper(input[0])
-	path := input[1]
-
-	if len(input) == 3 {
-		data = input[2]
-	}
 
 	// we're using port-forwards anyway
 	tr := &http.Transport{
@@ -108,69 +188,37 @@ func prettyfiJson(conf *cfg.Config, raw []byte) error {
 	return nil
 }
 
-func Repl(conf *cfg.Config) error {
+// interactively  read  arbitrary  JSON  data  from  STDIN,  which  is
-	verbs := []string{"post", "get", "put", "delete"}
+// virtually a repl inside the primary repl
+func readJSON(input string) (string, error) {
+	data := ""
 
-	fmt.Println("Input format: verb path [data]")
+	if input != "" {
-	fmt.Println("example: post /yourindex/_ccr/pause_follow")
+		data = input
+	} else {
+		scanner := bufio.NewScanner(os.Stdin)
+		for scanner.Scan() {
+			line := strings.TrimSpace(scanner.Text())
 
-	reader, err := readline.NewEx(&readline.Config{
+			if line == "" {
-		Prompt:            "> ",
-		HistoryFile:       os.Getenv("HOME") + "/.config/esctl/history",
-		HistoryLimit:      500,
-		InterruptPrompt:   "^C",
-		EOFPrompt:         "exit",
-		HistorySearchFold: true,
-	})
-
-	if err != nil {
-		return fmt.Errorf("failed to initialize readline lib: %s", err)
-	}
-
-	for {
-		text, err := reader.Readline()
-		if err != nil {
 				break
 			}
 
-		text = strings.TrimSpace(text)
+			data += line
-
+		}
-		if text == "" {
-			continue
 	}
 
-		parts := strings.SplitN(strings.TrimSpace(text), " ", 3)
+	if data == "" {
-		if len(parts) < 2 {
+		return data, nil
-			fmt.Println("error: you need to input a verb, uri [and post data]")
-			continue
 	}
 
-		if !slices.Contains(verbs, strings.ToLower(parts[0])) {
+	// validate
-			fmt.Println("error: verb must be one of " + strings.Join(verbs, ","))
+	check := map[string]any{}
-			continue
+	err := json.Unmarshal([]byte(data), &check)
-		}
-
-		if !strings.HasPrefix(parts[1], "/") {
-			fmt.Println("error: url path must start with /")
-			continue
-		}
-
-		if len(parts) == 3 {
-			data := map[string]any{}
-			err := json.Unmarshal([]byte(parts[2]), &data)
 	if err != nil {
-				fmt.Printf("error: input data is not proper JSON: %s", err)
+		return "", fmt.Errorf("error: input data is not proper JSON: %s", err)
-				continue
+
-			}
 	}
 
-		err = CallAPI(conf, parts)
+	return data, nil
-		if err != nil {
-			fmt.Printf("failed to call API: %s\n", err)
-		}
-
-		reader.SetPrompt("> ")
-	}
-
-	return nil
 }

pkg/es/role.go

@@ -0,0 +1,245 @@
+/*
+Copyright © 2026 Thomas von Dein
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+package es
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"strings"
+
+	"codeberg.org/scip/esctl/pkg/cfg"
+	"codeberg.org/scip/esctl/pkg/printer"
+	"github.com/elastic/go-elasticsearch/v9/typedapi/types"
+)
+
+func RoleNames(conf *cfg.Config) ([]string, error) {
+	res, err := conf.DefaultCluster.ES.Security.GetRole().
+		Do(context.Background())
+	if err != nil {
+		return nil, fmt.Errorf("failed to get roles: %s", esErrorString(err))
+	}
+
+	roles := make([]string, len(res))
+	idx := 0
+	for name := range res {
+		roles[idx] = name
+		idx++
+	}
+
+	return roles, nil
+}
+
+func RoleList(conf *cfg.Config) error {
+	res, err := conf.DefaultCluster.ES.Security.GetRole().
+		Do(context.Background())
+	if err != nil {
+		return fmt.Errorf("failed to get roles: %s", esErrorString(err))
+	}
+
+	slog.Debug("ES result", "roles", res)
+
+	table := printer.NewTable(conf, 3, len(res))
+	table.Addheaders("role", "index roles", "cluster roles")
+
+	idx := 0
+	for name, role := range res {
+		table.Entries[idx] = []string{
+			name,
+			fmt.Sprintf("%d", len(role.Cluster)),
+			fmt.Sprintf("%d", len(role.Indices)),
+		}
+		idx++
+	}
+
+	table.Sort()
+	return table.Print()
+}
+
+func RoleShow(conf *cfg.Config, rolename string) error {
+	res, err := conf.DefaultCluster.ES.Security.GetRole().
+		Name(rolename).
+		Do(context.Background())
+	if err != nil {
+		return fmt.Errorf("failed to get role: %s", esErrorString(err))
+	}
+
+	slog.Debug("ES result", "role", res)
+
+	role, exists := res[rolename]
+	if !exists {
+		return fmt.Errorf("role %s does not exist", rolename)
+	}
+
+	if len(role.Indices) > 0 {
+		if err := roleIndices(conf, role); err != nil {
+			return err
+		}
+
+		fmt.Println()
+	}
+
+	if len(role.RemoteIndices) > 0 {
+		if err := roleRemoteIndices(conf, role); err != nil {
+			return err
+		}
+
+		fmt.Println()
+	}
+
+	if len(role.Cluster) > 0 {
+		if err := roleClusters(conf, role); err != nil {
+			return err
+		}
+
+		fmt.Println()
+	}
+
+	if len(role.RemoteCluster) > 0 {
+		if err := roleRemoteClusters(conf, role); err != nil {
+			return err
+		}
+
+		fmt.Println()
+	}
+
+	if len(role.Applications) > 0 {
+		if err := roleApplications(conf, role); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func roleRemoteClusters(conf *cfg.Config, role types.Role) error {
+	if len(role.RemoteCluster) == 0 {
+		return nil
+	}
+
+	table := printer.NewTable(conf, 2, len(role.RemoteCluster))
+	table.Addheaders("remote cluster", "privilege")
+	idx := 0
+	for _, priv := range role.RemoteCluster {
+		perms := []string{}
+		for _, perm := range priv.Privileges {
+			perms = append(perms, perm.Name)
+		}
+
+		table.Entries[idx] = []string{
+			strings.Join(priv.Clusters, ","),
+			strings.Join(perms, ","),
+		}
+		idx++
+	}
+
+	table.Sort()
+	return table.Print()
+}
+
+func roleClusters(conf *cfg.Config, role types.Role) error {
+	if len(role.Cluster) == 0 {
+		return nil
+	}
+
+	table := printer.NewTable(conf, 1, len(role.Cluster))
+	table.Addheaders("cluster rights")
+	idx := 0
+	for _, cluster := range role.Cluster {
+		table.Entries[idx] = []string{cluster.Name}
+		idx++
+	}
+
+	table.Sort()
+	return table.Print()
+}
+
+func roleRemoteIndices(conf *cfg.Config, role types.Role) error {
+	if len(role.RemoteIndices) == 0 {
+		return nil
+	}
+
+	table := printer.NewTable(conf, 3, len(role.RemoteIndices))
+	table.Addheaders("remote index names", "index permissions", "allow restricted")
+	idx := 0
+	for _, priv := range role.RemoteIndices {
+		perms := []string{}
+		for _, perm := range priv.Privileges {
+			perms = append(perms, perm.Name)
+		}
+
+		table.Entries[idx] = []string{
+			strings.Join(priv.Names, ", "),
+			strings.Join(perms, ", "),
+			fmt.Sprintf("%t", *priv.AllowRestrictedIndices),
+		}
+
+		idx++
+	}
+
+	table.Sort()
+	return table.Print()
+}
+
+func roleIndices(conf *cfg.Config, role types.Role) error {
+	if len(role.Indices) == 0 {
+		return nil
+	}
+
+	table := printer.NewTable(conf, 3, len(role.Indices))
+	table.Addheaders("index names", "index permissions", "allow restricted")
+	idx := 0
+	for _, priv := range role.Indices {
+		perms := []string{}
+		for _, perm := range priv.Privileges {
+			perms = append(perms, perm.Name)
+		}
+
+		table.Entries[idx] = []string{
+			strings.Join(priv.Names, ", "),
+			strings.Join(perms, ", "),
+			fmt.Sprintf("%t", *priv.AllowRestrictedIndices),
+		}
+
+		idx++
+	}
+
+	table.Sort()
+	return table.Print()
+}
+
+func roleApplications(conf *cfg.Config, role types.Role) error {
+	if len(role.Applications) == 0 {
+		return nil
+	}
+
+	table := printer.NewTable(conf, 3, len(role.Applications))
+	table.Addheaders("application", "privileges", "resources")
+	idx := 0
+	for _, priv := range role.Applications {
+		table.Entries[idx] = []string{
+			priv.Application,
+			strings.Join(priv.Privileges, ", "),
+			strings.Join(priv.Resources, ", "),
+		}
+
+		idx++
+	}
+
+	table.Sort()
+	return table.Print()
+}

pkg/es/role_diff.go

@@ -0,0 +1,354 @@
+/*
+Copyright © 2026 Thomas von Dein
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+package es
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"encoding/csv"
+	"fmt"
+	"log"
+	"log/slog"
+	"os"
+	"slices"
+	"strings"
+
+	"codeberg.org/scip/esctl/pkg/cfg"
+	"codeberg.org/scip/esctl/pkg/printer"
+	"github.com/alecthomas/repr"
+	"github.com/elastic/go-elasticsearch/v9/typedapi/security/getrole"
+)
+
+// use static csv  record positions as const vars so  we can modify it
+// if the csv format ever changes
+const (
+	Rindexname = iota
+	Rrole
+	Rindexprivilege
+	Rclusterprivilege
+	Radgroup
+	Rspace
+	Rretention
+	Rkibanaprivilege
+	Rfieldprivilege
+)
+
+type Record struct {
+	// filled from CSV input
+	index_name        string
+	role              string
+	index_privilege   string
+	cluster_privilege []string
+	ad_group          []string
+	space             string
+	retention         string
+	kibana_privilege  string
+	field_privilege   string
+
+	// set by ourselfes
+	defined bool
+}
+
+type Register struct {
+	name              string
+	deployed, defined bool
+}
+
+// generic variant,  we do not account  for multiple rows of  the same
+// role,  in  such  cases  an   entry  will  simply  overwritten.  Use
+// getCsvRecord() for a single role.
+func getCsvRecords(conf *cfg.Config, csvfile string) (map[string]Record, error) {
+	data, err := os.ReadFile(csvfile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read CSV file: %s", err)
+	}
+
+	csvreader := csv.NewReader(bytes.NewReader(data))
+	csvreader.Comma = rune(conf.Separator[0])
+	csvreader.Comment = '#'
+	csvreader.TrimLeadingSpace = true
+
+	rows, err := csvreader.ReadAll()
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse CSV: %s", err)
+	}
+
+	records := make(map[string]Record, len(rows)-1)
+
+	for idx, row := range rows {
+		if idx == 0 {
+			continue // header
+		}
+
+		records[row[1]] = Record{
+			index_name:        row[Rindexname],
+			role:              row[Rrole],
+			index_privilege:   row[Rindexprivilege],
+			cluster_privilege: []string{row[Rindexprivilege]},
+			ad_group:          []string{row[Rclusterprivilege]},
+			space:             row[Rspace],
+			retention:         row[Rretention],
+			kibana_privilege:  row[Rkibanaprivilege],
+			field_privilege:   row[Rfieldprivilege],
+			defined:           true,
+		}
+	}
+
+	return records, nil
+}
+
+// same thing  as above but  for one specific role.  supports multiple
+// rows  of  the same  record  with  different  values which  will  be
+// combined.
+func getCsvRecord(conf *cfg.Config, csvfile, rolename string) (*Record, error) {
+	fd, err := os.Open(csvfile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open CSV file: %s", err)
+	}
+	defer func() {
+		if err := fd.Close(); err != nil {
+			log.Fatalf("failed to close file: %s", err)
+		}
+	}()
+
+	scanner := bufio.NewScanner(fd)
+	record := Record{role: rolename}
+
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if strings.HasPrefix(line, "#") || line == "" {
+			continue
+		}
+
+		row := strings.Split(line, conf.Separator)
+
+		if row[Rrole] == rolename {
+			record.index_name = row[Rindexname]
+			record.index_privilege = row[Rindexprivilege]
+			record.cluster_privilege = strings.Split(row[Rclusterprivilege], ",")
+			record.ad_group = append(record.ad_group, row[Radgroup])
+			record.space = row[Rspace]
+			record.retention = row[Rretention]
+			record.kibana_privilege = row[Rkibanaprivilege]
+			record.field_privilege = row[Rfieldprivilege]
+			record.defined = true
+		}
+	}
+
+	return &record, nil
+}
+
+func diffRoles(conf *cfg.Config, records map[string]Record, res getrole.Response) []Register {
+	rows := []Register{}
+	filtered := []Register{}
+	deployed := map[string]int{}
+
+	// iterate over deployed roles
+	for name := range res {
+		reg := Register{name: name}
+
+		_, defined := records[name]
+		if defined {
+			reg.deployed = true
+			reg.defined = true
+		} else {
+			reg.deployed = true
+			reg.defined = false
+		}
+
+		deployed[name] = 1
+
+		rows = append(rows, reg)
+	}
+
+	// iterate over records from CSV and register only those which are not deployed
+	for name := range records {
+		reg := Register{name: name, defined: true}
+		_, deployed := deployed[name]
+		if !deployed {
+			rows = append(rows, reg)
+		}
+	}
+
+	for _, reg := range rows {
+		if (conf.NotDeployed && !reg.deployed) ||
+			(conf.Undefined && !reg.defined) ||
+			(conf.Diff && reg.deployed != reg.defined) ||
+			(!conf.Undefined && !conf.NotDeployed && !conf.Diff) {
+			filtered = append(filtered, reg)
+		}
+	}
+
+	return filtered
+}
+
+func RoleDiff(conf *cfg.Config, csvfile, role string) error {
+	records, err := getCsvRecords(conf, csvfile)
+	if err != nil {
+		return err
+	}
+
+	if role != "" {
+		return RoleDiffSingle(conf, csvfile, role)
+	}
+
+	res, err := conf.DefaultCluster.ES.Security.GetRole().
+		Do(context.Background())
+	if err != nil {
+		return fmt.Errorf("failed to get roles: %s", esErrorString(err))
+	}
+
+	rows := diffRoles(conf, records, res)
+
+	table := printer.NewTable(conf, 3, len(rows))
+	table.Addheaders("role", "is deployed", "is defined")
+
+	for idx, row := range rows {
+		deployed := printer.Colorize(conf, "green", "deployed")
+		if !row.deployed {
+			deployed = printer.Colorize(conf, "red", "not deployed")
+		}
+
+		defined := printer.Colorize(conf, "green", "defined")
+		if !row.defined {
+			defined = printer.Colorize(conf, "red", "undefined")
+		}
+
+		table.Entries[idx] = []string{
+			row.name,
+			deployed,
+			defined,
+		}
+	}
+
+	table.Sort()
+
+	return table.Print()
+}
+
+func getRoleMappingGroups(conf *cfg.Config, rolename string) ([]string, error) {
+	mappings, err := conf.DefaultCluster.ES.Security.GetRoleMapping().
+		Do(context.Background())
+	if err != nil {
+		return nil, fmt.Errorf("failed to get role mappings: %s", esErrorString(err))
+	}
+
+	groups := []string{}
+	for _, mapping := range mappings {
+		if slices.Contains(mapping.Roles, rolename) {
+			for _, rule := range mapping.Rules.Any {
+				for _, group := range rule.Field["groups"] {
+					groups = append(groups, group.(string))
+				}
+			}
+		}
+	}
+
+	return groups, nil
+}
+
+func compareSlices(name string, a, b []string) {
+	slices.Sort(a)
+	slices.Sort(b)
+
+	if slices.Compare(a, b) != 0 {
+		fmt.Printf("%s differs:\ndeployed: %s\n     csv: %s\n",
+			name, strings.Join(a, ","), strings.Join(b, ","))
+	} else {
+		fmt.Printf("deployed %s matches csv definition\n", name)
+	}
+}
+
+func RoleDiffSingle(conf *cfg.Config, csvfile, rolename string) error {
+	res, err := conf.DefaultCluster.ES.Security.GetRole().
+		Name(rolename).
+		Do(context.Background())
+	if err != nil {
+		return fmt.Errorf("failed to get role: %s", esErrorString(err))
+	}
+
+	record, err := getCsvRecord(conf, csvfile, rolename)
+	if err != nil {
+		return err
+	}
+
+	if !record.defined {
+		fmt.Printf("role %s is not defined\n", rolename)
+		return nil
+	}
+
+	role, exists := res[rolename]
+	if !exists {
+		fmt.Printf("role %s is not deployed\n", rolename)
+		return nil
+	} else {
+		fmt.Printf("role %s is deployed\n", rolename)
+	}
+	slog.Debug("found role", "role", role)
+
+	if conf.Debug {
+		// slog.Debug doesn't print it, for whatever reason
+		repr.Println(record)
+	}
+
+	groups, err := getRoleMappingGroups(conf, rolename)
+	if err != nil {
+		return err
+	}
+	slog.Debug("group mappings", "groups", groups)
+
+	// check cluster setting
+	clusters := []string{}
+	for _, cluster := range role.Cluster {
+		clusters = append(clusters, cluster.Name)
+	}
+
+	// check index names+privs
+	indices := []string{}
+	privs := []string{}
+	for _, index := range role.Indices {
+		for _, name := range index.Names {
+			indices = append(indices, strings.ReplaceAll(name, "**", "*"))
+		}
+
+		for _, priv := range index.Privileges {
+			privs = append(privs, priv.Name)
+		}
+	}
+
+	// check kibana application space
+	spaces := []string{}
+	for _, app := range role.Applications {
+		for _, resource := range app.Resources {
+			if strings.Contains(resource, "space:") {
+				parts := strings.Split(resource, ":")
+				if len(parts) == 2 {
+					spaces = append(spaces, parts[1])
+				}
+			}
+		}
+	}
+
+	compareSlices("cluster_privilege", clusters, record.cluster_privilege)
+	compareSlices("ad_group", groups, record.ad_group)
+	compareSlices("index_name", indices, []string{record.index_name})
+	compareSlices("index_privilege", privs, []string{record.index_privilege})
+	compareSlices("space", spaces, []string{record.space})
+
+	return nil
+}

pkg/es/search.go

@@ -18,13 +18,27 @@ package es
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"log"
 	"log/slog"
+	"time"
 
 	"codeberg.org/scip/esctl/pkg/cfg"
+	"codeberg.org/scip/esctl/pkg/printer"
+	"github.com/alecthomas/repr"
+	"github.com/elastic/go-elasticsearch/v9/typedapi/core/search"
+	"github.com/elastic/go-elasticsearch/v9/typedapi/esdsl"
+	"github.com/elastic/go-elasticsearch/v9/typedapi/indices/validatequery"
+	"github.com/elastic/go-elasticsearch/v9/typedapi/types"
+	"github.com/elastic/go-elasticsearch/v9/typedapi/types/enums/sortorder"
 	"github.com/tidwall/gjson"
 )
 
+const (
+	MAXPAGE = 5000
+)
+
 /*
 Execute an ES search.
 
@@ -32,39 +46,219 @@ Execute an ES search.
 	additional filters can be given as -F key=value
 */
 func Search(conf *cfg.Config, queries []string) error {
-	search := conf.DefaultCluster.ES.Search().
+	if conf.Validate {
-		Index(conf.Index)
+		return validateSearch(conf, queries)
+	}
 
-	if len(queries) > 0 {
+	searchEs := conf.DefaultCluster.ES.Search().Index(conf.Index)
-		req, err := prepareQuery(conf, queries)
+
+	queryCaster, err := prepareQuery(conf, queries)
 	if err != nil {
 		return err
 	}
 
-		search.Request(req)
+	req := &search.Request{Query: queryCaster}
-	}
 
-	res, err := search.Do(context.Background())
+	searchEs.Request(req)
-	if err != nil {
-		return fmt.Errorf("failed to run search (esdsl): %s", err)
-	}
 
-	slog.Debug("ES result", "search", res)
+	searchEs = addSort(conf, searchEs)
 
-	for _, hit := range res.Hits.Hits {
+	switch {
-		docjson := fmt.Sprintf(`{"id":%s, "score":%0.4f, "index":"%s", "source":%s}`,
+	case conf.Tail:
-			*hit.Id_,
+		return searchTail(conf, searchEs)
-			*hit.Score_,
+	case conf.Explain:
-			hit.Index_,
+		return explainSearch(conf, searchEs)
-			hit.Source_)
+	default:
-
+		if conf.To > MAXPAGE {
-		if conf.Path != "" {
+			return searchPit(conf, req)
-			value := gjson.Get(docjson, conf.Path)
-			fmt.Println(value.String())
 		} else {
-			fmt.Println(docjson)
+			return searchOnce(conf, searchEs)
+		}
+	}
+}
+
+func explainSearch(conf *cfg.Config, search *search.Search) error {
+	res, err := search.
+		Explain(true).
+		Size(1). // one's enough for explain
+		Do(context.Background())
+	if err != nil {
+		return fmt.Errorf("failed to call explain search (esdsl): %s", esErrorString(err))
+	}
+
+	if conf.Debug {
+		raw, err := json.Marshal(res)
+		if err != nil {
+			return fmt.Errorf("failed to marshal explain result: %s", err)
+		}
+
+		value := gjson.Get(string(raw), "hits.hits.0._explanation")
+		fmt.Println(value.String())
+	}
+
+	if len(res.Hits.Hits) > 0 {
+		ex := res.Hits.Hits[0].Explanation_
+		fmt.Println(ex.Description)
+		fmt.Println(ex.Value)
+
+		// recurse into explanation details (it's a tree)
+		for _, ex := range ex.Details {
+			explain(&ex, "  ")
 		}
 	}
 
 	return nil
 }
+
+func explain(res *types.ExplanationDetail, indent string) {
+	fmt.Println(indent + "- " + res.Description)
+	for _, ex := range res.Details {
+		fmt.Println(indent + "  - " + ex.Description)
+		fmt.Println(indent + fmt.Sprintf("    score: %f", res.Value))
+
+		explain(&ex, indent+"  ")
+	}
+}
+
+func validateSearch(conf *cfg.Config, queries []string) error {
+	validate := conf.DefaultCluster.ES.Indices.ValidateQuery()
+
+	queryCaster, err := prepareQuery(conf, queries)
+	if err != nil {
+		return err
+	}
+
+	req := &validatequery.Request{Query: queryCaster}
+
+	validate.Request(req)
+
+	res, err := validate.
+		Do(context.Background())
+	if err != nil {
+		return fmt.Errorf("failed to validate search (esdsl): %s", esErrorString(err))
+	}
+
+	slog.Debug("ES result", "search", res)
+
+	if res.Valid {
+		fmt.Println(printer.Colorize(conf, "green", "valid"))
+	} else {
+		fmt.Println(printer.Colorize(conf, "red", "invalid"))
+	}
+
+	return nil
+}
+
+func Debug(conf *cfg.Config) error {
+	res, err := conf.DefaultCluster.ES.Search().
+		Index(conf.Index).
+		Size(0).
+		Aggregations(map[string]types.Aggregations{
+			"min_ts": *esdsl.NewMinAggregation().Field("@timestamp").AggregationsCaster(),
+			"max_ts": *esdsl.NewMaxAggregation().Field("@timestamp").AggregationsCaster(),
+		}).
+		Do(context.Background())
+
+	if err != nil {
+		return err
+	}
+
+	repr.Println(res)
+
+	return nil
+}
+
+func searchOnce(conf *cfg.Config, search *search.Search) error {
+	res, err := search.
+		From(conf.From).
+		Size(conf.To).
+		Do(context.Background())
+	if err != nil {
+		return fmt.Errorf("failed to run search (esdsl): %s", esErrorString(err))
+	}
+
+	slog.Debug("ES result", "search", res)
+
+	printer.PrintDocs(conf, res.Hits.Hits)
+
+	return nil
+}
+
+// https://www.elastic.co/docs/reference/elasticsearch/clients/go/using-the-api/searching#_pit_search_after
+func searchPit(conf *cfg.Config, req *search.Request) error {
+	ctx := context.Background()
+	pit, err := conf.DefaultCluster.ES.OpenPointInTime(conf.Index).KeepAlive("1m").Do(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to open point-in-time request for search: %s", err)
+	}
+	defer func() {
+		_, err := conf.DefaultCluster.ES.ClosePointInTime().Id(pit.Id).Do(ctx)
+		if err != nil {
+			log.Fatalf("failed to close PIT: %s", err)
+		}
+	}()
+
+	search := conf.DefaultCluster.ES.Search().
+		Request(req).
+		Pit(esdsl.NewPointInTimeReference().
+			Id(pit.Id).
+			KeepAlive(esdsl.NewDuration().String("1m"))).
+		Sort(esdsl.NewSortOptions().
+			AddSortOption("_shard_doc", esdsl.NewFieldSort(sortorder.Asc))).
+		Size(conf.To)
+
+	search = addSort(conf, search)
+
+	for {
+		res, err := search.Do(ctx)
+		if err != nil {
+			return fmt.Errorf("failed to run search (esdsl pit): %s", esErrorString(err))
+		}
+
+		if len(res.Hits.Hits) == 0 {
+			break
+		}
+
+		printer.PrintDocs(conf, res.Hits.Hits)
+
+		last := res.Hits.Hits[len(res.Hits.Hits)-1]
+		search = search.SearchAfterValues(last.Sort)
+
+		if res.PitId != nil {
+			search = search.Pit(esdsl.NewPointInTimeReference().
+				Id(*res.PitId).
+				KeepAlive(esdsl.NewDuration().String("1m")))
+		}
+	}
+
+	return nil
+}
+
+func searchTail(conf *cfg.Config, search *search.Search) error {
+	docs := map[string]int{}
+
+	fmt.Println("enter ctrl-c to abort...")
+
+	for {
+		res, err := search.Do(context.Background())
+		if err != nil {
+			return fmt.Errorf("failed to run search (esdsl): %s", esErrorString(err))
+		}
+
+		slog.Debug("ES result", "search", res)
+
+		for _, hit := range res.Hits.Hits {
+			_, exists := docs[*hit.Id_]
+			if exists {
+				continue
+			}
+
+			printer.PrintDoc(conf, hit)
+			fmt.Println()
+
+			docs[*hit.Id_] = 1
+		}
+
+		time.Sleep(100 * time.Millisecond)
+	}
+}

pkg/es/search_filter.go

@@ -17,13 +17,18 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
 package es
 
 import (
+	"context"
+	"errors"
 	"fmt"
+	"log/slog"
 	"strings"
 
 	"codeberg.org/scip/esctl/pkg/cfg"
 	"github.com/elastic/go-elasticsearch/v9/typedapi/core/search"
 	"github.com/elastic/go-elasticsearch/v9/typedapi/esdsl"
 	"github.com/elastic/go-elasticsearch/v9/typedapi/types"
+	"github.com/elastic/go-elasticsearch/v9/typedapi/types/enums/operator"
+	"github.com/elastic/go-elasticsearch/v9/typedapi/types/enums/sortorder"
 )
 
 const (
@@ -40,7 +45,22 @@ type filter struct {
 	multi        bool     // message,title=foo => filter:foo, multi: []string{"message","title"}
 }
 
-// build a new filter object
+// Build a new  filter object. We use this to  build our elastic query
+// out of it. We support differnt types of queries:
+//
+//   - nop filter: no query at all, just return the first N documents.
+//
+//   - simple ones like: "authenticated", which match across all fields
+//
+//   - simple ones combined like "authenticated monitoring", if -O was set,
+//     apply an OR logic
+//
+//   - specific field queries: "message=authenticated" (can be combined too like above)
+//
+//   - and specific field queries with negation: "message!=authenticated"
+//
+// All queries support additional  filters using -F field=value, which
+// must match literally, and range filters using -r "@timestamp:2026-05-28T10:00:00 to now"
 func NewFilter(query string) (*filter, error) {
 	var separator string
 	var criteria int // we use the constants on top for this
@@ -49,9 +69,6 @@ func NewFilter(query string) (*filter, error) {
 	case strings.Contains(query, "!="):
 		criteria = Fmustnot
 		separator = "!="
-	case strings.Contains(query, "?"):
-		criteria = Fshould
-		separator = "?"
 	default:
 		criteria = Fmust
 		separator = "="
@@ -59,48 +76,30 @@ func NewFilter(query string) (*filter, error) {
 
 	part := strings.Split(query, separator)
 	if len(part) != 2 {
-		return nil, fmt.Errorf("search queries must be in the form field<sep>pattern where <sep> must be one of: =, !=, ?")
+		return nil, fmt.Errorf("search queries must be in the form field<sep>pattern where <sep> must be one of: = or !=")
 	}
 
-	f := &filter{term: part[0], filter: part[1], criteria: criteria}
+	flt := &filter{term: part[0], filter: part[1], criteria: criteria}
 
 	if strings.Contains(part[0], ",") {
 		// a MultiMatchQuery, match across multiple fields at once
 		multi := strings.Split(part[0], ",")
-		f.multi = true
+		flt.multi = true
-		f.mterm = multi
+		flt.mterm = multi
 	}
 
-	return f, nil
+	return flt, nil
 }
 
-// prepare q user search query and turn it into a proper search request
+// Build a complex query set for queries containing field[!]=pattern
-func prepareQuery(conf *cfg.Config, queries []string) (*search.Request, error) {
+func mkMatchQueries(queries []string, op operator.Operator) ([]types.QueryVariant, []types.QueryVariant, error) {
-	if len(queries) == 0 {
+	matchqueries := []types.QueryVariant{}
-		// nothing given, just return all docs, if any
+	matchNotqueries := []types.QueryVariant{}
-		return &search.Request{
-			Query: esdsl.NewMatchAllQuery().QueryCaster(),
-			From:  &conf.From,
-			Size:  &conf.To,
-		}, nil
-	}
-
-	if len(queries) == 1 && !strings.ContainsAny(queries[0], "!=?") {
-		// a general query w/o fields, search across all fields
-		return &search.Request{
-			Query: esdsl.NewSimpleQueryStringQuery(queries[0]).QueryCaster(),
-			From:  &conf.From,
-			Size:  &conf.To,
-		}, nil
-	}
-
-	// complex query, form a proper query struct
-	query := esdsl.NewBoolQuery()
 
 	for _, q := range queries {
 		filter, err := NewFilter(q)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 
 		// by default we match on a single field
@@ -108,23 +107,89 @@ func prepareQuery(conf *cfg.Config, queries []string) (*search.Request, error) {
 
 		if filter.multi {
 			// ok, match across multiple given fields
-			match = esdsl.NewMultiMatchQuery(filter.filter).Fields(filter.mterm...)
+			match = esdsl.NewMultiMatchQuery(filter.filter).Fields(filter.mterm...).Operator(op)
 		}
 
-		// apply logic
+		if filter.criteria == Fmustnot {
-		switch filter.criteria {
+			matchNotqueries = append(matchNotqueries, match)
-		case Fmustnot:
+		} else {
-			query.MustNot(match)
+			matchqueries = append(matchqueries, match)
-		case Fmust:
-			query.Must(match)
-		case Fshould:
-			query.Should(match)
 		}
 	}
 
-	// there might be boolean filters as well
+	return matchqueries, matchNotqueries, nil
-	filters := make([]types.QueryVariant, len(conf.Filter))
+}
 
+// Prepare user search query and turn it into a proper search request
+func prepareQuery(conf *cfg.Config, queries []string) (*types.Query, error) {
+	// logical operator for simple and multimatch queries
+	op := operator.Operator{Name: "AND"}
+	if conf.Or {
+		op = operator.Operator{Name: "OR"}
+	}
+
+	query := esdsl.NewBoolQuery()
+	wholeQuery := strings.Join(queries, " ")
+
+	switch {
+	case len(queries) == 0:
+		// nothing provided via ARGs, so match any docs
+		query.Must(esdsl.NewMatchAllQuery())
+
+	case !strings.ContainsAny(wholeQuery, "!="):
+		// simple query w/o any field[!]=pattern style
+		query.Must(esdsl.NewSimpleQueryStringQuery(wholeQuery).DefaultOperator(op))
+
+	default:
+		// a complex query
+		matchqueries, matchNotqueries, err := mkMatchQueries(queries, op)
+		if err != nil {
+			return nil, err
+		}
+
+		// apply boolean logic
+		if conf.Or {
+			query.Should(matchqueries...)
+		} else {
+			// by default we use AND
+			query.Must(matchqueries...)
+		}
+
+		if matchNotqueries != nil {
+			query.MustNot(matchNotqueries...)
+		}
+	}
+
+	slog.Debug("complex query", "query", query)
+
+	// there might be boolean or range filters like -Ffield=value as well
+	filters, err := addFilters(conf)
+	if err != nil {
+		return nil, err
+	}
+
+	if filters != nil {
+		query.Filter(filters...)
+	}
+
+	return query.QueryCaster(), nil
+}
+
+// Build a slice of filters, to be fed into query.Filter() including static ones
+// like -Ffield=value and ranges like -r "@timestamp:2026-05-28T10:00:00 to now"
+func addFilters(conf *cfg.Config) ([]types.QueryVariant, error) {
+	count := len(conf.Filter)
+	if conf.Range != "" {
+		count++
+	}
+
+	if count == 0 {
+		return nil, nil
+	}
+
+	filters := make([]types.QueryVariant, count)
+
+	// static filters
 	for idx, filter := range conf.Filter {
 		parts := strings.Split(filter, "=")
 		if len(parts) != 2 {
@@ -134,13 +199,82 @@ func prepareQuery(conf *cfg.Config, queries []string) (*search.Request, error) {
 		filters[idx] = esdsl.NewTermQuery(parts[0], esdsl.NewFieldValue().String(parts[1]))
 	}
 
-	if len(filters) > 0 {
+	// range filters. format: @timestamp:2026-05-05 to 2026-05-15
-		query.Filter(filters...)
+	// see: https://www.elastic.co/docs/reference/elasticsearch/rest-apis/common-options#date-math
+	if conf.Range != "" {
+		parts := strings.SplitN(conf.Range, ":", 2)
+		if len(parts) != 2 {
+			return nil, errors.New("invalid range format, expected field:range")
 		}
 
-	return &search.Request{
+		field := parts[0]
-		Query: query.QueryCaster(),
+
-		From:  &conf.From,
+		parts = strings.Split(parts[1], " to ")
-		Size:  &conf.To,
+		if len(parts) != 2 {
-	}, nil
+			return nil, errors.New("invalid date range format, expected '<start> to <end>'")
+		}
+
+		from := parts[0]
+		to := parts[1]
+
+		slog.Debug("time range filter",
+			"from", from,
+			"to", to,
+			"format", conf.TimestampFormat,
+			"count", count,
+		)
+
+		rng := esdsl.NewDateRangeQuery(field).
+			Gte(from).
+			Lte(to)
+
+		if strings.Contains(parts[1], ":") {
+			// specific format not needed as long as there are no times specified
+			rng.Format(conf.TimestampFormat)
+		}
+
+		filters[count-1] = rng
+
+	}
+
+	return filters, nil
+}
+
+// check if the conf.SortBy  field (default: @timestamp) is searchable
+// by using the field capabilities API.
+func addSort(conf *cfg.Config, search *search.Search) *search.Search {
+	res, err := conf.DefaultCluster.ES.FieldCaps().
+		Index(conf.Index).
+		Fields(conf.SortBy).
+		Do(context.Background())
+
+	if err != nil {
+		// whatever it was, do not add Sort()
+		slog.Debug("get field capabilities", "field", conf.SortBy, "error", esErrorString(err))
+
+		return search
+	}
+
+	field, exists := res.Fields[conf.SortBy]
+
+	if exists {
+		// good, the field exists
+		for _, cap := range field {
+			if cap.Searchable {
+				// ok, ES would be willing to sort by this field
+				order := esdsl.NewFieldSort(sortorder.Desc)
+				if conf.Ascending {
+					order = esdsl.NewFieldSort(sortorder.Asc)
+				}
+
+				search = search.Sort(
+					esdsl.NewSortOptions().AddSortOption(conf.SortBy, order),
+				)
+
+				break
+			}
+		}
+	}
+
+	return search
 }

pkg/es/shard.go

@@ -88,7 +88,7 @@ func ShardList(conf *cfg.Config) error {
 		Header("accept", "application/json").
 		Do(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to get shards: %s", err)
+		return fmt.Errorf("failed to get shards: %s", esErrorString(err))
 	}
 
 	shardlist := filterShards(conf, res)
@@ -140,7 +140,7 @@ func ShardShow(conf *cfg.Config, index string) error {
 		Header("accept", "application/json").
 		Do(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to get shards: %s", err)
+		return fmt.Errorf("failed to get shards: %s", esErrorString(err))
 	}
 
 	slog.Debug("ES result", "shards", res)

pkg/es/snapshot.go

@@ -45,7 +45,7 @@ func SnapshotList(conf *cfg.Config) error {
 	// get partial indicies
 	ires, err := conf.DefaultCluster.ES.Cat.Indices().Do(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to get indicies: %s", err)
+		return fmt.Errorf("failed to get indicies: %s", esErrorString(err))
 	}
 
 	indicies := map[string]int{}
@@ -58,7 +58,7 @@ func SnapshotList(conf *cfg.Config) error {
 	// get snapshots
 	sres, err := conf.DefaultCluster.ES.Cat.Snapshots().Do(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to get snapshots: %s", err)
+		return fmt.Errorf("failed to get snapshots: %s", esErrorString(err))
 	}
 
 	slog.Debug("ES result", "indicies", sres)
@@ -108,7 +108,7 @@ func SnapshotList(conf *cfg.Config) error {
 func SnapshotShow(conf *cfg.Config, snapshot string) error {
 	res, err := conf.DefaultCluster.ES.Snapshot.Get("*", snapshot).Do(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to get snapshot: %s", err)
+		return fmt.Errorf("failed to get snapshot: %s", esErrorString(err))
 	}
 
 	slog.Debug("ES result", "snapshot", res)

pkg/printer/color.go

@@ -24,6 +24,7 @@ import (
 
 var (
 	green  = color.New(color.BgGreen, color.FgHiWhite).SprintFunc()
+	yellow = color.New(color.BgYellow, color.FgHiWhite).SprintFunc()
 	orange = color.BgRGB(255, 128, 0).AddRGB(0, 0, 0).SprintFunc()
 	red    = color.New(color.BgRed, color.FgHiWhite).SprintFunc()
 	bold   = color.New(color.Bold).SprintFunc()
@@ -42,6 +43,8 @@ func Colorize(conf *cfg.Config, col, what string) string {
 		what = orange(what)
 	case "red":
 		what = red(what)
+	case "yellow":
+		what = yellow(what)
 	}
 
 	return what

pkg/printer/doc.go

@@ -0,0 +1,57 @@
+/*
+Copyright © 2026 Thomas von Dein
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+package printer
+
+import (
+	"fmt"
+	"slices"
+
+	"codeberg.org/scip/esctl/pkg/cfg"
+	"github.com/elastic/go-elasticsearch/v9/typedapi/types"
+	"github.com/tidwall/gjson"
+)
+
+func PrintDocs(conf *cfg.Config, hits []types.Hit) {
+	if !conf.Ascending {
+		slices.Reverse(hits)
+	}
+
+	for _, hit := range hits {
+		PrintDoc(conf, hit)
+	}
+
+}
+
+func PrintDoc(conf *cfg.Config, hit types.Hit) {
+	var score types.Float64
+	if hit.Score_ != nil {
+		score = *hit.Score_
+	}
+
+	docjson := fmt.Sprintf(`{"id":"%s", "score":%0.4f, "index":"%s", "source":%s}`,
+		*hit.Id_,
+		score,
+		hit.Index_,
+		hit.Source_)
+
+	if conf.Path != "" {
+		value := gjson.Get(docjson, conf.Path)
+		fmt.Println(value.String())
+	} else {
+		fmt.Print(docjson)
+	}
+}