add role diff (#26)

README.md

@@ -28,8 +28,11 @@ Command tree:
          list
          set
      status
+ debug
  doc
      add
+     delete
+     show
  help
  index
      alias
@@ -40,6 +43,7 @@ Command tree:
      close
      create
      delete
+     fields
      list
      modify
      show
@@ -47,6 +51,9 @@ Command tree:
      list
      show
  repl
+ role
+     list
+     show
  search
  shard
      list

TODO.md

@@ -4,8 +4,30 @@
 - Fix index names custom completion
   https://github.com/urfave/cli/issues/2332
   https://github.com/urfave/cli/issues/2333
-  
+
 - index show: add more details, see screenshots
 
 - add shard explain, aka:
   get /_cluster/allocation/explain {"index":"yourindex", "primary": true, "shard":0}
+
+
+- add validate:
+
+> GET /mock/_validate/query?rewrite=true {"from":0,"query":{"bool":{"must":[{"match_all":{}}]}},"size":20,"sort":[{"name":{"order":"desc"}}]}
+{
+  "valid": false
+}
+
+- add explain to search (maybe option -e)
+
+> GET /mock/_explain/1780043878 {"query":{"bool":{"must":[{"match_all":{}}]}}}
+{
+  "_index": "mock",
+  "_id": "1780043878",
+  "matched": true,
+  "explanation": {
+    "value": 1.0,
+    "description": "*:*",
+    "details": []
+  }
+}

cmd/index.go

@@ -42,6 +42,7 @@ func Index(conf *cfg.Config) *cli.Command {
 			IndexAllocation(conf),
 			IndexModify(conf),
 			IndexAlias(conf),
+			IndexFields(conf),
 		},
 	}
 }
@@ -157,6 +158,8 @@ func IndexCreate(conf *cfg.Config) *cli.Command {
 		Name:    "create",
 		Aliases: []string{"+"},
 		Usage:   "create a new index",
+		UsageText: `create <name> <fieldmapping:type>...
+Valid field mapping types: integer, text, date, keyword`,
 
 		Flags: []cli.Flag{
 			&cli.BoolFlag{
@@ -250,3 +253,20 @@ func IndexModify(conf *cfg.Config) *cli.Command {
 		},
 	}
 }
+
+func IndexFields(conf *cfg.Config) *cli.Command {
+	return &cli.Command{
+		Name:      "fields",
+		Usage:     "show info about field capabilities",
+		UsageText: "index fields <index>",
+
+		Action: func(ctx context.Context, cmd *cli.Command) error {
+			index := cmd.Args().Get(0)
+			if index == "" {
+				return errors.New("no index specified")
+			}
+
+			return es.IndexFields(conf, index)
+		},
+	}
+}

cmd/roles.go

@@ -0,0 +1,137 @@
+/*
+Copyright © 2026 Thomas von Dein
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+package cmd
+
+import (
+	"context"
+	"errors"
+
+	"codeberg.org/scip/esctl/pkg/cfg"
+	"codeberg.org/scip/esctl/pkg/es"
+
+	"github.com/urfave/cli/v3"
+)
+
+func Roles(conf *cfg.Config) *cli.Command {
+	return &cli.Command{
+		Name:  "role",
+		Usage: "manage roles",
+
+		Commands: []*cli.Command{
+			RoleList(conf),
+			RoleShow(conf),
+			RoleDiff(conf),
+		},
+	}
+}
+
+func RoleList(conf *cfg.Config) *cli.Command {
+	return &cli.Command{
+		Name:    "list",
+		Aliases: []string{"ls"},
+		Usage:   "list roles",
+
+		Flags: []cli.Flag{
+			&cli.BoolFlag{
+				Name:        "orphaned",
+				Usage:       "include only orphaned roles",
+				Destination: &conf.Failed,
+				Aliases:     []string{"o"},
+			},
+		},
+
+		Action: func(ctx context.Context, cmd *cli.Command) error {
+			return es.RoleList(conf)
+		},
+	}
+}
+
+func RoleShow(conf *cfg.Config) *cli.Command {
+	return &cli.Command{
+		Name:      "show",
+		Aliases:   []string{"sh"},
+		Usage:     "show details about a role",
+		UsageText: "show [options] <role>",
+
+		Action: func(ctx context.Context, cmd *cli.Command) error {
+			index := cmd.Args().Get(0)
+			if index == "" {
+				return errors.New("no role specified")
+			}
+
+			return es.RoleShow(conf, cmd.Args().Get(0))
+		},
+	}
+}
+
+func RoleDiff(conf *cfg.Config) *cli.Command {
+	return &cli.Command{
+		Name:  "diff",
+		Usage: "show differences between roles and CSV baseline",
+		UsageText: `diff [options] <file.csv> [<role>]
+CSV format:
+index_name;role;index_privilege;cluster_privilege;ad_group;space;retention;kibana_privilege;field_privilege`,
+
+		MutuallyExclusiveFlags: []cli.MutuallyExclusiveFlags{{
+			Flags: [][]cli.Flag{
+				{
+					&cli.BoolFlag{
+						Name:        "not-deployed",
+						Usage:       "include only not deployed but defined roles",
+						Destination: &conf.NotDeployed,
+						Aliases:     []string{"n"},
+					},
+				},
+				{
+					&cli.BoolFlag{
+						Name:        "undefined",
+						Usage:       "include only deployed but undefined roles",
+						Destination: &conf.Undefined,
+						Aliases:     []string{"u"},
+					},
+				},
+				{
+					&cli.BoolFlag{
+						Name:        "diff",
+						Usage:       "include only differing roles",
+						Destination: &conf.Diff,
+						Aliases:     []string{"D"},
+					},
+				},
+			}},
+		},
+
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:        "separator",
+				Usage:       "CSV field separator",
+				Destination: &conf.Separator,
+				Aliases:     []string{"s"},
+				Value:       ",",
+			},
+		},
+
+		Action: func(ctx context.Context, cmd *cli.Command) error {
+			csvfile := cmd.Args().Get(0)
+			if csvfile == "" {
+				return errors.New("no CSV file specified")
+			}
+
+			return es.RoleDiff(conf, cmd.Args().Get(0), cmd.Args().Get(1))
+		},
+	}
+}

cmd/root.go

@@ -97,6 +97,7 @@ func Main() int {
 			Repl(conf),
 			Version(conf),
 			Debug(conf),
+			Roles(conf),
 		},
 
 		Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) {

cmd/search.go

@@ -100,6 +100,19 @@ func Search(conf *cfg.Config) *cli.Command {
 				Destination: &conf.TimestampFormat,
 				Value:       "strict_date_hour_minute",
 			},
+			&cli.StringFlag{
+				Name:        "sort-by",
+				Usage:       "sort by a field",
+				Destination: &conf.SortBy,
+				Value:       "@timestamp",
+				Aliases:     []string{"k"},
+			},
+			&cli.BoolFlag{
+				Name:        "ascending",
+				Usage:       "sort in ascending order (default: descending)",
+				Destination: &conf.Ascending,
+				Aliases:     []string{"a"},
+			},
 			&cli.BoolFlag{
 				Name:        "help-jsonpath",
 				Usage:       "show jsonPath help",

pkg/cfg/config.go

@@ -34,7 +34,7 @@ import (
 )
 
 const (
-	Version string = `v0.0.14`
+	Version string = `v0.0.16`
 )
 
 var (
@@ -67,12 +67,19 @@ type Config struct {
 	Or                             bool     // search: -O
 	Range                          string   // search: -r
 	TimestampFormat                string   // search: --timestamp-format
+	Explain                        bool     // search: -e
+	SortBy                         string   // sort: -k
+	Ascending                      bool     // sort: -a
 	Exclude                        string   // cluster compare: -e (regexp)
 	All, Verbose                   bool     // cluster status: -a -v
 	Persistent, Transient, Default bool     // -p -t -D cluster settings set
 	Force                          bool     // ccr follower renew: -f
 	HaveJQ                         bool     // determined at runtime by ourselfes
 	DebugHTTP                      bool     // root: --debug-http
+	Separator                      string   // role diff: -s
+	NotDeployed                    bool     // role diff: -n
+	Undefined                      bool     // role diff: -u
+	Diff                           bool     // role diff: -D
 }
 
 func NewConfig() *Config {

pkg/es/ccr.go

@@ -23,7 +23,7 @@ import (
 	"log/slog"
 
 	"codeberg.org/scip/esctl/pkg/cfg"
-    "codeberg.org/scip/esctl/pkg/printer"
+	"codeberg.org/scip/esctl/pkg/printer"
 	"github.com/elastic/go-elasticsearch/v9/typedapi/types"
 )
 
@@ -62,7 +62,7 @@ func CcrStatus(conf *cfg.Config, leader, follower string) error {
 
 		res, err := cat.Do(context.Background())
 		if err != nil {
-			return fmt.Errorf("failed to get indicies on %s: %s", alias, err)
+			return fmt.Errorf("failed to get indicies on %s: %s", alias, esErrorString(err))
 		}
 
 		indices[alias] = make(map[string]*types.IndicesRecord, len(res))
@@ -93,7 +93,7 @@ func CcrRemoteInfo(conf *cfg.Config, index string) error {
 		Header("accept", "application/json").
 		Do(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to retrieve follower info: %s", err)
+		return fmt.Errorf("failed to retrieve follower info: %s", esErrorString(err))
 	}
 
 	slog.Debug("ccr remote info", "info", res)

pkg/es/ccr_follower.go

@@ -22,7 +22,7 @@ import (
 	"log/slog"
 
 	"codeberg.org/scip/esctl/pkg/cfg"
-    "codeberg.org/scip/esctl/pkg/printer"
+	"codeberg.org/scip/esctl/pkg/printer"
 )
 
 func getRemoteName(conf *cfg.Config) (string, error) {
@@ -31,7 +31,7 @@ func getRemoteName(conf *cfg.Config) (string, error) {
 		Header("accept", "application/json").
 		Do(context.Background())
 	if err != nil {
-		return "", fmt.Errorf("failed to retrieve follower info: %s", err)
+		return "", fmt.Errorf("failed to retrieve follower info: %s", esErrorString(err))
 	}
 
 	remote := ""
@@ -41,7 +41,7 @@ func getRemoteName(conf *cfg.Config) (string, error) {
 	}
 
 	if remote == "" {
-		return "", fmt.Errorf("cluster doesn't have a follower: %s", err)
+		return "", fmt.Errorf("cluster doesn't have a follower")
 	}
 
 	return remote, nil
@@ -96,7 +96,7 @@ func CcrFollowerResume(conf *cfg.Config, index string) error {
 	_, err := create.Do(context.Background())
 
 	if err != nil {
-		return fmt.Errorf("failed to resume ccr following: %s", err)
+		return fmt.Errorf("failed to resume ccr following: %s", esErrorString(err))
 	}
 
 	return nil
@@ -110,7 +110,7 @@ func CcrFollowerPause(conf *cfg.Config, index string) error {
 	_, err := create.Do(context.Background())
 
 	if err != nil {
-		return fmt.Errorf("failed to pause ccr following: %s", err)
+		return fmt.Errorf("failed to pause ccr following: %s", esErrorString(err))
 	}
 
 	return nil
@@ -124,7 +124,7 @@ func CcrFollowerUnfollow(conf *cfg.Config, index string) error {
 	_, err := create.Do(context.Background())
 
 	if err != nil {
-		return fmt.Errorf("failed to unfollow index: %s", err)
+		return fmt.Errorf("failed to unfollow index: %s", esErrorString(err))
 	}
 
 	return nil
@@ -149,7 +149,7 @@ func CcrFollowerAdd(conf *cfg.Config, index string) error {
 	_, err = create.Do(context.Background())
 
 	if err != nil {
-		return fmt.Errorf("failed to create follower index: %s", err)
+		return fmt.Errorf("failed to create follower index: %s", esErrorString(err))
 	}
 
 	return nil
@@ -161,7 +161,7 @@ func CcrFollowerShow(conf *cfg.Config, index string) error {
 		Header("accept", "application/json").
 		Do(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to retrieve follower index info: %s", err)
+		return fmt.Errorf("failed to retrieve follower index info: %s", esErrorString(err))
 	}
 
 	slog.Debug("ES result", "follower stats", res.Indices)

pkg/es/cluster_settings.go

@@ -23,7 +23,7 @@ import (
 	"log/slog"
 
 	"codeberg.org/scip/esctl/pkg/cfg"
-    "codeberg.org/scip/esctl/pkg/printer"
+	"codeberg.org/scip/esctl/pkg/printer"
 	"github.com/urfave/cli/v3"
 )
 
@@ -63,7 +63,7 @@ func ClusterSettingsList(conf *cfg.Config) error {
 		Header("accept", "application/json").
 		Do(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to get cluster settings: %s", err)
+		return fmt.Errorf("failed to get cluster settings: %s", esErrorString(err))
 	}
 
 	table := printer.NewTable(conf, 2, 0)
@@ -135,7 +135,7 @@ func ClusterSettingsSet(conf *cfg.Config, args cli.Args) error {
 		Do(context.Background())
 
 	if err != nil {
-		return fmt.Errorf("failed to set settings: %s", err)
+		return fmt.Errorf("failed to set settings: %s", esErrorString(err))
 	}
 
 	return nil
@@ -154,7 +154,7 @@ func ClusterSettingsSetSingle(conf *cfg.Config, setting, value string) error {
 		Do(context.Background())
 
 	if err != nil {
-		return fmt.Errorf("failed to set %s: %s", setting, err)
+		return fmt.Errorf("failed to set %s: %s", setting, esErrorString(err))
 	}
 
 	return nil

pkg/es/cluster_util.go

@@ -41,7 +41,7 @@ func checkClusterIsLeader(conf *cfg.Config, leader string) bool {
 		Header("accept", "application/json").
 		Do(context.Background())
 	if err != nil {
-		fmt.Printf("failed to get ccr stats from %s: %s", leader, err)
+		fmt.Printf("failed to get ccr stats from %s: %s", leader, esErrorString(err))
 		return false
 	}
 
@@ -68,7 +68,7 @@ func checkClusterStatus(conf *cfg.Config, leader, follower string) bool {
 			Header("accept", "application/json").
 			Do(context.Background())
 		if err != nil {
-			fmt.Printf("failed to get health from %s: %s", cluster, err)
+			fmt.Printf("failed to get health from %s: %s", cluster, esErrorString(err))
 			return false
 		}
 
@@ -128,7 +128,7 @@ func findIlmErrors(conf *cfg.Config, leader, follower string) bool {
 			Header("accept", "application/json").
 			Do(context.Background())
 		if err != nil {
-			fmt.Printf("failed to get ilm status from %s: %s", cluster, err)
+			fmt.Printf("failed to get ilm status from %s: %s", cluster, esErrorString(err))
 			return false
 		}
 

pkg/es/doc.go

@@ -53,7 +53,7 @@ func DocAdd(conf *cfg.Config, jsondoc string) error {
 		Header("accept", "application/json").
 		Do(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to create new doc in index %s: %s", conf.Index, err)
+		return fmt.Errorf("failed to create new doc in index %s: %s", conf.Index, esErrorString(err))
 	}
 
 	fmt.Println(res.Id_)
@@ -67,7 +67,7 @@ func DocShow(conf *cfg.Config, id string) error {
 		Header("accept", "application/json").
 		Do(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to retrieve doc in index %s: %s", conf.Index, err)
+		return fmt.Errorf("failed to retrieve doc in index %s: %s", conf.Index, esErrorString(err))
 	}
 
 	if !res.Found {
@@ -100,7 +100,7 @@ func DocDelete(conf *cfg.Config, queries []string) error {
 			Header("accept", "application/json").
 			Do(context.Background())
 		if err != nil {
-			return fmt.Errorf("failed to delete doc in index %s: %s", conf.Index, err)
+			return fmt.Errorf("failed to delete doc in index %s: %s", conf.Index, esErrorString(err))
 		}
 
 		return nil
@@ -125,7 +125,7 @@ func DocDelete(conf *cfg.Config, queries []string) error {
 		Header("accept", "application/json").
 		Do(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to delete docs in index %s: %s", conf.Index, err)
+		return fmt.Errorf("failed to delete docs in index %s: %s", conf.Index, esErrorString(err))
 	}
 
 	return nil

pkg/es/errors.go

@@ -0,0 +1,45 @@
+/*
+Copyright © 2026 Thomas von Dein
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+package es
+
+import (
+	"fmt"
+
+	"github.com/elastic/go-elasticsearch/v9/typedapi/types"
+)
+
+func esErrorString(err error) string {
+	msg := err.Error()
+
+	switch e := err.(type) {
+	case *types.ElasticsearchError:
+		causes := ""
+
+		for _, cause := range e.ErrorCause.RootCause {
+			causes += fmt.Sprintf("%s\n", *cause.Reason)
+		}
+
+		if e.ErrorCause.Reason != nil {
+			msg = *e.ErrorCause.Reason + ": " + causes
+		} else {
+			msg = fmt.Sprintf("http status %d: ", e.Status)
+		}
+
+	}
+
+	return msg
+}

pkg/es/index.go

@@ -40,7 +40,7 @@ func IndexNames(conf *cfg.Config) ([]string, error) {
 		Do(context.Background())
 
 	if err != nil {
-		return nil, fmt.Errorf("failed to get indicies: %s", err)
+		return nil, fmt.Errorf("failed to get indicies: %s", esErrorString(err))
 	}
 
 	indices := make([]string, len(res))
@@ -81,7 +81,7 @@ func IndexList(conf *cfg.Config) error {
 
 	res, err := cat.Do(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to get indicies: %s", err)
+		return fmt.Errorf("failed to get indicies: %s", esErrorString(err))
 	}
 
 	slog.Debug("ES result", "indicies", res)
@@ -124,7 +124,7 @@ func IndexShow(conf *cfg.Config, index string) error {
 		Header("accept", "application/json").
 		Do(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to get index: %s", err)
+		return fmt.Errorf("failed to get index: %s", esErrorString(err))
 	}
 
 	slog.Debug("ES result", "index", res)
@@ -208,7 +208,7 @@ func IndexCreate(conf *cfg.Config, index string, mappings []string) error {
 		Do(context.Background())
 
 	if err != nil {
-		return fmt.Errorf("failed to create index: %s", err)
+		return fmt.Errorf("failed to create index: %s", esErrorString(err))
 	}
 
 	return nil
@@ -220,7 +220,7 @@ func IndexDelete(conf *cfg.Config, index string) error {
 		Header("accept", "application/json").
 		Do(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to delete index: %s", err)
+		return fmt.Errorf("failed to delete index: %s", esErrorString(err))
 	}
 
 	return nil
@@ -234,7 +234,7 @@ func IndexClose(conf *cfg.Config, index string) error {
 	_, err := create.Do(context.Background())
 
 	if err != nil {
-		return fmt.Errorf("failed to close index: %s", err)
+		return fmt.Errorf("failed to close index: %s", esErrorString(err))
 	}
 
 	return nil
@@ -249,7 +249,7 @@ func IndexAllocation(conf *cfg.Config, index string) error {
 		Header("accept", "application/json").
 		Do(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to get index allocation explain: %s", err)
+		return fmt.Errorf("failed to get index allocation explain: %s", esErrorString(err))
 	}
 
 	slog.Debug("ES result", "index", res)
@@ -294,7 +294,44 @@ func IndexModify(conf *cfg.Config, index string) error {
 		Header("accept", "application/json").
 		Do(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to modify index settings: %s", err)
+		return fmt.Errorf("failed to modify index settings: %s", esErrorString(err))
+	}
+
+	return nil
+}
+
+func IndexFields(conf *cfg.Config, index string) error {
+	res, err := conf.DefaultCluster.ES.FieldCaps().
+		Index(index).
+		Fields("*").
+		Header("content-type", "application/json").
+		Header("accept", "application/json").
+		Do(context.Background())
+	if err != nil {
+		return fmt.Errorf("failed to retrieve field capabilties: %s", esErrorString(err))
+	}
+
+	table := printer.NewTable(conf, 5, len(res.Fields))
+	table.Addheaders("field", "type", "searchable", "aggretable", "metadata")
+
+	idx := 0
+	for name, field := range res.Fields {
+		for fieldtype, caps := range field {
+			// fields only have 1 type, so this one is it
+			table.Entries[idx] = []string{name, fieldtype,
+				fmt.Sprintf("%t", caps.Searchable),
+				fmt.Sprintf("%t", caps.Aggregatable),
+				fmt.Sprintf("%t", *caps.MetadataField)}
+			break
+		}
+
+		idx++
+	}
+
+	table.Sort()
+
+	if err := table.Print(); err != nil {
+		return err
 	}
 
 	return nil

pkg/es/index_alias.go

@@ -37,7 +37,7 @@ func IndexAliasCreate(conf *cfg.Config, index, alias string) error {
 	slog.Debug("create alias", "result", res)
 
 	if err != nil {
-		return fmt.Errorf("failed to create index alias: %s", err)
+		return fmt.Errorf("failed to create index alias: %s", esErrorString(err))
 	}
 
 	return nil
@@ -58,7 +58,7 @@ func IndexAliasList(conf *cfg.Config) error {
 		Do(context.Background())
 
 	if err != nil {
-		return fmt.Errorf("failed to list index aliases: %s", err)
+		return fmt.Errorf("failed to list index aliases: %s", esErrorString(err))
 	}
 
 	slog.Debug("aliases list", "result", res)
@@ -108,7 +108,7 @@ func IndexAliasDelete(conf *cfg.Config, index, alias string) error {
 	slog.Debug("delete alias", "result", res)
 
 	if err != nil {
-		return fmt.Errorf("failed to delete index alias: %s", err)
+		return fmt.Errorf("failed to delete index alias: %s", esErrorString(err))
 	}
 
 	return nil

pkg/es/node.go

@@ -22,14 +22,14 @@ import (
 	"log/slog"
 
 	"codeberg.org/scip/esctl/pkg/cfg"
-    "codeberg.org/scip/esctl/pkg/printer"
+	"codeberg.org/scip/esctl/pkg/printer"
 )
 
 func NodeList(conf *cfg.Config) error {
 	// get nodes
 	nodes, err := conf.DefaultCluster.ES.Cat.Nodes().Do(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to get nodes: %s", err)
+		return fmt.Errorf("failed to get nodes: %s", esErrorString(err))
 	}
 
 	slog.Debug("ES result", "nodes", nodes)

pkg/es/repl.go

@@ -112,7 +112,7 @@ func Repl(conf *cfg.Config) error {
 
 		err = CallAPI(conf, parts[0], parts[1], data)
 		if err != nil {
-			fmt.Printf("failed to call API: %s\n", err)
+			fmt.Printf("failed to call API: %s\n", esErrorString(err))
 		}
 
 		reader.SetPrompt("> ")

pkg/es/role.go

@@ -0,0 +1,228 @@
+/*
+Copyright © 2026 Thomas von Dein
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+package es
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"strings"
+
+	"codeberg.org/scip/esctl/pkg/cfg"
+	"codeberg.org/scip/esctl/pkg/printer"
+	"github.com/elastic/go-elasticsearch/v9/typedapi/types"
+)
+
+func RoleList(conf *cfg.Config) error {
+	res, err := conf.DefaultCluster.ES.Security.GetRole().
+		Do(context.Background())
+	if err != nil {
+		return fmt.Errorf("failed to get roles: %s", esErrorString(err))
+	}
+
+	slog.Debug("ES result", "roles", res)
+
+	table := printer.NewTable(conf, 3, len(res))
+	table.Addheaders("role", "index roles", "cluster roles")
+
+	idx := 0
+	for name, role := range res {
+		table.Entries[idx] = []string{
+			name,
+			fmt.Sprintf("%d", len(role.Cluster)),
+			fmt.Sprintf("%d", len(role.Indices)),
+		}
+		idx++
+	}
+
+	table.Sort()
+	return table.Print()
+}
+
+func RoleShow(conf *cfg.Config, rolename string) error {
+	res, err := conf.DefaultCluster.ES.Security.GetRole().
+		Name(rolename).
+		Do(context.Background())
+	if err != nil {
+		return fmt.Errorf("failed to get role: %s", esErrorString(err))
+	}
+
+	slog.Debug("ES result", "role", res)
+
+	role, exists := res[rolename]
+	if !exists {
+		return fmt.Errorf("role %s does not exist", rolename)
+	}
+
+	if len(role.Indices) > 0 {
+		if err := roleIndices(conf, role); err != nil {
+			return err
+		}
+
+		fmt.Println()
+	}
+
+	if len(role.RemoteIndices) > 0 {
+		if err := roleRemoteIndices(conf, role); err != nil {
+			return err
+		}
+
+		fmt.Println()
+	}
+
+	if len(role.Cluster) > 0 {
+		if err := roleClusters(conf, role); err != nil {
+			return err
+		}
+
+		fmt.Println()
+	}
+
+	if len(role.RemoteCluster) > 0 {
+		if err := roleRemoteClusters(conf, role); err != nil {
+			return err
+		}
+
+		fmt.Println()
+	}
+
+	if len(role.Applications) > 0 {
+		if err := roleApplications(conf, role); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func roleRemoteClusters(conf *cfg.Config, role types.Role) error {
+	if len(role.RemoteCluster) == 0 {
+		return nil
+	}
+
+	table := printer.NewTable(conf, 2, len(role.RemoteCluster))
+	table.Addheaders("remote cluster", "privilege")
+	idx := 0
+	for _, priv := range role.RemoteCluster {
+		perms := []string{}
+		for _, perm := range priv.Privileges {
+			perms = append(perms, perm.Name)
+		}
+
+		table.Entries[idx] = []string{
+			strings.Join(priv.Clusters, ","),
+			strings.Join(perms, ","),
+		}
+		idx++
+	}
+
+	table.Sort()
+	return table.Print()
+}
+
+func roleClusters(conf *cfg.Config, role types.Role) error {
+	if len(role.Cluster) == 0 {
+		return nil
+	}
+
+	table := printer.NewTable(conf, 1, len(role.Cluster))
+	table.Addheaders("cluster rights")
+	idx := 0
+	for _, cluster := range role.Cluster {
+		table.Entries[idx] = []string{cluster.Name}
+		idx++
+	}
+
+	table.Sort()
+	return table.Print()
+}
+
+func roleRemoteIndices(conf *cfg.Config, role types.Role) error {
+	if len(role.RemoteIndices) == 0 {
+		return nil
+	}
+
+	table := printer.NewTable(conf, 3, len(role.RemoteIndices))
+	table.Addheaders("remote index names", "index permissions", "allow restricted")
+	idx := 0
+	for _, priv := range role.RemoteIndices {
+		perms := []string{}
+		for _, perm := range priv.Privileges {
+			perms = append(perms, perm.Name)
+		}
+
+		table.Entries[idx] = []string{
+			strings.Join(priv.Names, ", "),
+			strings.Join(perms, ", "),
+			fmt.Sprintf("%t", *priv.AllowRestrictedIndices),
+		}
+
+		idx++
+	}
+
+	table.Sort()
+	return table.Print()
+}
+
+func roleIndices(conf *cfg.Config, role types.Role) error {
+	if len(role.Indices) == 0 {
+		return nil
+	}
+
+	table := printer.NewTable(conf, 3, len(role.Indices))
+	table.Addheaders("index names", "index permissions", "allow restricted")
+	idx := 0
+	for _, priv := range role.Indices {
+		perms := []string{}
+		for _, perm := range priv.Privileges {
+			perms = append(perms, perm.Name)
+		}
+
+		table.Entries[idx] = []string{
+			strings.Join(priv.Names, ", "),
+			strings.Join(perms, ", "),
+			fmt.Sprintf("%t", *priv.AllowRestrictedIndices),
+		}
+
+		idx++
+	}
+
+	table.Sort()
+	return table.Print()
+}
+
+func roleApplications(conf *cfg.Config, role types.Role) error {
+	if len(role.Applications) == 0 {
+		return nil
+	}
+
+	table := printer.NewTable(conf, 3, len(role.Applications))
+	table.Addheaders("application", "privileges", "resources")
+	idx := 0
+	for _, priv := range role.Applications {
+		table.Entries[idx] = []string{
+			priv.Application,
+			strings.Join(priv.Privileges, ", "),
+			strings.Join(priv.Resources, ", "),
+		}
+
+		idx++
+	}
+
+	table.Sort()
+	return table.Print()
+}

pkg/es/role_diff.go

@@ -0,0 +1,354 @@
+/*
+Copyright © 2026 Thomas von Dein
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+package es
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"encoding/csv"
+	"fmt"
+	"log"
+	"log/slog"
+	"os"
+	"slices"
+	"strings"
+
+	"codeberg.org/scip/esctl/pkg/cfg"
+	"codeberg.org/scip/esctl/pkg/printer"
+	"github.com/alecthomas/repr"
+	"github.com/elastic/go-elasticsearch/v9/typedapi/security/getrole"
+)
+
+// use static csv  record positions as const vars so  we can modify it
+// if the csv format ever changes
+const (
+	Rindexname = iota
+	Rrole
+	Rindexprivilege
+	Rclusterprivilege
+	Radgroup
+	Rspace
+	Rretention
+	Rkibanaprivilege
+	Rfieldprivilege
+)
+
+type Record struct {
+	// filled from CSV input
+	index_name        string
+	role              string
+	index_privilege   string
+	cluster_privilege []string
+	ad_group          []string
+	space             string
+	retention         string
+	kibana_privilege  string
+	field_privilege   string
+
+	// set by ourselfes
+	defined bool
+}
+
+type Register struct {
+	name              string
+	deployed, defined bool
+}
+
+// generic variant,  we do not account  for multiple rows of  the same
+// role,  in  such  cases  an   entry  will  simply  overwritten.  Use
+// getCsvRecord() for a single role.
+func getCsvRecords(conf *cfg.Config, csvfile string) (map[string]Record, error) {
+	data, err := os.ReadFile(csvfile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read CSV file: %s", err)
+	}
+
+	csvreader := csv.NewReader(bytes.NewReader(data))
+	csvreader.Comma = rune(conf.Separator[0])
+	csvreader.Comment = '#'
+	csvreader.TrimLeadingSpace = true
+
+	rows, err := csvreader.ReadAll()
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse CSV: %s", err)
+	}
+
+	records := make(map[string]Record, len(rows)-1)
+
+	for idx, row := range rows {
+		if idx == 0 {
+			continue // header
+		}
+
+		records[row[1]] = Record{
+			index_name:        row[Rindexname],
+			role:              row[Rrole],
+			index_privilege:   row[Rindexprivilege],
+			cluster_privilege: []string{row[Rindexprivilege]},
+			ad_group:          []string{row[Rclusterprivilege]},
+			space:             row[Rspace],
+			retention:         row[Rretention],
+			kibana_privilege:  row[Rkibanaprivilege],
+			field_privilege:   row[Rfieldprivilege],
+			defined:           true,
+		}
+	}
+
+	return records, nil
+}
+
+// same thing  as above but  for one specific role.  supports multiple
+// rows  of  the same  record  with  different  values which  will  be
+// combined.
+func getCsvRecord(conf *cfg.Config, csvfile, rolename string) (*Record, error) {
+	fd, err := os.Open(csvfile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open CSV file: %s", err)
+	}
+	defer func() {
+		if err := fd.Close(); err != nil {
+			log.Fatalf("failed to close file: %s", err)
+		}
+	}()
+
+	scanner := bufio.NewScanner(fd)
+	record := Record{role: rolename}
+
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if strings.HasPrefix(line, "#") || line == "" {
+			continue
+		}
+
+		row := strings.Split(line, conf.Separator)
+
+		if row[Rrole] == rolename {
+			record.index_name = row[Rindexname]
+			record.index_privilege = row[Rindexprivilege]
+			record.cluster_privilege = strings.Split(row[Rclusterprivilege], ",")
+			record.ad_group = append(record.ad_group, row[Radgroup])
+			record.space = row[Rspace]
+			record.retention = row[Rretention]
+			record.kibana_privilege = row[Rkibanaprivilege]
+			record.field_privilege = row[Rfieldprivilege]
+			record.defined = true
+		}
+	}
+
+	return &record, nil
+}
+
+func diffRoles(conf *cfg.Config, records map[string]Record, res getrole.Response) []Register {
+	rows := []Register{}
+	filtered := []Register{}
+	deployed := map[string]int{}
+
+	// iterate over deployed roles
+	for name := range res {
+		reg := Register{name: name}
+
+		_, defined := records[name]
+		if defined {
+			reg.deployed = true
+			reg.defined = true
+		} else {
+			reg.deployed = true
+			reg.defined = false
+		}
+
+		deployed[name] = 1
+
+		rows = append(rows, reg)
+	}
+
+	// iterate over records from CSV and register only those which are not deployed
+	for name := range records {
+		reg := Register{name: name, defined: true}
+		_, deployed := deployed[name]
+		if !deployed {
+			rows = append(rows, reg)
+		}
+	}
+
+	for _, reg := range rows {
+		if (conf.NotDeployed && !reg.deployed) ||
+			(conf.Undefined && !reg.defined) ||
+			(conf.Diff && reg.deployed != reg.defined) ||
+			(!conf.Undefined && !conf.NotDeployed && !conf.Diff) {
+			filtered = append(filtered, reg)
+		}
+	}
+
+	return filtered
+}
+
+func RoleDiff(conf *cfg.Config, csvfile, role string) error {
+	records, err := getCsvRecords(conf, csvfile)
+	if err != nil {
+		return err
+	}
+
+	if role != "" {
+		return RoleDiffSingle(conf, csvfile, role)
+	}
+
+	res, err := conf.DefaultCluster.ES.Security.GetRole().
+		Do(context.Background())
+	if err != nil {
+		return fmt.Errorf("failed to get roles: %s", esErrorString(err))
+	}
+
+	rows := diffRoles(conf, records, res)
+
+	table := printer.NewTable(conf, 3, len(rows))
+	table.Addheaders("role", "is deployed", "is defined")
+
+	for idx, row := range rows {
+		deployed := printer.Colorize(conf, "green", "deployed")
+		if !row.deployed {
+			deployed = printer.Colorize(conf, "red", "not deployed")
+		}
+
+		defined := printer.Colorize(conf, "green", "defined")
+		if !row.defined {
+			defined = printer.Colorize(conf, "red", "undefined")
+		}
+
+		table.Entries[idx] = []string{
+			row.name,
+			deployed,
+			defined,
+		}
+	}
+
+	table.Sort()
+
+	return table.Print()
+}
+
+func getRoleMappingGroups(conf *cfg.Config, rolename string) ([]string, error) {
+	mappings, err := conf.DefaultCluster.ES.Security.GetRoleMapping().
+		Do(context.Background())
+	if err != nil {
+		return nil, fmt.Errorf("failed to get role mappings: %s", esErrorString(err))
+	}
+
+	groups := []string{}
+	for _, mapping := range mappings {
+		if slices.Contains(mapping.Roles, rolename) {
+			for _, rule := range mapping.Rules.Any {
+				for _, group := range rule.Field["groups"] {
+					groups = append(groups, group.(string))
+				}
+			}
+		}
+	}
+
+	return groups, nil
+}
+
+func compareSlices(name string, a, b []string) {
+	slices.Sort(a)
+	slices.Sort(b)
+
+	if slices.Compare(a, b) != 0 {
+		fmt.Printf("%s differs:\ndeployed: %s\n     csv: %s\n",
+			name, strings.Join(a, ","), strings.Join(b, ","))
+	} else {
+		fmt.Printf("deployed %s matches csv definition\n", name)
+	}
+}
+
+func RoleDiffSingle(conf *cfg.Config, csvfile, rolename string) error {
+	res, err := conf.DefaultCluster.ES.Security.GetRole().
+		Name(rolename).
+		Do(context.Background())
+	if err != nil {
+		return fmt.Errorf("failed to get role: %s", esErrorString(err))
+	}
+
+	record, err := getCsvRecord(conf, csvfile, rolename)
+	if err != nil {
+		return err
+	}
+
+	if !record.defined {
+		fmt.Printf("role %s is not defined\n", rolename)
+		return nil
+	}
+
+	role, exists := res[rolename]
+	if !exists {
+		fmt.Printf("role %s is not deployed\n", rolename)
+		return nil
+	} else {
+		fmt.Printf("role %s is deployed\n", rolename)
+	}
+	slog.Debug("found role", "role", role)
+
+	if conf.Debug {
+		// slog.Debug doesn't print it, for whatever reason
+		repr.Println(record)
+	}
+
+	groups, err := getRoleMappingGroups(conf, rolename)
+	if err != nil {
+		return err
+	}
+	slog.Debug("group mappings", "groups", groups)
+
+	// check cluster setting
+	clusters := []string{}
+	for _, cluster := range role.Cluster {
+		clusters = append(clusters, cluster.Name)
+	}
+
+	// check index names+privs
+	indices := []string{}
+	privs := []string{}
+	for _, index := range role.Indices {
+		for _, name := range index.Names {
+			indices = append(indices, strings.ReplaceAll(name, "**", "*"))
+		}
+
+		for _, priv := range index.Privileges {
+			privs = append(privs, priv.Name)
+		}
+	}
+
+	// check kibana application space
+	spaces := []string{}
+	for _, app := range role.Applications {
+		for _, resource := range app.Resources {
+			if strings.Contains(resource, "space:") {
+				parts := strings.Split(resource, ":")
+				if len(parts) == 2 {
+					spaces = append(spaces, parts[1])
+				}
+			}
+		}
+	}
+
+	compareSlices("cluster_privilege", clusters, record.cluster_privilege)
+	compareSlices("ad_group", groups, record.ad_group)
+	compareSlices("index_name", indices, []string{record.index_name})
+	compareSlices("index_privilege", privs, []string{record.index_privilege})
+	compareSlices("space", spaces, []string{record.space})
+
+	return nil
+}

pkg/es/search.go

@@ -21,7 +21,6 @@ import (
 	"fmt"
 	"log"
 	"log/slog"
-	"strings"
 	"time"
 
 	"codeberg.org/scip/esctl/pkg/cfg"
@@ -56,18 +55,18 @@ func Search(conf *cfg.Config, queries []string) error {
 
 	searchEs.Request(req)
 
+	searchEs = addSort(conf, searchEs)
+
 	switch conf.Tail {
 	case true:
 		return searchTail(conf, searchEs)
-	case false:
+	default:
 		if conf.To > MAXPAGE {
 			return searchPit(conf, req)
 		} else {
 			return searchOnce(conf, searchEs)
 		}
 	}
-
-	return nil
 }
 
 func Debug(conf *cfg.Config) error {
@@ -95,11 +94,7 @@ func searchOnce(conf *cfg.Config, search *search.Search) error {
 		Size(conf.To).
 		Do(context.Background())
 	if err != nil {
-		if strings.Contains(err.Error(), "reason: all shards failed") {
+		return fmt.Errorf("failed to run search (esdsl): %s", esErrorString(err))
-			return nil
-		}
-
-		return fmt.Errorf("failed to run search (esdsl): %s", err)
 	}
 
 	slog.Debug("ES result", "search", res)
@@ -134,14 +129,12 @@ func searchPit(conf *cfg.Config, req *search.Request) error {
 			AddSortOption("_shard_doc", esdsl.NewFieldSort(sortorder.Asc))).
 		Size(conf.To)
 
+	search = addSort(conf, search)
+
 	for {
 		res, err := search.Do(ctx)
 		if err != nil {
-			if strings.Contains(err.Error(), "reason: all shards failed") {
+			return fmt.Errorf("failed to run search (esdsl pit): %s", esErrorString(err))
-				return nil
-			}
-
-			return fmt.Errorf("failed to run search (esdsl pit): %s", err)
 		}
 
 		if len(res.Hits.Hits) == 0 {
@@ -173,11 +166,7 @@ func searchTail(conf *cfg.Config, search *search.Search) error {
 	for {
 		res, err := search.Do(context.Background())
 		if err != nil {
-			if strings.Contains(err.Error(), "reason: all shards failed") {
+			return fmt.Errorf("failed to run search (esdsl): %s", esErrorString(err))
-				return nil
-			}
-
-			return fmt.Errorf("failed to run search (esdsl): %s", err)
 		}
 
 		slog.Debug("ES result", "search", res)

pkg/es/search_filter.go

@@ -17,15 +17,18 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
 package es
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"log/slog"
 	"strings"
 
 	"codeberg.org/scip/esctl/pkg/cfg"
+	"github.com/elastic/go-elasticsearch/v9/typedapi/core/search"
 	"github.com/elastic/go-elasticsearch/v9/typedapi/esdsl"
 	"github.com/elastic/go-elasticsearch/v9/typedapi/types"
 	"github.com/elastic/go-elasticsearch/v9/typedapi/types/enums/operator"
+	"github.com/elastic/go-elasticsearch/v9/typedapi/types/enums/sortorder"
 )
 
 const (
@@ -236,3 +239,42 @@ func addFilters(conf *cfg.Config) ([]types.QueryVariant, error) {
 
 	return filters, nil
 }
+
+// check if the conf.SortBy  field (default: @timestamp) is searchable
+// by using the field capabilities API.
+func addSort(conf *cfg.Config, search *search.Search) *search.Search {
+	res, err := conf.DefaultCluster.ES.FieldCaps().
+		Index(conf.Index).
+		Fields(conf.SortBy).
+		Do(context.Background())
+
+	if err != nil {
+		// whatever it was, do not add Sort()
+		slog.Debug("get field capabilities", "field", conf.SortBy, "error", esErrorString(err))
+
+		return search
+	}
+
+	field, exists := res.Fields[conf.SortBy]
+
+	if exists {
+		// good, the field exists
+		for _, cap := range field {
+			if cap.Searchable {
+				// ok, ES would be willing to sort by this field
+				order := esdsl.NewFieldSort(sortorder.Desc)
+				if conf.Ascending {
+					order = esdsl.NewFieldSort(sortorder.Asc)
+				}
+
+				search = search.Sort(
+					esdsl.NewSortOptions().AddSortOption(conf.SortBy, order),
+				)
+
+				break
+			}
+		}
+	}
+
+	return search
+}

pkg/es/shard.go

@@ -88,7 +88,7 @@ func ShardList(conf *cfg.Config) error {
 		Header("accept", "application/json").
 		Do(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to get shards: %s", err)
+		return fmt.Errorf("failed to get shards: %s", esErrorString(err))
 	}
 
 	shardlist := filterShards(conf, res)
@@ -140,7 +140,7 @@ func ShardShow(conf *cfg.Config, index string) error {
 		Header("accept", "application/json").
 		Do(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to get shards: %s", err)
+		return fmt.Errorf("failed to get shards: %s", esErrorString(err))
 	}
 
 	slog.Debug("ES result", "shards", res)

pkg/es/snapshot.go

@@ -25,7 +25,7 @@ import (
 	"strings"
 
 	"codeberg.org/scip/esctl/pkg/cfg"
-    "codeberg.org/scip/esctl/pkg/printer"
+	"codeberg.org/scip/esctl/pkg/printer"
 )
 
 var (
@@ -45,7 +45,7 @@ func SnapshotList(conf *cfg.Config) error {
 	// get partial indicies
 	ires, err := conf.DefaultCluster.ES.Cat.Indices().Do(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to get indicies: %s", err)
+		return fmt.Errorf("failed to get indicies: %s", esErrorString(err))
 	}
 
 	indicies := map[string]int{}
@@ -58,7 +58,7 @@ func SnapshotList(conf *cfg.Config) error {
 	// get snapshots
 	sres, err := conf.DefaultCluster.ES.Cat.Snapshots().Do(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to get snapshots: %s", err)
+		return fmt.Errorf("failed to get snapshots: %s", esErrorString(err))
 	}
 
 	slog.Debug("ES result", "indicies", sres)
@@ -108,7 +108,7 @@ func SnapshotList(conf *cfg.Config) error {
 func SnapshotShow(conf *cfg.Config, snapshot string) error {
 	res, err := conf.DefaultCluster.ES.Snapshot.Get("*", snapshot).Do(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to get snapshot: %s", err)
+		return fmt.Errorf("failed to get snapshot: %s", esErrorString(err))
 	}
 
 	slog.Debug("ES result", "snapshot", res)