bump version

fixes #36

.gitignore

@@ -32,3 +32,6 @@ esctl
 *.log
 
 cpu.profile
+
+t
+single

Makefile

@@ -62,9 +62,8 @@ install: buildlocal
 clean:
 	rm -rf $(tool) coverage.out testdata t/out pkg/es/openspec.go
 
-test: clean
+test: clean buildlocal
-	mkdir -p t/out
+	make -C t test
-	go test ./... $(ARGS)
 
 testlint: test lint
 

README.md

@@ -23,6 +23,8 @@ Features:
 - Shell completion support (bash, zsh and fish). Put this into your
   rc: `source <(esctl completion bash)`.
 - Cluster settings can be viewed and modified.
+- Comprehensive cluster status.
+- Cluster reroute support.
 - Search: you can search indices using full text or by fields, select
   logical condition (OR, AND), use PIT, limit datetime (ES date math
   can be used), etc. It is however not yet possible to create
@@ -47,7 +49,9 @@ Features:
   elasticsearch API.  You can  run API calls  on the  current selected
   cluster w/o the hassle to specify the whole url, credentials etc. It
   has line  editing and history  support. If `jq` is  installed output
-  JSON will be syntax highlighted.
+  JSON will  be syntax  highlighted. A simple  internal pager  will be
+  used if output exceeds the terminal height. You can tweak this using
+  the `$ES_JSON_PAGER` environment variable (I'd recommend [fx](https://fx.wtf/)).
 - Doc support. You can put, delete and show docs for an index. Very
   handy if you want to play with it. Just create a new index:
   `esctl index create foo` and then insert docs into it for search
@@ -113,6 +117,7 @@ Configure `esctl` with environment variables:
 - `ES_URI`: elasticsearch uri
 - `ES_USER`: username
 - `ES_PASS`: password
+- `ES_TOKEN`: API token, instead of user+password
 
 Or create a config file such as this:
 
@@ -121,11 +126,10 @@ clusters:
   foobar:
     uri: https://es.foo.bar:9200/
     user: elastic
-    pass: 123456
+    pass: ******
   other:
     uri: https://myes.foo:9200/
-    user: elastic
+    token: ******
-    pass: asdasdasd
 ```
 
 and specify it with `-c configfile`.  You may also put clusters into a
@@ -433,6 +437,40 @@ $ esctl search -i hyperdrive -l 2 | jq
 }
 ```
 
+There are also some uniq features which are not directly available via
+the ES API or GUI. Here's one example: say you have a number of
+indices with ILM policies each. At some day there were a surge in
+incoming data in some of them and you want to know, when the excess
+data will be rolled over to cold storage:
+
+```console
+$ esctl ilm forecast sh -f warm -w 3d 
+1.1 TB bytes of data in warm phase will be rolled within 72h0m0s to the next phase
+```
+
+You may also look at a detailed ilm forecast list:
+
+```console
+$ esctl ilm forecast ls -f warm -w 3d
+INDEX                           CURRENT-SIZE  CURRENT-AGE  VIRTUAL-AGE  MIN-AGE  MIN-SIZE  CURRENT-PHASE  NEXT-PHASE
+foobar-n1-p01-elastic-000126    12 GB         2d:8h:12m    3d:5h:16m    7d       25 GB     hot            warm      
+foobar-n1-p01-misc-000070       16 GB         5d:16h:31m   4d:10h:31m   7d       25 GB     hot            warm      
+foobar-n1-q01-elastic-000122    23 GB         5d:0h:11m    6d:10h:33m   7d       25 GB     hot            warm      
+foobar-n1-q01-misc-000072       18 GB         6d:15h:21m   4d:21h:36m   7d       25 GB     hot            warm      
+delaware-f1-p01-kafka-000023    1.3 MB        15h:12m      0s           7d       25 GB     hot            warm      
+delaware-f1-p01-misc-000024     17 GB         6d:16h:1m    4d:16h:33m   7d       25 GB     hot            warm      
+delaware-f1-q01-kafka-000023    1.5 MB        15h:12m      0s           7d       25 GB     hot            warm      
+delaware-f1-q01-misc-000024     16 GB         6d:15h:31m   4d:13h:52m   7d       25 GB     hot            warm      
+delaware-n1-p01-elastic-000417  16 GB         12h:42m      4d:10h:31m   7d       25 GB     hot            warm 
+```
+So you can see, which index will be rolled over when. Note the
+**VIRTUAL-AGE** field however: it is calculated from the current
+storage usage of the index in relation to rollover max shard size. So
+you can see, when an index will be rolled over either because it aged
+out or because its storage exceeded the limit.
+
+---
+
 Please note, that `esctl` is still in its early stages and things are
 changing heavily every now and then. New commands are being added
 constantly as well.
@@ -464,6 +502,12 @@ cluster               - manage cluster[s]
   settings                  - cluster settings management
     list                    - show cluster settings
     set                     - set|update cluster settings
+  reroute                   - manually change the allocation of individual shards in the cluster.
+    move                    - move shard to another node
+    allocate-replica        - allocate-replica replica to another node
+    cancel                  - cancel a reroute operation
+    allocate-empty-primary  - allocate an empty primary shard to a node
+    allocate-stale-primary  - allocate a stale primary shard to a node
 datastream                  - manage data streams
   list                      - list indicies
   show                      - show details about an data stream
@@ -479,15 +523,17 @@ ilm                   - manage index lifecycle
   status                    - get the current index lifecycle management status
   list                      - list index lifecycle policies
   show                      - show details about an index lifecycle policy
-  create              - create a index lifecycle policy
+  create                    - create a new lifecycle policy
+  update                    - update an lifecycle policy
   forecast                  - calculate index phase movements
     list                    - list index rollover config
     show                    - show rollover forecast over all indices
+  explain                   - explain ilm condition of an index
 index                       - manage indicies
   list                      - list indicies
   show                      - show details about an index
   create                    - create a new index
-  modify              - modify anindex
+  update                    - update an index
   delete                    - delete an index
   close                     - close an index
   allocation                - explain index allocation
@@ -502,11 +548,14 @@ index                 - manage indicies
     list                    - list index templates
     show                    - show details about an index template
     create                  - create a new index template
-    modify            - modify a new index template
+    update                  - update a new index template
     delete                  - delete an index template
+license                     - manage cluster license
+  show                      - show details about the cluster license
 node                        - manage nodes
   list                      - list nodes
   show                      - show details about a node
+  clients                   - show node http clients
 role                        - manage roles
   list                      - list roles
   show                      - show details about a role
@@ -524,11 +573,7 @@ task                  - manage tasks
 version                     - show esctl version information
 debug                       - developer only
 help-jsonpath               - show jsonpath help
-completion            - Output shell completion script for bash, zsh, fish, or Powershell
+help-usage                  - show overview of all available commands
-  pwsh                - Output pwsh completion script
-  bash                - Output bash completion script
-  zsh                 - Output zsh completion script
-  fish                - Output fish completion script
 ```
 
 # Development

cmd/api.go

@@ -83,6 +83,16 @@ func ApiRepl(conf *cfg.Config) *cli.Command {
 		Aliases: []string{"shell"},
 		Usage:   "interactive API repl",
 
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:        "pager",
+				Usage:       "external viewer program (default:internal)",
+				Destination: &conf.Pager,
+				Aliases:     []string{"p"},
+				Sources:     cli.EnvVars("PAGER", "ES_JSON_PAGER"),
+			},
+		},
+
 		Action: func(ctx context.Context, cmd *cli.Command) error {
 			return es.ApiRepl(conf)
 		},

cmd/cluster.go

@@ -37,6 +37,7 @@ func Cluster(conf *cfg.Config) *cli.Command {
 			ClusterSwitch(conf),
 			ClusterList(conf),
 			ClusterSettings(conf),
+			ClusterReroute(conf),
 		},
 	}
 }
@@ -60,12 +61,6 @@ func ClusterStatus(conf *cfg.Config) *cli.Command {
 		Aliases: []string{"s"},
 
 		Flags: []cli.Flag{
-			&cli.BoolFlag{
-				Name:        "all",
-				Usage:       "show status of all clusters",
-				Destination: &conf.All,
-				Aliases:     []string{"a"},
-			},
 			&cli.BoolFlag{
 				Name:        "verbose",
 				Usage:       "include verbose statistics",

cmd/cluster_reroute.go

@@ -0,0 +1,225 @@
+/*
+Copyright © 2026 Thomas von Dein
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+package cmd
+
+import (
+	"context"
+	"errors"
+
+	"codeberg.org/scip/esctl/pkg/cfg"
+	"codeberg.org/scip/esctl/pkg/es"
+
+	"github.com/urfave/cli/v3"
+)
+
+func ClusterReroute(conf *cfg.Config) *cli.Command {
+	return &cli.Command{
+		Name:      "reroute",
+		Usage:     "manually change the allocation of individual shards in the cluster.",
+		UsageText: "reroute <cluster-name>",
+		Aliases:   []string{"ctx"},
+
+		Commands: []*cli.Command{
+			ClusterRerouteMove(conf),
+			ClusterRerouteAllocateReplica(conf),
+			ClusterRerouteCancel(conf),
+			ClusterRerouteAllocatePrimary(conf, false),
+			ClusterRerouteAllocatePrimary(conf, true),
+		},
+	}
+}
+
+func ClusterRerouteMove(conf *cfg.Config) *cli.Command {
+	return &cli.Command{
+		Name:      "move",
+		Usage:     "move shard to another node",
+		UsageText: "move [options] <index>",
+
+		ShellComplete: func(ctx context.Context, cmd *cli.Command) {
+			complete(cmd, Cindex)
+		},
+
+		Flags: []cli.Flag{
+			&cli.IntFlag{
+				Name:        "shard",
+				Usage:       "shard to move",
+				Destination: &conf.Shards,
+				Aliases:     []string{"s"},
+				Required:    true,
+			},
+			&cli.StringFlag{
+				Name:        "from-node",
+				Usage:       "current node",
+				Destination: &conf.FromNode,
+				Aliases:     []string{"f"},
+				Required:    true,
+			},
+			&cli.StringFlag{
+				Name:        "to-node",
+				Usage:       "node to move to",
+				Destination: &conf.ToNode,
+				Aliases:     []string{"t"},
+				Required:    true,
+			},
+		},
+
+		Action: func(ctx context.Context, cmd *cli.Command) error {
+			index := cmd.Args().Get(0)
+			if index == "" {
+				return errors.New("no index specified")
+			}
+
+			return es.ClusterRerouteMove(conf, index)
+		},
+	}
+}
+
+func ClusterRerouteAllocateReplica(conf *cfg.Config) *cli.Command {
+	return &cli.Command{
+		Name:      "allocate-replica",
+		Usage:     "allocate-replica replica to another node",
+		UsageText: "allocate-replica [options] <index>",
+
+		ShellComplete: func(ctx context.Context, cmd *cli.Command) {
+			complete(cmd, Cindex)
+		},
+
+		Flags: []cli.Flag{
+			&cli.IntFlag{
+				Name:        "shard",
+				Usage:       "shard to move",
+				Destination: &conf.Shards,
+				Aliases:     []string{"s"},
+				Required:    true,
+			},
+			&cli.StringFlag{
+				Name:        "to-node",
+				Usage:       "node to move to",
+				Destination: &conf.ToNode,
+				Aliases:     []string{"t"},
+				Required:    true,
+			},
+		},
+
+		Action: func(ctx context.Context, cmd *cli.Command) error {
+			index := cmd.Args().Get(0)
+			if index == "" {
+				return errors.New("no index specified")
+			}
+
+			return es.ClusterRerouteAllocateReplica(conf, index)
+		},
+	}
+}
+
+func ClusterRerouteCancel(conf *cfg.Config) *cli.Command {
+	return &cli.Command{
+		Name:      "cancel",
+		Usage:     "cancel a reroute operation",
+		UsageText: "cancel [options] <index>",
+
+		ShellComplete: func(ctx context.Context, cmd *cli.Command) {
+			complete(cmd, Cindex)
+		},
+
+		Flags: []cli.Flag{
+			&cli.IntFlag{
+				Name:        "shard",
+				Usage:       "shard to move",
+				Destination: &conf.Shards,
+				Aliases:     []string{"s"},
+				Required:    true,
+			},
+			&cli.StringFlag{
+				Name:        "to-node",
+				Usage:       "node to move to",
+				Destination: &conf.ToNode,
+				Aliases:     []string{"t"},
+				Required:    true,
+			},
+			&cli.BoolFlag{
+				Name:        "allow-primary",
+				Usage:       "allow primary shard to cancel",
+				Destination: &conf.AllowPrimary,
+				Aliases:     []string{"a"},
+			},
+		},
+
+		Action: func(ctx context.Context, cmd *cli.Command) error {
+			index := cmd.Args().Get(0)
+			if index == "" {
+				return errors.New("no index specified")
+			}
+
+			return es.ClusterRerouteCancel(conf, index)
+		},
+	}
+}
+
+func ClusterRerouteAllocatePrimary(conf *cfg.Config, stale bool) *cli.Command {
+	name := "allocate-empty-primary"
+	usage := "allocate an empty primary shard to a node"
+
+	if stale {
+		name = "allocate-stale-primary"
+		usage = "allocate a stale primary shard to a node"
+
+	}
+
+	return &cli.Command{
+		Name:      name,
+		Usage:     usage,
+		UsageText: name + " [options] <index>",
+
+		ShellComplete: func(ctx context.Context, cmd *cli.Command) {
+			complete(cmd, Cindex)
+		},
+
+		Flags: []cli.Flag{
+			&cli.IntFlag{
+				Name:        "shard",
+				Usage:       "shard to move",
+				Destination: &conf.Shards,
+				Aliases:     []string{"s"},
+				Required:    true,
+			},
+			&cli.StringFlag{
+				Name:        "to-node",
+				Usage:       "node to move to",
+				Destination: &conf.ToNode,
+				Aliases:     []string{"t"},
+				Required:    true,
+			},
+			&cli.BoolFlag{
+				Name:        "accept-data-loss",
+				Usage:       "",
+				Destination: &conf.AcceptDataLoss,
+				Aliases:     []string{"a"},
+				Required:    true,
+			},
+		},
+
+		Action: func(ctx context.Context, cmd *cli.Command) error {
+			index := cmd.Args().Get(0)
+			if index == "" {
+				return errors.New("no index specified")
+			}
+
+			return es.ClusterRerouteAllocatePrimary(conf, index, stale)
+		},
+	}
+}

cmd/cluster_settings.go

@@ -104,6 +104,10 @@ func ClusterSettingsSet(conf *cfg.Config) *cli.Command {
 			},
 		},
 
+		ShellComplete: func(ctx context.Context, cmd *cli.Command) {
+			complete(cmd, Cclustersettings)
+		},
+
 		Action: func(ctx context.Context, cmd *cli.Command) error {
 			args := cmd.Args()
 

cmd/completion.go

@@ -32,6 +32,8 @@ const (
 	Ccluster
 	Capi
 	Cilm
+	Cnode
+	Cclustersettings
 )
 
 func complete(cmd *cli.Command, what int) {
@@ -64,6 +66,10 @@ func complete(cmd *cli.Command, what int) {
 		list = es.ApiPathNames()
 	case Cilm:
 		list, err = es.IlmNames(conf)
+	case Cnode:
+		list, err = es.NodeNames(conf)
+	case Cclustersettings:
+		list, err = es.ClusterSettingsNames(conf)
 	}
 
 	if err != nil {

cmd/ilm.go

@@ -36,7 +36,8 @@ func Ilm(conf *cfg.Config) *cli.Command {
 			IlmStatus(conf),
 			IlmList(conf),
 			IlmShow(conf),
-			IlmCreate(conf),
+			IlmCreate(conf, false),
+			IlmCreate(conf, true),
 			IlmForecast(conf),
 			IlmExplain(conf),
 		},
@@ -103,6 +104,15 @@ func IlmShow(conf *cfg.Config) *cli.Command {
 		Usage:     "show details about an index lifecycle policy",
 		UsageText: "show <policy>",
 
+		Flags: []cli.Flag{
+			&cli.BoolFlag{
+				Name:        "tree",
+				Usage:       "display policy as a tree",
+				Destination: &conf.Ilm.Tree,
+				Aliases:     []string{"t"},
+			},
+		},
+
 		Action: func(ctx context.Context, cmd *cli.Command) error {
 			policy := cmd.Args().Get(0)
 			if policy == "" {
@@ -118,12 +128,22 @@ func IlmShow(conf *cfg.Config) *cli.Command {
 	}
 }
 
-func IlmCreate(conf *cfg.Config) *cli.Command {
+func IlmCreate(conf *cfg.Config, modify bool) *cli.Command {
+	name := "create"
+	alias := "+"
+	usage := "create a new lifecycle policy"
+
+	if modify {
+		name = "update"
+		usage = "update an lifecycle policy"
+		alias = "upd"
+	}
+
 	return &cli.Command{
-		Name:      "create",
+		Name:      name,
-		Aliases:   []string{"+"},
+		Aliases:   []string{alias},
-		Usage:     "create a index lifecycle policy",
+		Usage:     usage,
-		UsageText: "create [options] <policy>",
+		UsageText: name + " [options] <policy>",
 
 		Flags: []cli.Flag{
 			&cli.StringFlag{

cmd/index.go

@@ -154,8 +154,8 @@ func IndexCreate(conf *cfg.Config, modify bool) *cli.Command {
 	usage := "create a new index"
 
 	if modify {
-		name = "modify"
+		name = "update"
-		usage = "modify anindex"
+		usage = "update an index"
 	}
 
 	return &cli.Command{

cmd/index_template.go

@@ -83,8 +83,8 @@ func IndexTemplateCreate(conf *cfg.Config, modify bool) *cli.Command {
 	required := true
 
 	if modify {
-		name = "modify"
+		name = "update"
-		alias = "mod"
+		alias = "upd"
 		required = false
 	}
 

cmd/license.go

@@ -0,0 +1,49 @@
+/*
+Copyright © 2026 Thomas von Dein
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+package cmd
+
+import (
+	"context"
+
+	"codeberg.org/scip/esctl/pkg/cfg"
+	"codeberg.org/scip/esctl/pkg/es"
+
+	"github.com/urfave/cli/v3"
+)
+
+func License(conf *cfg.Config) *cli.Command {
+	return &cli.Command{
+		Name:  "license",
+		Usage: "manage cluster license",
+
+		Commands: []*cli.Command{
+			LicenseShow(conf),
+		},
+	}
+}
+
+func LicenseShow(conf *cfg.Config) *cli.Command {
+	return &cli.Command{
+		Name:    "show",
+		Aliases: []string{"sh"},
+		Usage:   "show details about the cluster license",
+
+		Action: func(ctx context.Context, cmd *cli.Command) error {
+			return es.LicenseShow(conf)
+		},
+	}
+}

cmd/node.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"context"
+	"errors"
 
 	"codeberg.org/scip/esctl/pkg/cfg"
 	"codeberg.org/scip/esctl/pkg/es"
@@ -34,6 +35,7 @@ func Node(conf *cfg.Config) *cli.Command {
 		Commands: []*cli.Command{
 			NodeList(conf),
 			NodeShow(conf),
+			NodeClients(conf),
 		},
 	}
 }
@@ -57,10 +59,47 @@ func NodeShow(conf *cfg.Config) *cli.Command {
 		Usage:     "show details about a node",
 		UsageText: "show [options] <node>",
 
+		ShellComplete: func(ctx context.Context, cmd *cli.Command) {
+			complete(cmd, Cnode)
+		},
+
 		Action: func(ctx context.Context, cmd *cli.Command) error {
-			// FIXME: implement es.NodeShow()
+			node := cmd.Args().Get(0)
-			// return es.NodeShow(conf, cmd.Args().Get(0))
+			if node == "" {
-			return nil
+				return errors.New("no node specified")
+			}
+
+			return es.NodeShow(conf, node)
+		},
+	}
+}
+
+func NodeClients(conf *cfg.Config) *cli.Command {
+	return &cli.Command{
+		Name:      "clients",
+		Usage:     "show node http clients",
+		UsageText: "clients [options] <node>",
+
+		Flags: []cli.Flag{
+			&cli.BoolFlag{
+				Name:        "query",
+				Usage:       "include query parameters",
+				Destination: &conf.All,
+				Aliases:     []string{"q"},
+			},
+		},
+
+		ShellComplete: func(ctx context.Context, cmd *cli.Command) {
+			complete(cmd, Cnode)
+		},
+
+		Action: func(ctx context.Context, cmd *cli.Command) error {
+			node := cmd.Args().Get(0)
+			if node == "" {
+				return errors.New("no node specified")
+			}
+
+			return es.NodeClients(conf, node)
 		},
 	}
 }

cmd/root.go

@@ -42,7 +42,6 @@ func Finish(err error) int {
 
 func Main() int {
 	conf := cfg.NewConfig()
-	tree := false
 
 	cmd := &cli.Command{
 		Name:                  "esctl",
@@ -64,13 +63,6 @@ func Main() int {
 				Usage:       "enable HTTP debugging",
 				Destination: &conf.DebugHTTP,
 			},
-			&cli.BoolFlag{
-				Name:        "show-command-tree",
-				Value:       false,
-				Usage:       "generate a command tree",
-				Destination: &tree,
-				Hidden:      true,
-			},
 			&cli.BoolFlag{
 				Name:        "align-ints",
 				Aliases:     []string{"I"},
@@ -116,6 +108,7 @@ func Main() int {
 			Doc(conf),
 			Ilm(conf),
 			Index(conf),
+			License(conf),
 			Node(conf),
 			Roles(conf),
 			Search(conf),
@@ -125,17 +118,10 @@ func Main() int {
 			Version(conf),
 			Debug(conf),
 			HelpJsonPath(conf),
+			HelpUsage(conf),
 		},
 
 		Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) {
-			if tree {
-				if err := Tree(cmd); err != nil {
-					return nil, err
-				}
-
-				os.Exit(0)
-			}
-
 			if err := conf.Init(); err != nil {
 				if len(os.Args) > 1 {
 					return nil, err
@@ -261,22 +247,109 @@ func Debug(conf *cfg.Config) *cli.Command {
 	}
 }
 
-func Tree(cmd *cli.Command) error {
+func HelpUsage(conf *cfg.Config) *cli.Command {
-	max := 20
+	return &cli.Command{
+		Name:               "help-usage",
+		Usage:              "show overview of all available commands",
+		UsageText:          "help-usage [<filter>]",
+		Aliases:            []string{"usage"},
+		CustomHelpTemplate: addReference(`<filter> implies -f`),
 
-	return cmd.Walk(func(cmd *cli.Command) error {
+		Flags: []cli.Flag{
+			&cli.BoolFlag{
+				Name:        "hidden",
+				Usage:       "include hidden commands",
+				Destination: &conf.Hidden,
+				Aliases:     []string{"H"},
+			},
+			&cli.BoolFlag{
+				Name:        "full-commands",
+				Usage:       "show full commands",
+				Destination: &conf.Force,
+				Aliases:     []string{"f"},
+			},
+		},
+
+		Action: func(ctx context.Context, cmd *cli.Command) error {
+			maxCommandWidth := 0
+			filter := cmd.Args().Get(0)
+
+			if filter != "" {
+				conf.Force = true
+			}
+
+			// first pass, determine max command width
+			if err := walkVisible(conf, cmd.Root(), func(cmd *cli.Command) error {
 				path := cmd.Path()
-		command := path[len(path)-1]
+				size := len(path[len(path)-1])
 
-		if len(path) == 1 || command == "help" {
+				if conf.Force {
+					path := strings.Join(cmd.Path(), " ")
+					size = len(path)
+				}
+
+				if size > maxCommandWidth {
+					maxCommandWidth = size
+				}
+
+				return nil
+			}); err != nil {
+				return err
+			}
+
+			maxCommandWidth += 4 // account for indent width
+
+			// second pass, build tree
+			return walkVisible(conf, cmd.Root(), func(cmd *cli.Command) error {
+				path := cmd.Path()
+
+				if filter != "" {
+					if !strings.Contains(strings.Join(path, " "), filter) {
+						return nil
+					}
+				}
+
+				command := path[len(path)-1]
+				if conf.Force {
+					command = strings.Join(cmd.Path(), " ")
+				}
+
+				if len(path) == 1 || strings.HasSuffix(command, "help") {
 					return nil
 				}
 
 				indent := strings.Repeat("  ", len(path[1:])-1)
-		space := strings.Repeat(" ", max-(len(command)+len(indent)))
+				space := strings.Repeat(" ", maxCommandWidth-(len(command)+len(indent)))
 
 				fmt.Printf("%s%s %s - %s\n", indent, command, space, cmd.Usage)
 
 				return nil
 			})
+		},
+	}
+
+}
+
+// copy of cmd.Walk() with the exception to skip hidden commands and its siblings
+// see: https://github.com/urfave/cli/issues/2372
+func walkVisible(conf *cfg.Config, cmd *cli.Command, fn func(*cli.Command) error) error {
+	if fn == nil {
+		return nil
+	}
+
+	if !conf.Hidden && cmd.Hidden {
+		return nil
+	}
+
+	if err := fn(cmd); err != nil {
+		return err
+	}
+
+	for _, sub := range cmd.Commands {
+		if err := walkVisible(conf, sub, fn); err != nil {
+			return err
+		}
+	}
+
+	return nil
 }

go.mod

@@ -85,6 +85,7 @@ require (
 	github.com/olekukonko/errors v1.2.0 // indirect
 	github.com/olekukonko/ll v0.1.8 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/seeruk/go-wordwrap v0.0.0-20191208221741-14ec4aac9550 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect

go.sum

@@ -171,6 +171,8 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
 github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
+github.com/seeruk/go-wordwrap v0.0.0-20191208221741-14ec4aac9550 h1:C3CfUXH/qmWuQFRqnPm3Sx8PFxa+pqACjhV5CaNO8pw=
+github.com/seeruk/go-wordwrap v0.0.0-20191208221741-14ec4aac9550/go.mod h1:Sl541M2Em6rRG3V9WObycR7MYFZiERVkd/TJg0Gt0U4=
 github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

pkg/cfg/cluster.go

@@ -17,26 +17,32 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
 package cfg
 
 import (
+	"crypto/tls"
 	"errors"
 	"fmt"
+	"net"
 	"net/http"
 	"os"
+	"strings"
+	"syscall"
+	"time"
 
 	"github.com/elastic/elastic-transport-go/v8/elastictransport"
 	"github.com/elastic/go-elasticsearch/v9"
+	"golang.org/x/term"
 	"gopkg.in/yaml.v3"
 )
 
 // used in general config struct
 type Cluster struct {
-	Uri, User, Pass string
+	Name, Uri, User, Pass, Token string
 	client                       *elasticsearch.TypedClient
-	Default         bool
+	Default, DebugHTTP           bool
 }
 
 // used just for writing back to the config file
 type ClusterConfig struct {
-	Uri, User, Pass string
+	Uri, User, Pass, Token string
 	Default                bool
 }
 
@@ -45,17 +51,96 @@ type WriteConfig struct {
 	Clusters map[string]*ClusterConfig
 }
 
+func (cluster *Cluster) SetClient(client *elasticsearch.TypedClient) {
+	cluster.client = client
+}
+
+func (cluster *Cluster) getTransport() elastictransport.Option {
+	transport := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+
+	if cluster.DebugHTTP {
+		return elastictransport.WithTransport(
+			&DebugTransport{Transport: transport},
+		)
+	}
+
+	return elastictransport.WithTransport(transport)
+}
+
+func (cluster *Cluster) getDefaultOptions() []elasticsearch.Option {
+	// These headers are  not needed with ES 9, but with  ES 8, we set
+	// them here  so every API call uses it.  The only exception being
+	// the api repl, which does it on its own.
+	headers := http.Header{}
+	headers.Add("content-type", "application/json")
+	headers.Add("Accept", "application/json")
+
+	return []elasticsearch.Option{
+		elasticsearch.WithAddresses(cluster.Uri),
+		elasticsearch.WithTransportOptions(
+			cluster.getTransport(),
+			elastictransport.WithHeader(headers),
+		),
+	}
+}
+
+// return the  go-elasticsearch client  object but before  doing that,
+// check if we need to tune auth
 func (cluster *Cluster) ES() *elasticsearch.TypedClient {
 	if cluster.client == nil {
 		fmt.Println("no current cluster, use 'esctl cluster switch <name>' to set one")
 		os.Exit(1)
 	}
 
+	if err := cluster.CheckAuth(); err != nil {
+		fmt.Printf("Error: %s", err)
+		os.Exit(1)
+	}
+
 	return cluster.client
 }
 
-func (cluster *Cluster) SetClient(client *elasticsearch.TypedClient) {
+// add authentication to es client, if not yet done
-	cluster.client = client
+func (cluster *Cluster) CheckAuth() error {
+	if cluster.Pass == "" && cluster.User != "" && cluster.Token == "" && cluster.Default {
+		// no token - user is set, but no password.
+
+		// check if the env var is set
+		pass := os.Getenv("ES_PASS")
+		if pass != "" {
+			cluster.Pass = pass
+		} else {
+			// k, try interactively
+			fmt.Fprintf(os.Stderr, "Enter password for elasticsearch user %s@%s: ", cluster.User, cluster.Name)
+			pass, err := term.ReadPassword(int(syscall.Stdin))
+			if err != nil {
+				return err
+			}
+
+			passwd := strings.TrimSpace(string(pass))
+			if passwd == "" {
+				return errors.New("password empty")
+			}
+
+			cluster.Pass = string(pass)
+			fmt.Println()
+		}
+
+		opts := cluster.getDefaultOptions()
+		opts = append(opts, elasticsearch.WithBasicAuth(cluster.User, cluster.Pass))
+
+		es, err := elasticsearch.NewTyped(opts...)
+
+		if err != nil {
+			return fmt.Errorf("failed to setup elasticsearch connection: %w", err)
+		}
+
+		cluster.SetClient(es)
+	}
+
+	return nil
 }
 
 // set Default=true for the given cluster in the config (if exists)
@@ -72,6 +157,7 @@ func (conf *Config) SwitchCluster(name string) error {
 			Uri:     cluster.Uri,
 			User:    cluster.User,
 			Pass:    cluster.Pass,
+			Token:   cluster.Token,
 			Default: false,
 		}
 
@@ -97,23 +183,43 @@ func (conf *Config) SwitchCluster(name string) error {
 	return nil
 }
 
-func (conf *Config) SetupES() error {
+// We do NOT  use go-elasticsearch to check  for cluster reachability,
-	// These headers are  not needed with ES 9, but with  ES 8, we set
+// because  at this  stage,  auth  may not  have  been configured.  So
-	// them here  so every API call uses it.  The only exception being
+// instead we just connect to  the cluster using plan net/tcp
-	// the api repl, which does it on its own.
+func (cluster *Cluster) IsReachable() (bool, error) {
-	headers := http.Header{}
+	timeout := 500 * time.Millisecond
-	headers.Add("content-type", "application/json")
-	headers.Add("Accept", "application/json")
 
-	for _, cluster := range conf.Clusters {
+	url := strings.TrimPrefix(strings.TrimPrefix(cluster.Uri, "https://"), "http://")
-		es, err := elasticsearch.NewTyped(
+
-			elasticsearch.WithAddresses(cluster.Uri),
+	host := strings.Split(url, "/")
-			elasticsearch.WithBasicAuth(cluster.User, cluster.Pass),
+
-			elasticsearch.WithTransportOptions(
+	if !strings.Contains(host[0], ":") {
-				conf.getTransport(),
+		host[0] += ":443"
-				elastictransport.WithHeader(headers),
+	}
-			),
+
-		)
+	conn, err := net.DialTimeout("tcp", host[0], timeout)
+	if err != nil {
+		return false, err
+	}
+
+	return true, conn.Close()
+}
+
+func (conf *Config) SetupES() error {
+	for name, cluster := range conf.Clusters {
+		cluster.Name = name
+		cluster.DebugHTTP = conf.DebugHTTP
+
+		opts := cluster.getDefaultOptions()
+
+		switch {
+		case cluster.Pass != "" && cluster.User != "":
+			opts = append(opts, elasticsearch.WithBasicAuth(cluster.User, cluster.Pass))
+		case cluster.Token != "":
+			opts = append(opts, elasticsearch.WithAPIKey(cluster.Token))
+		}
+
+		es, err := elasticsearch.NewTyped(opts...)
 
 		if err != nil {
 			return fmt.Errorf("failed to setup elasticsearch connection: %w", err)

pkg/cfg/config.go

@@ -28,7 +28,7 @@ import (
 )
 
 const (
-	Version string = `v0.0.22`
+	Version string = `v0.0.26`
 )
 
 var (
@@ -103,6 +103,11 @@ type Config struct {
 	Tag string // api ls: -t
 
 	Ilm Ilm // ilm create
+
+	FromNode, ToNode             string // cluster reroute move: -f + -t
+	AllowPrimary, AcceptDataLoss bool   // cluster reroute cancel: -p,-a
+
+	Pager string // api repl: -p || PAGER
 }
 
 func NewConfig() *Config {
@@ -116,10 +121,11 @@ func getDefaultPath() string {
 func (conf *Config) Init() error {
 	DefaultConfig := getDefaultPath()
 
-	switch {
+	if conf.ConfigFile == "" && fileExists(DefaultConfig) {
-	case fileExists(DefaultConfig):
 		conf.ConfigFile = DefaultConfig
-		fallthrough
+	}
+
+	switch {
 	case conf.ConfigFile != "":
 		if err := conf.LoadConfig(); err != nil {
 			return err
@@ -134,6 +140,18 @@ func (conf *Config) Init() error {
 		return err
 	}
 
+	if err := conf.determineDefaultCluster(); err != nil {
+		return err
+	}
+
+	conf.HaveJQ = isJQinstalled()
+
+	conf.PrintDebug()
+
+	return nil
+}
+
+func (conf *Config) determineDefaultCluster() error {
 	if conf.CurrentCluster != "" {
 		// -C specified, set current cluster explicitly, no matter what the config says
 		current, exists := conf.Clusters[conf.CurrentCluster]
@@ -169,10 +187,6 @@ func (conf *Config) Init() error {
 		}
 	}
 
-	conf.HaveJQ = isJQinstalled()
-
-	conf.PrintDebug()
-
 	return nil
 }
 
@@ -213,15 +227,14 @@ func (conf *Config) LoadEnv() error {
 		Uri:   os.Getenv("ES_URI"),
 		User:  os.Getenv("ES_USER"),
 		Pass:  os.Getenv("ES_PASS"),
+		Token: os.Getenv("ES_TOKEN"),
 	}
 
 	switch {
 	case cluster.Uri == "":
 		return errors.New("ES_URI unset")
-	case cluster.User == "":
+	case cluster.User == "" || cluster.Token == "":
-		return errors.New("ES_USER unset")
+		return errors.New("ES_USER and ES_TOKEN unset")
-	case cluster.Pass == "":
-		return errors.New("ES_PASS unset")
 	}
 
 	conf.Clusters["default"] = &cluster

pkg/cfg/ilm.go

@@ -42,6 +42,7 @@ type Ilm struct {
 	DeleteSearchableSnapshots bool
 
 	MinAge, FromPhase, Within string // forecast
+	Tree                      bool
 }
 
 func (cfg *Ilm) HaveHot() bool {

pkg/cfg/term.go

@@ -0,0 +1,49 @@
+/*
+Copyright © 2026 Thomas von Dein
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+package cfg
+
+import (
+	"os"
+
+	"golang.org/x/term"
+)
+
+const (
+	DefaultMargin = 4
+)
+
+func GetTermWidth() int {
+	if term.IsTerminal(int(os.Stdout.Fd())) {
+		width, _, err := term.GetSize(int(os.Stdout.Fd()))
+		if err == nil {
+			return width - DefaultMargin
+		}
+	}
+
+	return 80
+}
+
+func GetTermHeight() int {
+	if term.IsTerminal(int(os.Stdout.Fd())) {
+		_, height, err := term.GetSize(int(os.Stdout.Fd()))
+		if err == nil {
+			return height
+		}
+	}
+
+	return 25
+}

pkg/cfg/transport.go

@@ -18,13 +18,10 @@ package cfg
 
 import (
 	"bytes"
-	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"log/slog"
 	"net/http"
-
-	"github.com/elastic/elastic-transport-go/v8/elastictransport"
 )
 
 // used to print uri, path and body of a request made by the go-client
@@ -62,17 +59,3 @@ func (t *DebugTransport) RoundTrip(req *http.Request) (*http.Response, error) {
 
 	return t.Transport.RoundTrip(req)
 }
-
-func (conf *Config) getTransport() elastictransport.Option {
-	transport := &http.Transport{
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-	}
-
-	if conf.DebugHTTP {
-		return elastictransport.WithTransport(
-			&DebugTransport{Transport: transport},
-		)
-	}
-
-	return elastictransport.WithTransport(transport)
-}

pkg/es/api.go

@@ -41,11 +41,9 @@ import (
 	"github.com/charmbracelet/lipgloss"
 	"github.com/chzyer/readline"
 	"github.com/go-openapi/spec"
-	"golang.org/x/term"
 )
 
 const (
-	DefaultMargin = 4
 	intro = `Input format: verb path [data]"
 
 Example:
@@ -143,14 +141,54 @@ func ApiRepl(conf *cfg.Config) error {
 			fmt.Printf("failed to call API: %s\n", esErrorString(err))
 		}
 
-		if err := prettyfiJson(conf, raw); err != nil {
+		pageJsonOutput(conf, raw)
-			fmt.Println(err)
-		}
 	}
 
 	return nil
 }
 
+func pageJsonOutput(conf *cfg.Config, raw []byte) {
+	tmpconf := &cfg.Config{HaveJQ: conf.HaveJQ}
+
+	if conf.Pager != "" {
+		tmpconf.HaveJQ = false
+	}
+
+	output, err := prettyfiJson(tmpconf, raw)
+	if err != nil {
+		fmt.Println(err)
+	}
+
+	lines := len(strings.Split(output, "\n"))
+	height := cfg.GetTermHeight()
+
+	if lines > height {
+		if conf.Pager != "" {
+			cmd := strings.Split(conf.Pager, " ")
+			pager := exec.Command(cmd[0], cmd[1:]...)
+
+			var buf bytes.Buffer
+			buf.WriteString(output)
+
+			pager.Stdout = os.Stdout
+			pager.Stdin = &buf
+			pager.Stderr = os.Stderr
+
+			err := pager.Run()
+
+			if err != nil {
+				fmt.Printf("failed to execute pager '%s': %s", conf.Pager, err)
+			}
+		} else {
+			printer.Pager("json output", output)
+		}
+
+		return
+	}
+
+	fmt.Println(output)
+}
+
 func encodeAuth(username, password string) string {
 	return base64.StdEncoding.EncodeToString([]byte(username + ":" + password))
 }
@@ -165,6 +203,12 @@ func CallAPI(conf *cfg.Config, verb, path, data string) ([]byte, error) {
 
 	client := &http.Client{Transport: tr}
 
+	if conf.DebugHTTP {
+		client = &http.Client{
+			Transport: &cfg.DebugTransport{
+				Transport: tr}}
+	}
+
 	req, err := http.NewRequest(verb, conf.DefaultCluster.Uri+path, bytes.NewBuffer([]byte(data)))
 	if err != nil {
 		return nil, err
@@ -172,7 +216,17 @@ func CallAPI(conf *cfg.Config, verb, path, data string) ([]byte, error) {
 
 	req.Header.Add("Content-Type", "application/json")
 	req.Header.Add("accept", "application/json")
+
+	// make sure we have got all we need
+	if err := conf.DefaultCluster.CheckAuth(); err != nil {
+		return nil, err
+	}
+
+	if conf.DefaultCluster.Token != "" {
+		req.Header.Add("Authorization", "APIKey "+conf.DefaultCluster.Token)
+	} else {
 		req.Header.Add("Authorization", "Basic "+encodeAuth(conf.DefaultCluster.User, conf.DefaultCluster.Pass))
+	}
 
 	// actually execute the request
 	resp, err := client.Do(req)
@@ -189,7 +243,7 @@ func CallAPI(conf *cfg.Config, verb, path, data string) ([]byte, error) {
 	return body, nil
 }
 
-func prettyfiJson(conf *cfg.Config, raw []byte) error {
+func prettyfiJson(conf *cfg.Config, raw []byte) (string, error) {
 	if conf.HaveJQ {
 		cmd := exec.CommandContext(context.Background(), "jq", "-C")
 		cmd.Stdin = bytes.NewReader(raw)
@@ -199,23 +253,18 @@ func prettyfiJson(conf *cfg.Config, raw []byte) error {
 
 		err := cmd.Run()
 		if err != nil {
-			return err
+			return "", err
 		}
 
-		fmt.Println(out.String())
+		return out.String(), nil
-
-		return nil
 	}
-
 	var pretty bytes.Buffer
 	err := json.Indent(&pretty, raw, "", "\t")
 	if err != nil {
-		return fmt.Errorf("json parse error: %s", err)
+		return "", fmt.Errorf("json parse error: %s", err)
 	}
 
-	fmt.Println(pretty.String())
+	return pretty.String(), nil
-
-	return nil
 }
 
 // interactively  read  arbitrary  JSON  data  from  STDIN,  which  is
@@ -318,19 +367,19 @@ func ApiShow(conf *cfg.Config, showpath, verb string) error {
 
 	cleanMarkup := regexp.MustCompile(`<[^<>]+>`)
 
-	width := getTermWidth()
+	width := cfg.GetTermWidth()
 	params := getApiParameters(op, showpath, width)
 	sample := getApiExample(op)
-	description := markdown.Render(cleanMarkup.ReplaceAllString(op.Op.Description, ""), width, DefaultMargin)
+	description := markdown.Render(cleanMarkup.ReplaceAllString(op.Op.Description, ""), width, cfg.DefaultMargin)
 
 	var bold = lipgloss.NewStyle().
 		Bold(true)
 	var paragraph = lipgloss.NewStyle().
 		MarginBottom(1).
-		MarginLeft(DefaultMargin)
+		MarginLeft(cfg.DefaultMargin)
 	var boldparagraph = lipgloss.NewStyle().
 		MarginBottom(1).
-		MarginLeft(DefaultMargin).
+		MarginLeft(cfg.DefaultMargin).
 		Bold(true)
 	var indentparagraph = lipgloss.NewStyle().
 		MarginBottom(1).
@@ -413,7 +462,7 @@ func getApiParameters(op *Op, path string, width int) Params {
 			}
 		} else {
 			par := Param{
-				Description: string(markdown.Render(param.Description, width, DefaultMargin)),
+				Description: string(markdown.Render(param.Description, width, cfg.DefaultMargin)),
 				Param:       param.Name,
 			}
 
@@ -517,14 +566,3 @@ func findOperation(item spec.PathItemProps) []*Op {
 
 	return ops
 }
-
-func getTermWidth() int {
-	if term.IsTerminal(int(os.Stdout.Fd())) {
-		width, _, err := term.GetSize(int(os.Stdout.Fd()))
-		if err == nil {
-			return width - DefaultMargin
-		}
-	}
-
-	return 80
-}

pkg/es/ccr.go

@@ -102,7 +102,7 @@ func CcrRemoteInfo(conf *cfg.Config, index string) error {
 	}
 
 	if remote == "" {
-		return fmt.Errorf("cluster doesn't follow any other: %s", err)
+		return errors.New("cluster doesn't follow any other")
 	}
 
 	mode := "follower"

pkg/es/cluser_health_report.go

@@ -0,0 +1,59 @@
+/*
+Copyright © 2026 Thomas von Dein
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+package es
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"codeberg.org/scip/esctl/pkg/cfg"
+)
+
+type HealthReport struct {
+	Indicators map[string]HealthReportIndicator
+}
+
+type HealthReportIndicator struct {
+	Status, Symptom string
+	Diagnosis       []HealthReportDiagnosis
+}
+
+type HealthReportDiagnosis struct {
+	Id, Cause, Action string
+	AffectedResources map[string][]string `json:"affected_resources"`
+}
+
+// we do not use the go-elasticsearch  client API here but call the ES
+// API    directly,     because    the    returned     structure    (a
+// healthreport.Response) is  not iterable,  you'd have  to explicitly
+// call  every indicator  type and  every  cause etc  which also  have
+// different types  each. To check which  is !green would result  in a
+// gigantic function.
+func getHealthReport(conf *cfg.Config) (*HealthReport, error) {
+	raw, err := CallAPI(conf, "GET", "/_health_report", "")
+	if err != nil {
+		return nil, err
+	}
+
+	report := HealthReport{}
+
+	if err := json.Unmarshal(raw, &report); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal healthreport response: %w", err)
+	}
+
+	return &report, nil
+}

pkg/es/cluster.go

@@ -17,7 +17,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
 package es
 
 import (
-	"context"
+	"errors"
 	"fmt"
 	"log/slog"
 	"strings"
@@ -26,142 +26,154 @@ import (
 	"codeberg.org/scip/esctl/pkg/cfg"
 	"codeberg.org/scip/esctl/pkg/printer"
 	"github.com/dustin/go-humanize"
-	"github.com/elastic/go-elasticsearch/v9/typedapi/cat/indices"
-	"github.com/elastic/go-elasticsearch/v9/typedapi/cat/tasks"
-	"github.com/elastic/go-elasticsearch/v9/typedapi/ccr/stats"
-	"github.com/elastic/go-elasticsearch/v9/typedapi/cluster/health"
 	clusterstats "github.com/elastic/go-elasticsearch/v9/typedapi/cluster/stats"
-	"github.com/elastic/go-elasticsearch/v9/typedapi/core/info"
 	"github.com/elastic/go-elasticsearch/v9/typedapi/types"
 )
 
 type ClusterIndices map[string]map[string]*types.IndicesRecord
 
-func ClusterList(conf *cfg.Config) error {
+type clusterReachable struct {
-	table := printer.NewTable(conf, 4, len(conf.Clusters))
+	reachable bool
+	err       error
+}
 
-	table.Addheaders("cluster", "uri", "reachable", "current")
+func ClusterList(conf *cfg.Config) error {
+	var mu sync.Mutex
+	var wg sync.WaitGroup
+	reachable := make(map[string]clusterReachable, len(conf.Clusters))
+
+	// check endpoints in parallel to speed things up
+	for name, cluster := range conf.Clusters {
+		wg.Add(1)
+
+		go func() {
+			defer wg.Done()
+			online, err := cluster.IsReachable()
+
+			mu.Lock()
+			reachable[name] = clusterReachable{reachable: online, err: err}
+			mu.Unlock()
+		}()
+	}
+
+	wg.Wait()
+
+	table := printer.NewTable(conf, 5, len(conf.Clusters))
+	table.Addheaders("cluster", "uri", "reachable", "current", "error")
 
 	idx := 0
 
 	for name, cluster := range conf.Clusters {
-		reachable := "no"
+		reachableStr := "no"
 		current := "no"
+		errmsg := ""
 
-		_, err := cluster.ES().Cluster.Health().
+		if reachable[name].reachable {
-			Do(context.Background())
+			reachableStr = printer.Colorize(conf, "green", "reachable")
-
-		if err == nil {
-			reachable = printer.Colorize(conf, "green", "reachable")
 		}
 
 		if cluster.Default {
 			current = printer.Colorize(conf, "green", "yes")
 
-			if err != nil {
+			if !reachable[name].reachable {
-				reachable = printer.Colorize(conf, "red", "no")
+				reachableStr = printer.Colorize(conf, "red", "no")
+				errmsg = reachable[name].err.Error()
 			}
 		}
 
-		table.Entries[idx] = []string{name, cluster.Uri, reachable, current}
+		table.Entries[idx] = []string{name, cluster.Uri, reachableStr, current, errmsg}
 		idx++
 	}
 
 	table.Sort()
 
-	if err := table.Print(); err != nil {
+	return table.Print()
-		return err
-	}
-
-	return nil
 }
 
 // We're using goroutines  here to parallelize API  requests, since we
 // have to do 3 of'em for each cluster. This speeds things up.
-func ClusterStatus(conf *cfg.Config) error {
+func getClusterStatus(conf *cfg.Config) (*apiResponse, error) {
-	clusters := []string{}
+	gocount := 6
-	gocount := 5
 	if conf.Verbose {
 		gocount++
 	}
 
-	if conf.All {
-		for key := range conf.Clusters {
-			clusters = append(clusters, key)
-		}
-	} else {
-		clusters = []string{"default"}
-	}
-
-	for _, cluster := range clusters {
 	es := conf.DefaultCluster.ES()
-		if cluster != "default" {
-			es = conf.Clusters[cluster].ES()
-		}
 
 	responses := make(chan apiResponse, gocount)
 	wg := &sync.WaitGroup{}
 
 	wg.Add(gocount)
-		go getApiData(es, wg, responses, "health")
+	go getApiData(conf, es, wg, responses, "health")
-		go getApiData(es, wg, responses, "info")
+	go getApiData(conf, es, wg, responses, "healthreport")
-		go getApiData(es, wg, responses, "ccrstats")
+	go getApiData(conf, es, wg, responses, "info")
-		go getApiData(es, wg, responses, "indices")
+	go getApiData(conf, es, wg, responses, "ccr")
-		go getApiData(es, wg, responses, "tasks")
+	go getApiData(conf, es, wg, responses, "indices")
+	go getApiData(conf, es, wg, responses, "tasks")
 
 	if conf.Verbose {
-			go getApiData(es, wg, responses, "stats")
+		go getApiData(conf, es, wg, responses, "stats")
 	}
 
 	wg.Wait()
 
-		var clusterhealth *health.Response
+	all := apiResponse{}
-		var info *info.Response
+
-		var ccrstats *stats.Response
+	var err error
-		var clusterstats *clusterstats.Response
-		var indexstats *indices.Response
-		var taskstatus *tasks.Response
 
 	for i := 0; i < gocount; i++ {
 		r := <-responses
 
-			if r.error != nil {
+		err = errors.Join(err, r.error)
-				return r.error
-			}
 
 		switch r.which {
 		case ResponseHealth:
-				clusterhealth = r.health
+			all.health = r.health
 		case ResponseCcr:
-				ccrstats = r.ccr
+			all.ccr = r.ccr
 		case ResponseInfo:
-				info = r.info
+			all.info = r.info
 		case ResponseStats:
-				clusterstats = r.stats
+			all.stats = r.stats
 		case ResponseIndices:
-				indexstats = r.indices
+			all.indices = r.indices
 		case ResponseTasks:
-				taskstatus = r.tasks
+			all.tasks = r.tasks
+		case ResponseHealthReport:
+			all.healthreport = r.healthreport
 		}
 	}
 
-		slog.Debug("ES result", "cluster health", clusterhealth)
+	return &all, err
+}
 
-		isleader := len(ccrstats.AutoFollowStats.AutoFollowedClusters) == 0
+func ClusterStatus(conf *cfg.Config) error {
+	res, err := getClusterStatus(conf)
+	if err != nil && !strings.Contains(err.Error(), "current license is non-compliant") {
+		return err
+	}
 
-		ccrfollowing := ""
+	slog.Debug("ES result", "cluster health", res.health)
-		if len(ccrstats.AutoFollowStats.AutoFollowedClusters) > 0 {
+
+	var isleader bool
+	var ccrfollowing string
+
+	if res.ccr != nil {
+		isleader = len(res.ccr.AutoFollowStats.AutoFollowedClusters) == 0
+
+		if len(res.ccr.AutoFollowStats.AutoFollowedClusters) > 0 {
 			// is following another cluster
 			ccrfollowing = fmt.Sprintf("%s (%d/%d)",
-				ccrstats.AutoFollowStats.AutoFollowedClusters[0].ClusterName,
+				res.ccr.AutoFollowStats.AutoFollowedClusters[0].ClusterName,
-				ccrstats.AutoFollowStats.NumberOfSuccessfulFollowIndices,
+				res.ccr.AutoFollowStats.NumberOfSuccessfulFollowIndices,
-				ccrstats.AutoFollowStats.NumberOfFailedFollowIndices,
+				res.ccr.AutoFollowStats.NumberOfFailedFollowIndices,
 			)
 		}
+	}
 
 	// look for red indices, if any
 	redindices := 0
-		for _, index := range *indexstats {
+	for _, index := range *res.indices {
 		if *index.Health == "red" {
 			redindices++
 		}
@@ -169,45 +181,62 @@ func ClusterStatus(conf *cfg.Config) error {
 
 	// look for long running tasks
 	longtasks := 0
-		for _, task := range *taskstatus {
+	for _, task := range *res.tasks {
 		if strings.Contains(*task.RunningTime, "d") {
 			longtasks++
 		}
 	}
 
 	table := printer.NewTable(conf, 2, 7)
-		table.Addheaders(cluster, "status")
+	table.Addheaders(conf.DefaultCluster.Name, "status")
 
 	table.Entries = [][]string{
-			{"Cluster Name", clusterhealth.ClusterName},
+		{"Cluster Name", res.health.ClusterName},
-			{"ES Status", printer.Colorize(conf, clusterhealth.Status.Name, clusterhealth.Status.Name)},
+		{"ES Status", printer.Colorize(conf, res.health.Status.Name, res.health.Status.Name)},
-			{"ES Version", info.Version.Int},
+		{"ES Version", res.info.Version.Int},
 		{"Is Leader", fmt.Sprintf("%t", isleader)},
-			{"Active Shards", fmt.Sprintf("%d", clusterhealth.ActiveShards)},
+		{"Active Shards", fmt.Sprintf("%d", res.health.ActiveShards)},
-			{"Active Primary Shards", fmt.Sprintf("%d", clusterhealth.ActivePrimaryShards)},
+		{"Active Primary Shards", fmt.Sprintf("%d", res.health.ActivePrimaryShards)},
-			{"Unassigned Shards", fmt.Sprintf("%d", clusterhealth.UnassignedShards)},
+		{"Unassigned Shards", fmt.Sprintf("%d", res.health.UnassignedShards)},
-			{"Unassigned Primary Shards", fmt.Sprintf("%d", clusterhealth.UnassignedPrimaryShards)},
+		{"Unassigned Primary Shards", fmt.Sprintf("%d", res.health.UnassignedPrimaryShards)},
-			{"Pending Tasks", fmt.Sprintf("%d", clusterhealth.NumberOfPendingTasks)},
+		{"Pending Tasks", fmt.Sprintf("%d", res.health.NumberOfPendingTasks)},
-			{"Nodes", fmt.Sprintf("%d", clusterhealth.NumberOfNodes)},
+		{"Nodes", fmt.Sprintf("%d", res.health.NumberOfNodes)},
 		{"Red Indices", fmt.Sprintf("%d", redindices)},
 		{"Long Running Tasks", fmt.Sprintf("%d", longtasks)},
 	}
 
-		if !isleader {
+	if !isleader && res.ccr != nil {
 		table.Entries = append(table.Entries, [][]string{
 			{"AutoFollow (success/failed indices)", ccrfollowing},
-				{"Followed Indices", fmt.Sprintf("%d", len(ccrstats.FollowStats.Indices))},
+			{"Followed Indices", fmt.Sprintf("%d", len(res.ccr.FollowStats.Indices))},
 		}...)
 	}
 
 	if conf.Verbose {
-			table = gatherClusterStats(conf, clusterstats, table)
+		table = gatherClusterStats(conf, res.stats, table)
+	}
+
+	if res.health.Status.Name != "green" {
+		for name, indicator := range res.healthreport.Indicators {
+			if indicator.Status != "green" {
+				table.Entries = append(table.Entries, []string{
+					printer.Colorize(conf, indicator.Status, "Bad health "+name), indicator.Symptom,
+				})
+
+				for _, diag := range indicator.Diagnosis {
+					table.Entries = append(table.Entries, []string{" -> cause", diag.Cause})
+
+					for resource, items := range diag.AffectedResources {
+						table.Entries = append(table.Entries, []string{"    -> affected " + resource, strings.Join(items, ",")})
+					}
+				}
+			}
+		}
 	}
 
 	if err := table.Print(); err != nil {
 		return err
 	}
-	}
 
 	return nil
 }

pkg/es/cluster_reroute.go

@@ -0,0 +1,125 @@
+/*
+Copyright © 2026 Thomas von Dein
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+package es
+
+import (
+	"context"
+	"fmt"
+
+	"codeberg.org/scip/esctl/pkg/cfg"
+	"github.com/elastic/go-elasticsearch/v9/typedapi/esdsl"
+	"github.com/elastic/go-elasticsearch/v9/typedapi/types"
+)
+
+func ClusterRerouteMove(conf *cfg.Config, index string) error {
+	move := conf.DefaultCluster.ES().Cluster.Reroute()
+
+	commands := esdsl.NewCommand()
+	moveCommand := &types.CommandMoveAction{
+		Shard:    conf.Shards,
+		FromNode: conf.FromNode,
+		ToNode:   conf.ToNode,
+		Index:    index,
+	}
+
+	commands.CommandCaster().Move = moveCommand
+
+	move.Commands(commands)
+
+	_, err := move.Do(context.Background())
+	if err != nil {
+		return fmt.Errorf("failed to reroute move: %w", err)
+	}
+
+	return nil
+}
+
+func ClusterRerouteAllocateReplica(conf *cfg.Config, index string) error {
+	move := conf.DefaultCluster.ES().Cluster.Reroute()
+
+	commands := esdsl.NewCommand()
+	allocCommand := &types.CommandAllocateReplicaAction{
+		Shard: conf.Shards,
+		Node:  conf.ToNode,
+		Index: index,
+	}
+
+	commands.CommandCaster().AllocateReplica = allocCommand
+
+	move.Commands(commands)
+
+	_, err := move.Do(context.Background())
+	if err != nil {
+		return fmt.Errorf("failed to allocate a replica shard: %w", err)
+	}
+
+	return nil
+}
+
+func ClusterRerouteCancel(conf *cfg.Config, index string) error {
+	move := conf.DefaultCluster.ES().Cluster.Reroute()
+
+	commands := esdsl.NewCommand()
+	cancelCommand := &types.CommandCancelAction{
+		Shard:        conf.Shards,
+		Node:         conf.ToNode,
+		Index:        index,
+		AllowPrimary: &conf.AllowPrimary,
+	}
+
+	commands.CommandCaster().Cancel = cancelCommand
+
+	move.Commands(commands)
+
+	_, err := move.Do(context.Background())
+	if err != nil {
+		return fmt.Errorf("failed to cancel a reroute process: %w", err)
+	}
+
+	return nil
+}
+
+func ClusterRerouteAllocatePrimary(conf *cfg.Config, index string, stale bool) error {
+	move := conf.DefaultCluster.ES().Cluster.Reroute()
+
+	commands := esdsl.NewCommand()
+	allocCommand := &types.CommandAllocatePrimaryAction{
+		Shard:          conf.Shards,
+		Node:           conf.ToNode,
+		Index:          index,
+		AcceptDataLoss: conf.AcceptDataLoss,
+	}
+
+	if stale {
+		commands.CommandCaster().AllocateStalePrimary = allocCommand
+	} else {
+		commands.CommandCaster().AllocateEmptyPrimary = allocCommand
+	}
+
+	move.Commands(commands)
+
+	_, err := move.Do(context.Background())
+	if err != nil {
+		which := "empty"
+		if stale {
+			which = "stale"
+		}
+		return fmt.Errorf("failed to allocate %s primary shard: %w", which, err)
+	}
+
+	return nil
+}

pkg/es/cluster_settings.go

@@ -20,15 +20,19 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"log/slog"
 
 	"codeberg.org/scip/esctl/pkg/cfg"
 	"codeberg.org/scip/esctl/pkg/printer"
 	"github.com/urfave/cli/v3"
 )
 
+func ClusterSettingsNames(conf *cfg.Config) ([]string, error) {
+	return validClusterSettings, nil
+}
+
 func ClusterSettingsList(conf *cfg.Config) error {
 	res, err := conf.DefaultCluster.ES().Cluster.GetSettings().
+		FlatSettings(true).
 		Do(context.Background())
 	if err != nil {
 		return fmt.Errorf("failed to get cluster settings: %s", esErrorString(err))
@@ -48,19 +52,7 @@ func ClusterSettingsList(conf *cfg.Config) error {
 	}
 
 	for topic, val := range settingshash {
-		data := map[string]any{}
+		entries = append(entries, []string{topic, string(val)})
-
-		err := json.Unmarshal(val, &data)
-		if err != nil {
-			return fmt.Errorf("failed to unmarshall setting for topic %s: %s", topic, err)
-		}
-
-		paths := getJsonPath(map[string]string{}, data, topic)
-		slog.Debug("settings", topic, paths)
-
-		for setting, value := range paths {
-			entries = append(entries, []string{setting, fmt.Sprintf("%v", value)})
-		}
 	}
 
 	table.Entries = entries

pkg/es/cluster_settings_const.go

@@ -0,0 +1,914 @@
+/*
+Copyright © 2026 Thomas von Dein
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+package es
+
+// manually extracted from https://www.elastic.co/docs/reference/elasticsearch/configuration-reference
+
+var validClusterSettings []string = []string{
+	"xpack.security.audit.enabled",
+	"xpack.security.audit.logfile.events.include",
+	"xpack.security.audit.logfile.events.exclude",
+	"xpack.security.audit.logfile.events.emit_request_body",
+	"xpack.security.audit.logfile.emit_node_name",
+	"xpack.security.audit.logfile.emit_node_host_address",
+	"xpack.security.audit.logfile.emit_node_host_name",
+	"xpack.security.audit.logfile.emit_node_id",
+	"indices.breaker.total.use_real_memory",
+	"indices.breaker.total.limit",
+	"indices.breaker.fielddata.limit",
+	"indices.breaker.fielddata.overhead",
+	"indices.breaker.request.limit",
+	"indices.breaker.request.overhead",
+	"network.breaker.inflight_requests.limit",
+	"network.breaker.inflight_requests.overhead",
+	"script.max_compilations_rate",
+	"script.painless.regex.enabled",
+	"breaker.eql_sequence.limit",
+	"breaker.eql_sequence.overhead",
+	"breaker.eql_sequence.type",
+	"breaker.model_inference.limit",
+	"breaker.model_inference.overhead",
+	"breaker.model_inference.type",
+	"cluster.routing.allocation.enable",
+	"cluster.routing.allocation.same_shard.host",
+	"cluster.routing.allocation.total_shards_per_node",
+	"cluster.routing.allocation.node_concurrent_incoming_recoveries",
+	"cluster.routing.allocation.node_concurrent_outgoing_recoveries",
+	"cluster.routing.allocation.node_concurrent_recoveries",
+	"cluster.routing.allocation.node_initial_primaries_recoveries",
+	"cluster.routing.allocation.allow_rebalance",
+	"cluster.routing.rebalance.enable",
+	"cluster.routing.allocation.cluster_concurrent_rebalance",
+	"cluster.routing.allocation.type",
+	"cluster.routing.allocation.balance.threshold",
+	"cluster.routing.allocation.balance.shard",
+	"cluster.routing.allocation.balance.index",
+	"cluster.routing.allocation.balance.disk_usage",
+	"cluster.routing.allocation.balance.write_load",
+	"cluster.routing.allocation.disk.threshold_enabled",
+	"cluster.routing.allocation.disk.watermark.low",
+	"cluster.routing.allocation.disk.watermark.low.max_headroom",
+	"cluster.routing.allocation.disk.watermark.high",
+	"cluster.routing.allocation.disk.watermark.high.max_headroom",
+	"cluster.routing.allocation.disk.watermark.enable_for_single_data_node",
+	"cluster.routing.allocation.disk.watermark.flood_stage",
+	"cluster.routing.allocation.disk.watermark.flood_stage.max_headroom",
+	"cluster.routing.allocation.disk.watermark.flood_stage.frozen",
+	"cluster.routing.allocation.disk.watermark.flood_stage.frozen.max_headroom",
+	"cluster.info.update.interval",
+	"cluster.routing.allocation.awareness.attributes",
+	"cluster.routing.allocation.awareness.force.*",
+	"cluster.routing.allocation.include.{{attribute}}",
+	"cluster.routing.allocation.require.{{attribute}}",
+	"cluster.routing.allocation.exclude.{{attribute}}",
+	"cluster.routing.allocation.stats.cache.ttl",
+	"ccr.indices.recovery.max_bytes_per_sec",
+	"ccr.indices.recovery.max_concurrent_file_chunks",
+	"ccr.indices.recovery.chunk_size",
+	"ccr.indices.recovery.recovery_activity_timeout",
+	"ccr.indices.recovery.internal_action_timeout",
+	"data_streams.lifecycle.retention.max",
+	"data_streams.lifecycle.retention.default",
+	"data_streams.lifecycle.poll_interval",
+	"cluster.lifecycle.default.rollover",
+	"data_streams.lifecycle.target.merge.policy.merge_factor",
+	"data_streams.lifecycle.target.merge.policy.floor_segment",
+	"data_streams.lifecycle.signalling.error_retry_interval",
+	"data_streams.lifecycle.downsampling.max_indices_in_progress",
+	"dlm.frozen.transition.poll_interval",
+	"dlm.frozen.transition.thread_pool.size",
+	"dlm.frozen.transition.thread_pool.queue_size",
+	"dlm.frozen.cleanup.poll_interval",
+	"index.lifecycle.prefer_ilm",
+	"index.lifecycle.origination_date",
+	"index.dlm.frozen.created",
+	"discovery.seed_hosts",
+	"discovery.seed_providers",
+	"discovery.type",
+	"cluster.initial_master_nodes",
+	"discovery.cluster_formation_warning_timeout",
+	"discovery.find_peers_interval",
+	"discovery.probe.connect_timeout",
+	"discovery.probe.handshake_timeout",
+	"discovery.request_peers_timeout",
+	"discovery.find_peers_warning_timeout",
+	"discovery.seed_resolver.max_concurrent_resolvers",
+	"discovery.seed_resolver.timeout",
+	"cluster.auto_shrink_voting_configuration",
+	"cluster.election.duration",
+	"cluster.election.initial_timeout",
+	"cluster.election.max_timeout",
+	"cluster.fault_detection.follower_check.timeout",
+	"cluster.fault_detection.follower_check.retry_count",
+	"cluster.fault_detection.leader_check.interval",
+	"cluster.fault_detection.leader_check.timeout",
+	"cluster.fault_detection.leader_check.retry_count",
+	"cluster.follower_lag.timeout",
+	"cluster.max_voting_config_exclusions",
+	"cluster.publish.info_timeout",
+	"cluster.publish.timeout",
+	"cluster.discovery_configuration_check.interval",
+	"cluster.join_validation.cache_timeout",
+	"cluster.no_master_block",
+	"monitor.fs.health.enabled",
+	"monitor.fs.health.refresh_interval",
+	"monitor.fs.health.slow_path_logging_threshold",
+	"enrich.cache_size",
+	"enrich.coordinator_proxy.max_concurrent_requests",
+	"enrich.coordinator_proxy.max_lookups_per_request",
+	"enrich.coordinator_proxy.queue_capacity",
+	"enrich.fetch_size",
+	"enrich.max_force_merge_attempts",
+	"enrich.cleanup_period",
+	"enrich.max_concurrent_policy_executions",
+	"indices.fielddata.cache.size",
+	"health.master_history.has_master_lookup_timeframe",
+	"master_history.max_age",
+	"health.master_history.identity_changes_threshold",
+	"health.master_history.no_master_transitions_threshold",
+	"health.node.enabled",
+	"health.reporting.local.monitor.interval",
+	"health.ilm.max_time_on_action",
+	"health.ilm.max_time_on_step",
+	"health.ilm.max_retries_per_step",
+	"health.periodic_logger.enabled",
+	"health.periodic_logger.poll_interval",
+	"health.shard_capacity.unhealthy_threshold.yellow",
+	"health.shard_capacity.unhealthy_threshold.red",
+	"health.master_history.has_master_lookup_timeframe",
+	"master_history.max_age",
+	"health.master_history.identity_changes_threshold",
+	"health.master_history.no_master_transitions_threshold",
+	"health.node.enabled",
+	"health.reporting.local.monitor.interval",
+	"health.ilm.max_time_on_action",
+	"health.ilm.max_time_on_step",
+	"health.ilm.max_retries_per_step",
+	"health.periodic_logger.enabled",
+	"health.periodic_logger.poll_interval",
+	"health.shard_capacity.unhealthy_threshold.yellow",
+	"health.shard_capacity.unhealthy_threshold.red",
+	"indices.memory.index_buffer_size",
+	"indices.memory.min_index_buffer_size",
+	"indices.memory.max_index_buffer_size",
+	"indexing_pressure.memory.limit",
+	"xpack.ilm.enabled",
+	"indices.lifecycle.history_index_enabled",
+	"indices.lifecycle.poll_interval",
+	"indices.lifecycle.rollover.only_if_has_documents",
+	"index.lifecycle.indexing_complete",
+	"index.lifecycle.name",
+	"index.lifecycle.origination_date",
+	"index.lifecycle.parse_origination_date",
+	"index.lifecycle.step.wait_time_threshold",
+	"index.lifecycle.rollover_alias",
+	"action.auto_create_index",
+	"action.destructive_requires_name",
+	"cluster.indices.close.enable",
+	"stack.templates.enabled",
+	"xpack.profiling.enabled",
+	"xpack.profiling.templates.enabled",
+	"xpack.otel_data.registry.enabled",
+	"xpack.otel_data.histogram_field_type",
+	"reindex.remote.whitelist",
+	"reindex.remote.blocklist",
+	"cluster.reindex.pit.keep_alive",
+	"reindex.ssl.certificate",
+	"reindex.ssl.certificate_authorities",
+	"reindex.ssl.key",
+	"reindex.ssl.key_passphrase",
+	"reindex.ssl.keystore.key_password",
+	"reindex.ssl.keystore.password",
+	"reindex.ssl.keystore.path",
+	"reindex.ssl.keystore.type",
+	"reindex.ssl.secure_key_passphrase",
+	"reindex.ssl.keystore.secure_key_password",
+	"reindex.ssl.keystore.secure_password",
+	"reindex.ssl.truststore.password",
+	"reindex.ssl.truststore.path",
+	"reindex.ssl.truststore.secure_password",
+	"reindex.ssl.truststore.type",
+	"reindex.ssl.verification_mode",
+	"indices.recovery.max_bytes_per_sec",
+	"indices.recovery.max_concurrent_file_chunks",
+	"indices.recovery.max_concurrent_operations",
+	"indices.recovery.use_snapshots",
+	"indices.recovery.max_concurrent_snapshot_file_downloads",
+	"indices.recovery.max_concurrent_snapshot_file_downloads_per_node",
+	"node.bandwidth.recovery.disk.read",
+	"node.bandwidth.recovery.disk.write",
+	"node.bandwidth.recovery.network",
+	"node.bandwidth.recovery.factor.read",
+	"node.bandwidth.recovery.factor.write",
+	"node.bandwidth.recovery.operator.factor.read",
+	"node.bandwidth.recovery.operator.factor.write",
+	"node.bandwidth.recovery.operator.factor",
+	"node.bandwidth.recovery.operator.factor.max_overcommit",
+	"xpack.inference.query_timeout",
+	"xpack.inference.logging.reset_interval",
+	"xpack.inference.logging.wait_duration",
+	"xpack.inference.http.max_response_size",
+	"xpack.inference.http.max_total_connections",
+	"xpack.inference.http.max_route_connections",
+	"xpack.inference.http.connection_eviction_interval",
+	"xpack.inference.http.connection_eviction_max_idle_time",
+	"xpack.inference.http.request_executor.queue_capacity",
+	"xpack.inference.http.retry.initial_delay",
+	"xpack.inference.http.retry.max_delay_bound",
+	"xpack.inference.http.retry.timeout",
+	"xpack.inference.truncator.reduction_percentage",
+	"xpack.inference.endpoint.cache.enabled",
+	"xpack.inference.endpoint.cache.weight",
+	"xpack.inference.endpoint.cache.expiry_time",
+	"xpack.inference.oauth2.token_cache.enabled",
+	"xpack.inference.oauth2.token_cache.weight",
+	"xpack.inference.oauth2.token_cache.expiry_time",
+	"xpack.inference.ccm.cache.weight",
+	"xpack.inference.ccm.cache.expiry_time",
+	"xpack.license.self_generated.type",
+	"gateway.expected_data_nodes",
+	"gateway.recover_after_time",
+	"gateway.recover_after_data_nodes",
+	"node.roles: [ ml ]",
+	"xpack.ml.enabled",
+	"xpack.ml.inference_model.cache_size",
+	"xpack.ml.inference_model.time_to_live",
+	"xpack.ml.max_inference_processors",
+	"xpack.ml.max_machine_memory_percent",
+	"xpack.ml.max_model_memory_limit",
+	"xpack.ml.max_open_jobs",
+	"xpack.ml.nightly_maintenance_requests_per_second",
+	"xpack.ml.results_index_rollover_max_size",
+	"xpack.ml.anomalies.heal_reindexed_v7.enabled",
+	"xpack.ml.idle_job_auto_close_timeout",
+	"xpack.ml.node_concurrent_job_allocations",
+	"xpack.ml.enable_config_migration",
+	"xpack.ml.max_anomaly_records",
+	"xpack.ml.max_lazy_ml_nodes",
+	"xpack.ml.max_ml_node_size",
+	"xpack.ml.trained_models.graph_validation_enabled",
+	"xpack.ml.model_repository",
+	"xpack.ml.persist_results_max_retries",
+	"xpack.ml.process_connect_timeout",
+	"xpack.ml.use_auto_machine_memory_percent",
+	"xpack.monitoring.enabled",
+	"xpack.monitoring.collection.enabled",
+	"xpack.monitoring.collection.interval",
+	"xpack.monitoring.elasticsearch.collection.enabled",
+	"xpack.monitoring.collection.cluster.stats.timeout",
+	"xpack.monitoring.collection.node.stats.timeout",
+	"xpack.monitoring.collection.indices",
+	"xpack.monitoring.collection.index.stats.timeout",
+	"xpack.monitoring.collection.index.recovery.active_only",
+	"xpack.monitoring.collection.index.recovery.timeout",
+	"xpack.monitoring.history.duration",
+	"xpack.monitoring.exporters",
+	"cluster_alerts.management.enabled",
+	"wait_master.timeout",
+	"auth.username",
+	"auth.secure_password",
+	"connection.timeout",
+	"connection.read_timeout",
+	"proxy.base_path",
+	"index.name.time_format",
+	"cluster_alerts.management.enabled",
+	"cluster_alerts.management.blacklist",
+	"xpack.monitoring.exporters.$NAME.ssl.supported_protocols",
+	"xpack.monitoring.exporters.$NAME.ssl.verification_mode",
+	"xpack.monitoring.exporters.$NAME.ssl.cipher_suites",
+	"xpack.monitoring.exporters.$NAME.ssl.key",
+	"xpack.monitoring.exporters.$NAME.ssl.key_passphrase",
+	"xpack.monitoring.exporters.$NAME.ssl.secure_key_passphrase",
+	"xpack.monitoring.exporters.$NAME.ssl.certificate",
+	"xpack.monitoring.exporters.$NAME.ssl.certificate_authorities",
+	"xpack.monitoring.exporters.$NAME.ssl.keystore.path",
+	"xpack.monitoring.exporters.$NAME.ssl.keystore.password",
+	"xpack.monitoring.exporters.$NAME.ssl.keystore.secure_password",
+	"xpack.monitoring.exporters.$NAME.ssl.keystore.key_password",
+	"xpack.monitoring.exporters.$NAME.ssl.keystore.secure_key_password",
+	"xpack.monitoring.exporters.$NAME.ssl.truststore.path",
+	"xpack.monitoring.exporters.$NAME.ssl.truststore.password",
+	"xpack.monitoring.exporters.$NAME.ssl.truststore.secure_password",
+	"xpack.monitoring.exporters.$NAME.ssl.keystore.path",
+	"xpack.monitoring.exporters.$NAME.ssl.keystore.type",
+	"xpack.monitoring.exporters.$NAME.ssl.keystore.password",
+	"xpack.monitoring.exporters.$NAME.ssl.keystore.secure_password",
+	"xpack.monitoring.exporters.$NAME.ssl.keystore.key_password",
+	"xpack.monitoring.exporters.$NAME.ssl.keystore.secure_key_password",
+	"xpack.monitoring.exporters.$NAME.ssl.truststore.path",
+	"xpack.monitoring.exporters.$NAME.ssl.truststore.type",
+	"xpack.monitoring.exporters.$NAME.ssl.truststore.password",
+	"xpack.monitoring.exporters.$NAME.ssl.truststore.secure_password",
+	"network.host",
+	"http.port",
+	"transport.port",
+	"remote_cluster.port",
+	"0.0.0.0",
+	"network.bind_host",
+	"network.publish_host",
+	"network.tcp.keep_alive",
+	"network.tcp.keep_idle",
+	"network.tcp.keep_interval",
+	"network.tcp.keep_count",
+	"network.tcp.no_delay",
+	"network.tcp.reuse_address",
+	"network.tcp.send_buffer_size",
+	"network.tcp.receive_buffer_size",
+	"http.host",
+	"http.bind_host",
+	"http.publish_host",
+	"http.publish_port",
+	"http.max_content_length",
+	"http.max_initial_line_length",
+	"http.max_header_size",
+	"http.compression",
+	"http.compression_level",
+	"http.cors.enabled",
+	"http.detailed_errors.enabled",
+	"http.pipelining.max_events",
+	"http.max_warning_header_count",
+	"http.max_warning_header_size",
+	"http.tcp.keep_alive",
+	"http.tcp.keep_idle",
+	"http.tcp.keep_interval",
+	"http.tcp.keep_count",
+	"http.tcp.no_delay",
+	"http.tcp.reuse_address",
+	"http.tcp.send_buffer_size",
+	"http.tcp.receive_buffer_size",
+	"http.client_stats.enabled",
+	"http.client_stats.closed_channels.max_count",
+	"http.client_stats.closed_channels.max_age",
+	"transport.host",
+	"transport.bind_host",
+	"transport.publish_host",
+	"transport.publish_port",
+	"transport.connect_timeout",
+	"transport.compress",
+	"transport.compression_scheme",
+	"transport.tcp.keep_alive",
+	"transport.tcp.keep_idle",
+	"transport.tcp.keep_interval",
+	"transport.tcp.keep_count",
+	"transport.tcp.no_delay",
+	"transport.tcp.reuse_address",
+	"transport.tcp.send_buffer_size",
+	"transport.tcp.receive_buffer_size",
+	"transport.ping_schedule",
+	"remote_cluster_server.enabled",
+	"remote_cluster.host",
+	"remote_cluster.bind_host",
+	"remote_cluster.publish_host",
+	"remote_cluster.publish_port",
+	"remote_cluster.tcp.keep_alive",
+	"remote_cluster.tcp.keep_idle",
+	"remote_cluster.tcp.keep_interval",
+	"remote_cluster.tcp.keep_count",
+	"remote_cluster.tcp.no_delay",
+	"remote_cluster.tcp.reuse_address",
+	"remote_cluster.tcp.send_buffer_size",
+	"remote_cluster.tcp.receive_buffer_size",
+	"org.elasticsearch.transport.InboundHandler",
+	"org.elasticsearch.transport.OutboundHandler",
+	"org.elasticsearch.common.network.ThreadWatchdog",
+	"network.thread.watchdog.interval",
+	"network.thread.watchdog.quiet_time",
+	"indices.queries.cache.size",
+	"index.queries.cache.enabled",
+	"vectors.indexing.use_gpu",
+	"cluster.remote.initial_connect_timeout",
+	"cluster.remote.node.attr",
+	"cluster.remote.signing.certificate_authorities",
+	"cluster.remote.signing.truststore.path",
+	"cluster.remote.signing.truststore.secure_password",
+	"cluster.remote.signing.truststore.algorithm",
+	"cluster.remote.signing.truststore.type",
+	"cluster.remote.signing.diagnose.trust",
+	"indices.query.bool.max_clause_count",
+	"search.max_buckets",
+	"search.aggs.only_allowed_metric_scripts",
+	"search.aggs.allowed_inline_metric_scripts",
+	"search.aggs.allowed_stored_metric_scripts",
+	"indices.query.bool.max_nested_depth",
+	"search.task_watchdog.enabled",
+	"search.task_watchdog.coordinator_threshold",
+	"search.task_watchdog.data_node_threshold",
+	"search.task_watchdog.interval",
+	"search.task_watchdog.cooldown_period",
+	"xpack.security.enabled",
+	"xpack.security.autoconfiguration.enabled",
+	"xpack.security.enrollment.enabled",
+	"xpack.security.hide_settings",
+	"xpack.security.fips_mode.enabled",
+	"xpack.security.fips_mode.required_providers",
+	"xpack.security.authc.password_hashing.algorithm",
+	"xpack.security.authc.anonymous.username",
+	"xpack.security.authc.anonymous.roles",
+	"xpack.security.authc.anonymous.authz_exception",
+	"xpack.security.automata.max_determinized_states",
+	"xpack.security.automata.cache.enabled",
+	"xpack.security.automata.cache.size",
+	"xpack.security.automata.cache.ttl",
+	"xpack.security.dls_fls.enabled",
+	"xpack.security.dls.bitset.cache.ttl",
+	"xpack.security.dls.bitset.cache.size",
+	"xpack.security.authc.token.enabled",
+	"xpack.security.authc.token.timeout",
+	"xpack.security.authc.api_key.enabled",
+	"xpack.security.authc.api_key.cache.ttl",
+	"xpack.security.authc.api_key.cache.max_keys",
+	"xpack.security.authc.api_key.cache.hash_algo",
+	"xpack.security.authc.api_key.delete.retention_period",
+	"xpack.security.authc.api_key.delete.interval",
+	"xpack.security.authc.api_key.delete.timeout",
+	"xpack.security.authc.api_key.hashing.algorithm",
+	"xpack.security.authc.realms.saml.*",
+	"xpack.security.authc.realms.oidc.*",
+	"xpack.security.authc.realms.kerberos.*",
+	"xpack.security.authc.realms.jwt.*",
+	"cache.ttl",
+	"cache.max_users",
+	"cache.hash_algo",
+	"authentication.enabled",
+	"cache.ttl",
+	"cache.max_users",
+	"cache.hash_algo",
+	"authentication.enabled",
+	"load_balance.type",
+	"load_balance.cache_ttl",
+	"user_search.base_dn",
+	"user_search.scope",
+	"user_search.filter",
+	"user_search.attribute",
+	"user_search.pool.enabled",
+	"user_search.pool.size",
+	"user_search.pool.initial_size",
+	"user_search.pool.health_check.enabled",
+	"user_search.pool.health_check.dn",
+	"user_search.pool.health_check.interval",
+	"group_search.base_dn",
+	"group_search.scope",
+	"group_search.filter",
+	"group_search.user_attribute",
+	"files.role_mapping",
+	"timeout.tcp_connect",
+	"timeout.tcp_read",
+	"timeout.response",
+	"timeout.ldap_search",
+	"ssl.key",
+	"ssl.key_passphrase",
+	"ssl.secure_key_passphrase",
+	"ssl.certificate",
+	"ssl.certificate_authorities",
+	"ssl.keystore.path",
+	"ssl.keystore.type",
+	"ssl.keystore.password",
+	"ssl.keystore.secure_password",
+	"ssl.keystore.key_password",
+	"ssl.keystore.secure_key_password",
+	"ssl.truststore.path",
+	"ssl.truststore.password",
+	"ssl.truststore.secure_password",
+	"ssl.truststore.type",
+	"ssl.verification_mode",
+	"ssl.supported_protocols",
+	"ssl.cipher_suites",
+	"cache.ttl",
+	"cache.max_users",
+	"cache.hash_algo",
+	"authentication.enabled",
+	"load_balance.type",
+	"load_balance.cache_ttl",
+	"files.role_mapping",
+	"user_search.base_dn",
+	"user_search.scope",
+	"user_search.filter",
+	"user_search.upn_filter",
+	"user_search.down_level_filter",
+	"user_search.pool.enabled",
+	"user_search.pool.size",
+	"user_search.pool.initial_size",
+	"user_search.pool.health_check.enabled",
+	"user_search.pool.health_check.dn",
+	"user_search.pool.health_check.interval",
+	"group_search.base_dn",
+	"group_search.scope",
+	"timeout.tcp_connect",
+	"timeout.tcp_read",
+	"timeout.response",
+	"timeout.ldap_search",
+	"ssl.certificate",
+	"ssl.certificate_authorities",
+	"ssl.key",
+	"ssl.key_passphrase",
+	"ssl.secure_key_passphrase",
+	"ssl.keystore.key_password",
+	"ssl.keystore.secure_key_password",
+	"ssl.keystore.password",
+	"ssl.secure_keystore.password",
+	"ssl.keystore.path",
+	"ssl.keystore.type",
+	"ssl.truststore.password",
+	"ssl.truststore.secure_password",
+	"ssl.truststore.path",
+	"ssl.truststore.type",
+	"ssl.verification_mode",
+	"ssl.supported_protocols",
+	"ssl.cipher_suites",
+	"cache.ttl",
+	"cache.max_users",
+	"cache.hash_algo",
+	"authentication.enabled",
+	"truststore.algorithm",
+	"truststore.password",
+	"truststore.secure_password",
+	"truststore.path",
+	"files.role_mapping",
+	"cache.ttl",
+	"cache.max_users",
+	"delegation.enabled",
+	"idp.entity_id",
+	"idp.metadata.path",
+	"idp.metadata.http.fail_on_error",
+	"idp.metadata.http.connect_timeout",
+	"idp.metadata.http.read_timeout",
+	"idp.metadata.http.refresh",
+	"idp.metadata.http.minimum_refresh",
+	"idp.use_single_logout",
+	"sp.entity_id",
+	"sp.acs",
+	"sp.logout",
+	"attributes.principal",
+	"attributes.groups",
+	"attributes.name",
+	"attributes.mail",
+	"attributes.dn",
+	"attribute_patterns.principal",
+	"attribute_patterns.groups",
+	"attribute_patterns.name",
+	"attribute_patterns.mail",
+	"attribute_patterns.dn",
+	"attribute_delimiters.groups",
+	"nameid.allow_create",
+	"nameid.sp_qualifier",
+	"signing.saml_messages",
+	"signing.key",
+	"signing.secure_key_passphrase",
+	"signing.certificate",
+	"signing.keystore.path",
+	"signing.keystore.type",
+	"signing.keystore.alias",
+	"signing.keystore.secure_password",
+	"signing.keystore.secure_key_password",
+	"encryption.key",
+	"encryption.secure_key_passphrase",
+	"encryption.certificate",
+	"encryption.keystore.path",
+	"encryption.keystore.type",
+	"encryption.keystore.alias",
+	"encryption.keystore.secure_password",
+	"encryption.keystore.secure_key_password",
+	"ssl.key",
+	"ssl.key_passphrase",
+	"ssl.secure_key_passphrase",
+	"ssl.certificate",
+	"ssl.certificate_authorities",
+	"ssl.keystore.path",
+	"ssl.keystore.type",
+	"ssl.keystore.password",
+	"ssl.keystore.secure_password",
+	"ssl.keystore.key_password",
+	"ssl.keystore.secure_key_password",
+	"ssl.truststore.path",
+	"ssl.truststore.type",
+	"ssl.truststore.password",
+	"ssl.truststore.secure_password",
+	"ssl.verification_mode",
+	"ssl.supported_protocols",
+	"ssl.cipher_suites",
+	"keytab.path",
+	"krb.debug",
+	"cache.ttl",
+	"cache.max_users",
+	"op.issuer",
+	"op.authorization_endpoint",
+	"op.token_endpoint",
+	"op.userinfo_endpoint",
+	"op.endsession_endpoint",
+	"op.jwkset_path",
+	"rp.client_id",
+	"rp.client_secret",
+	"rp.client_auth_method",
+	"rp.client_auth_jwt_signature_algorithm",
+	"rp.redirect_uri",
+	"rp.response_type",
+	"rp.signature_algorithm",
+	"rp.requested_scopes",
+	"rp.post_logout_redirect_uri",
+	"claims.principal",
+	"claims.groups",
+	"claims.name",
+	"claims.mail",
+	"claims.dn",
+	"claim_patterns.principal",
+	"claim_patterns.groups",
+	"claim_patterns.name",
+	"claim_patterns.mail",
+	"claim_patterns.dn",
+	"http.proxy.host",
+	"http.proxy.scheme",
+	"http.proxy.port",
+	"http.connect_timeout",
+	"http.connection_read_timeout",
+	"http.socket_timeout",
+	"http.max_connections",
+	"http.max_endpoint_connections",
+	"http.tcp.keep_alive",
+	"http.connection_pool_ttl",
+	"ssl.key",
+	"ssl.key_passphrase",
+	"ssl.secure_key_passphrase",
+	"ssl.certificate",
+	"ssl.certificate_authorities",
+	"ssl.keystore.path",
+	"ssl.keystore.type",
+	"ssl.keystore.password",
+	"ssl.keystore.secure_password",
+	"ssl.keystore.key_password",
+	"ssl.keystore.secure_key_password",
+	"ssl.truststore.path",
+	"ssl.truststore.type",
+	"ssl.truststore.password",
+	"ssl.truststore.secure_password",
+	"ssl.verification_mode",
+	"ssl.supported_protocols",
+	"ssl.cipher_suites",
+	"fallback_claims.sub",
+	"fallback_claims.aud",
+	"claims.dn",
+	"claim_patterns.dn",
+	"claims.groups",
+	"claim_patterns.group",
+	"claims.mail",
+	"claim_patterns.mail",
+	"claims.name",
+	"claim_patterns.name",
+	"claims.principal",
+	"claim_patterns.principal",
+	"client_authentication.type",
+	"client_authentication.shared_secret",
+	"client_authentication.rotation_grace_period",
+	"http.proxy.host",
+	"http.proxy.scheme",
+	"http.proxy.port",
+	"http.connect_timeout",
+	"http.connection_read_timeout",
+	"http.socket_timeout",
+	"http.max_connections",
+	"http.max_endpoint_connections",
+	"jwt.cache.size",
+	"jwt.cache.ttl",
+	"pkc_jwkset_reload.enabled",
+	"pkc_jwkset_reload.file_interval",
+	"pkc_jwkset_reload.url_interval_min",
+	"pkc_jwkset_reload.url_interval_max",
+	"ssl.key",
+	"ssl.key_passphrase",
+	"ssl.secure_key_passphrase",
+	"ssl.certificate",
+	"ssl.certificate_authorities",
+	"ssl.keystore.path",
+	"ssl.keystore.type",
+	"ssl.keystore.password",
+	"ssl.keystore.secure_password",
+	"ssl.keystore.key_password",
+	"ssl.keystore.secure_key_password",
+	"ssl.truststore.path",
+	"ssl.truststore.type",
+	"ssl.truststore.password",
+	"ssl.truststore.secure_password",
+	"ssl.verification_mode",
+	"ssl.supported_protocols",
+	"ssl.cipher_suites",
+	"xpack.security.ssl.diagnose.trust",
+	"xpack.security.http.ssl.enabled",
+	"xpack.security.http.ssl.supported_protocols",
+	"xpack.security.http.ssl.client_authentication",
+	"xpack.security.http.ssl.verification_mode",
+	"xpack.security.http.ssl.cipher_suites",
+	"xpack.security.http.ssl.key",
+	"xpack.security.http.ssl.key_passphrase",
+	"xpack.security.http.ssl.secure_key_passphrase",
+	"xpack.security.http.ssl.certificate",
+	"xpack.security.http.ssl.certificate_authorities",
+	"xpack.security.http.ssl.keystore.path",
+	"xpack.security.http.ssl.keystore.password",
+	"xpack.security.http.ssl.keystore.secure_password",
+	"xpack.security.http.ssl.keystore.key_password",
+	"xpack.security.http.ssl.keystore.secure_key_password",
+	"xpack.security.http.ssl.truststore.path",
+	"xpack.security.http.ssl.truststore.password",
+	"xpack.security.http.ssl.truststore.secure_password",
+	"xpack.security.http.ssl.keystore.path",
+	"xpack.security.http.ssl.keystore.type",
+	"xpack.security.http.ssl.keystore.password",
+	"xpack.security.http.ssl.keystore.secure_password",
+	"xpack.security.http.ssl.keystore.key_password",
+	"xpack.security.http.ssl.keystore.secure_key_password",
+	"xpack.security.http.ssl.truststore.path",
+	"xpack.security.http.ssl.truststore.type",
+	"xpack.security.http.ssl.truststore.password",
+	"xpack.security.http.ssl.truststore.secure_password",
+	"xpack.security.transport.ssl.enabled",
+	"xpack.security.transport.ssl.supported_protocols",
+	"xpack.security.transport.ssl.client_authentication",
+	"xpack.security.transport.ssl.verification_mode",
+	"xpack.security.transport.ssl.cipher_suites",
+	"xpack.security.transport.ssl.trust_restrictions.x509_fields",
+	"xpack.security.transport.ssl.handshake_timeout",
+	"xpack.security.transport.ssl.key",
+	"xpack.security.transport.ssl.key_passphrase",
+	"xpack.security.transport.ssl.secure_key_passphrase",
+	"xpack.security.transport.ssl.certificate",
+	"xpack.security.transport.ssl.certificate_authorities",
+	"xpack.security.loginAssistanceMessage",
+	"xpack.security.transport.ssl.keystore.path",
+	"xpack.security.transport.ssl.keystore.password",
+	"xpack.security.transport.ssl.keystore.secure_password",
+	"xpack.security.transport.ssl.keystore.key_password",
+	"xpack.security.transport.ssl.keystore.secure_key_password",
+	"xpack.security.transport.ssl.truststore.path",
+	"xpack.security.transport.ssl.truststore.password",
+	"xpack.security.transport.ssl.truststore.secure_password",
+	"xpack.security.transport.ssl.keystore.path",
+	"xpack.security.transport.ssl.keystore.type",
+	"xpack.security.transport.ssl.keystore.password",
+	"xpack.security.transport.ssl.keystore.secure_password",
+	"xpack.security.transport.ssl.keystore.key_password",
+	"xpack.security.transport.ssl.keystore.secure_key_password",
+	"xpack.security.transport.ssl.truststore.path",
+	"xpack.security.transport.ssl.truststore.type",
+	"xpack.security.transport.ssl.truststore.password",
+	"xpack.security.transport.ssl.truststore.secure_password",
+	"xpack.security.remote_cluster_server.ssl.enabled",
+	"xpack.security.remote_cluster_server.ssl.supported_protocols",
+	"xpack.security.remote_cluster_server.ssl.client_authentication",
+	"xpack.security.remote_cluster_server.ssl.verification_mode",
+	"xpack.security.remote_cluster_server.ssl.cipher_suites",
+	"xpack.security.remote_cluster_server.ssl.handshake_timeout",
+	"xpack.security.remote_cluster_server.ssl.key",
+	"xpack.security.remote_cluster_server.ssl.secure_key_passphrase",
+	"xpack.security.remote_cluster_server.ssl.certificate",
+	"xpack.security.remote_cluster_server.ssl.certificate_authorities",
+	"xpack.security.remote_cluster_server.ssl.keystore.path",
+	"xpack.security.remote_cluster_server.ssl.keystore.secure_password",
+	"xpack.security.remote_cluster_server.ssl.keystore.secure_key_password",
+	"xpack.security.remote_cluster_server.ssl.truststore.path",
+	"xpack.security.remote_cluster_server.ssl.truststore.secure_password",
+	"xpack.security.remote_cluster_server.ssl.keystore.path",
+	"xpack.security.remote_cluster_server.ssl.keystore.type",
+	"xpack.security.remote_cluster_server.ssl.keystore.secure_password",
+	"xpack.security.remote_cluster_server.ssl.keystore.secure_key_password",
+	"xpack.security.remote_cluster_server.ssl.truststore.path",
+	"xpack.security.remote_cluster_server.ssl.truststore.type",
+	"xpack.security.remote_cluster_server.ssl.truststore.secure_password",
+	"xpack.security.remote_cluster_client.ssl.enabled",
+	"xpack.security.remote_cluster_client.ssl.supported_protocols",
+	"xpack.security.remote_cluster_client.ssl.verification_mode",
+	"xpack.security.remote_cluster_client.ssl.cipher_suites",
+	"xpack.security.remote_cluster_client.ssl.handshake_timeout",
+	"xpack.security.remote_cluster_client.ssl.key",
+	"xpack.security.remote_cluster_client.ssl.secure_key_passphrase",
+	"xpack.security.remote_cluster_client.ssl.certificate",
+	"xpack.security.remote_cluster_client.ssl.certificate_authorities",
+	"xpack.security.remote_cluster_client.ssl.keystore.path",
+	"xpack.security.remote_cluster_client.ssl.keystore.secure_password",
+	"xpack.security.remote_cluster_client.ssl.keystore.secure_key_password",
+	"xpack.security.remote_cluster_client.ssl.truststore.path",
+	"xpack.security.remote_cluster_client.ssl.truststore.secure_password",
+	"xpack.security.remote_cluster_client.ssl.keystore.path",
+	"xpack.security.remote_cluster_client.ssl.keystore.type",
+	"xpack.security.remote_cluster_client.ssl.keystore.secure_password",
+	"xpack.security.remote_cluster_client.ssl.keystore.secure_key_password",
+	"xpack.security.remote_cluster_client.ssl.truststore.path",
+	"xpack.security.remote_cluster_client.ssl.truststore.type",
+	"xpack.security.remote_cluster_client.ssl.truststore.secure_password",
+	"xpack.security.transport.filter.allow",
+	"xpack.security.transport.filter.deny",
+	"xpack.security.http.filter.allow",
+	"xpack.security.http.filter.deny",
+	"transport.profiles.$PROFILE.xpack.security.filter.allow",
+	"transport.profiles.$PROFILE.xpack.security.filter.deny",
+	"xpack.security.remote_cluster.filter.allow",
+	"xpack.security.remote_cluster.filter.deny",
+	"indices.requests.cache.size",
+	"indices.requests.cache.expire",
+	"snapshot.max_concurrent_operations",
+	"repositories.default_repository",
+	"slm.history_index_enabled",
+	"slm.retention_schedule",
+	"slm.retention_duration",
+	"slm.health.failed_snapshot_warn_threshold",
+	"node.roles: [ transform ]",
+	"xpack.transform.enabled",
+	"xpack.transform.num_transform_failure_retries",
+	"xpack.watcher.enabled",
+	"xpack.watcher.encrypt_sensitive_data",
+	"xpack.watcher.encryption_key",
+	"xpack.watcher.max.history.record.size",
+	"xpack.watcher.trigger.schedule.engine",
+	"xpack.watcher.history.cleaner_service.enabled",
+	"xpack.http.proxy.host",
+	"xpack.http.proxy.port",
+	"xpack.http.proxy.scheme",
+	"xpack.http.default_connection_timeout",
+	"xpack.http.default_read_timeout",
+	"xpack.http.tcp.keep_alive",
+	"xpack.http.connection_pool_ttl",
+	"xpack.http.max_response_size",
+	"xpack.http.whitelist",
+	"xpack.http.ssl.supported_protocols",
+	"xpack.http.ssl.verification_mode",
+	"xpack.http.ssl.cipher_suites",
+	"xpack.http.ssl.key",
+	"xpack.http.ssl.secure_key_passphrase",
+	"xpack.http.ssl.certificate",
+	"xpack.http.ssl.certificate_authorities",
+	"xpack.http.ssl.keystore.path",
+	"xpack.http.ssl.keystore.secure_password",
+	"xpack.http.ssl.keystore.secure_key_password",
+	"xpack.http.ssl.truststore.path",
+	"xpack.http.ssl.truststore.secure_password",
+	"xpack.http.ssl.keystore.path",
+	"xpack.http.ssl.keystore.type",
+	"xpack.http.ssl.keystore.secure_password",
+	"xpack.http.ssl.keystore.secure_key_password",
+	"xpack.http.ssl.truststore.path",
+	"xpack.http.ssl.truststore.type",
+	"xpack.http.ssl.truststore.secure_password",
+	"xpack.notification.email.default_account",
+	"xpack.notification.email.recipient_allowlist",
+	"xpack.notification.email.account",
+	"xpack.notification.email.account.domain_allowlist",
+	"email_defaults.*",
+	"smtp.auth",
+	"smtp.host",
+	"smtp.port",
+	"smtp.user",
+	"smtp.secure_password",
+	"smtp.starttls.enable",
+	"smtp.starttls.required",
+	"smtp.ssl.trust",
+	"smtp.timeout",
+	"smtp.connection_timeout",
+	"smtp.write_timeout",
+	"smtp.local_address",
+	"smtp.local_port",
+	"smtp.send_partial",
+	"smtp.wait_on_quit",
+	"xpack.notification.email.html.sanitization.allow",
+	"xpack.notification.email.html.sanitization.disallow",
+	"xpack.notification.email.html.sanitization.enabled",
+	"xpack.notification.email.ssl.supported_protocols",
+	"xpack.notification.email.ssl.verification_mode",
+	"xpack.notification.email.ssl.cipher_suites",
+	"xpack.notification.email.ssl.key",
+	"xpack.notification.email.ssl.secure_key_passphrase",
+	"xpack.notification.email.ssl.certificate",
+	"xpack.notification.email.ssl.certificate_authorities",
+	"xpack.notification.email.ssl.keystore.path",
+	"xpack.notification.email.ssl.keystore.secure_password",
+	"xpack.notification.email.ssl.keystore.secure_key_password",
+	"xpack.notification.email.ssl.truststore.path",
+	"xpack.notification.email.ssl.truststore.secure_password",
+	"xpack.notification.email.ssl.keystore.path",
+	"xpack.notification.email.ssl.keystore.type",
+	"xpack.notification.email.ssl.keystore.secure_password",
+	"xpack.notification.email.ssl.keystore.secure_key_password",
+	"xpack.notification.email.ssl.truststore.path",
+	"xpack.notification.email.ssl.truststore.type",
+	"xpack.notification.email.ssl.truststore.secure_password",
+	"xpack.notification.slack",
+	"xpack.notification.slack.default_account",
+	"xpack.notification.slack.account",
+	"xpack.notification.jira.default_account",
+	"xpack.notification.jira.account",
+	"xpack.notification.pagerduty",
+	"xpack.notification.pagerduty.default_account",
+	"xpack.notification.pagerduty.account",
+	"xpack.notification.webhook.additional_token_enabled",
+}

pkg/es/cluster_util.go

@@ -20,7 +20,6 @@ import (
 	"context"
 	"fmt"
 	"regexp"
-	"strconv"
 	"strings"
 
 	"codeberg.org/scip/esctl/pkg/cfg"
@@ -330,44 +329,3 @@ func splitArg(arg string) (string, string) {
 		return parts[0], parts[1]
 	}
 }
-
-// recursively traverse  the raw  settings hash and  build a  flat map
-// consisting of the translated path and its value.
-//
-// e.g.
-// logger:
-//
-//	org:
-//	  elasticsearch:
-//	    transport:
-//	      OutboundHandler: "ERROR"
-//
-// gets:
-//
-// logger.org.elasticsearch.transport.OutboundHandler: "ERROR"
-func getJsonPath(paths map[string]string, raw map[string]any, topic string) map[string]string {
-	for name, data := range raw {
-		path := topic + "." + name
-
-		switch value := data.(type) {
-		case string:
-			paths[path] = value
-		case *string:
-			paths[path] = *value
-		case int:
-			paths[path] = strconv.Itoa(value)
-		case *int:
-			paths[path] = strconv.Itoa(*value)
-		case map[string]any:
-			paths = getJsonPath(paths, value, path)
-		case []any:
-			val := []string{}
-			for _, item := range value {
-				val = append(val, fmt.Sprintf("%v", item))
-			}
-			paths[path] = strings.Join(val, ",")
-		}
-	}
-
-	return paths
-}

pkg/es/debug.go

@@ -0,0 +1,48 @@
+/*
+Copyright © 2026 Thomas von Dein
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+package es
+
+import (
+	"codeberg.org/scip/esctl/pkg/cfg"
+	"github.com/alecthomas/repr"
+)
+
+func Debug(conf *cfg.Config) error {
+	report, err := getHealthReport(conf)
+	if err != nil {
+		return err
+	}
+
+	repr.Println(report)
+	/*
+		res, err := conf.DefaultCluster.ES().Search().
+			Index(conf.Index).
+			Size(0).
+			Aggregations(map[string]types.Aggregations{
+				"min_ts": *esdsl.NewMinAggregation().Field("@timestamp").AggregationsCaster(),
+				"max_ts": *esdsl.NewMaxAggregation().Field("@timestamp").AggregationsCaster(),
+			}).
+			Do(context.Background())
+
+		if err != nil {
+			return err
+		}
+
+		repr.Println(res)
+	*/
+	return nil
+}

pkg/es/ilm.go

@@ -67,6 +67,7 @@ func IlmNames(conf *cfg.Config) ([]string, error) {
 
 	for name := range res {
 		names[idx] = name
+		idx++
 	}
 
 	return names, nil
@@ -121,6 +122,10 @@ func IlmShow(conf *cfg.Config, policy string) error {
 		return errors.New("no ilm policy retrieved")
 	}
 
+	if conf.Ilm.Tree {
+		return IlmShowTree(conf, ilm.Policy)
+	}
+
 	table := printer.NewTable(conf, 2, 5)
 	table.Addheaders("ilm policy setting", "value")
 
@@ -135,6 +140,101 @@ func IlmShow(conf *cfg.Config, policy string) error {
 	return table.Print()
 }
 
+// Visualize  phases or  a ILM  policy, taking  into account  that the
+// minAge for the current phases is set in the next phase
+func IlmShowTree(conf *cfg.Config, ilm types.IlmPolicy) error {
+	indent := ""
+
+	table := printer.NewTable(conf, 4, 0)
+	table.Addheaders("phase", "min age", "min size", "snapshot repo")
+
+	for _, phase := range IlmPhaseOrder {
+		if phase == "hot" {
+			fmt.Printf("%s%s phase:\n%s      rollover after %s\n",
+				indent, phase,
+				indent, formatDuration(parseDuration(ilm.Phases.Hot.Actions.Rollover.MaxAge.(string))),
+			)
+
+			if ilm.Phases.Hot.Actions.Rollover.MaxPrimaryShardSize != "" {
+				fmt.Printf("%s      rollover when storage > %s\n",
+					indent, ilm.Phases.Hot.Actions.Rollover.MaxPrimaryShardSize)
+			}
+
+			if ilm.Phases.Hot.Actions.Rollover.MaxDocs != nil {
+				fmt.Printf("%s      rollover docs > %d\n",
+					indent, *ilm.Phases.Hot.Actions.Rollover.MaxDocs)
+			}
+
+			indent += "    "
+			continue
+		}
+
+		current := getIlmCurrentPhase(ilm, phase)
+
+		if current == nil {
+			continue
+		}
+
+		next := findNextPhase(ilm, phase)
+
+		fmt.Printf("%s%s phase:\n", indent, phase)
+
+		if next != nil {
+			fmt.Printf("%s      rollover after %s\n",
+				indent, formatDuration(next.minage),
+			)
+
+			if current.Actions.SearchableSnapshot != nil {
+				fmt.Printf("%s      roll to snapshot repo: %s\n",
+					indent, current.Actions.SearchableSnapshot.SnapshotRepository)
+			}
+
+			if current.Actions.Forcemerge != nil {
+				fmt.Printf("%s      force merge segments: %d\n",
+					indent, current.Actions.Forcemerge.MaxNumSegments)
+			}
+
+			if current.Actions.Shrink != nil {
+				fmt.Printf("%s      shrink shards: %d\n",
+					indent, *current.Actions.Shrink.NumberOfShards)
+			}
+
+			if current.Actions.SetPriority != nil {
+				fmt.Printf("%s      priority: %d\n",
+					indent, *current.Actions.SetPriority.Priority)
+			}
+
+			if current.Actions.Delete != nil && current.Actions.Delete.DeleteSearchableSnapshot != nil {
+				fmt.Printf("%s      delete searchable snapshots: %t\n",
+					indent, *current.Actions.Delete.DeleteSearchableSnapshot)
+			}
+		}
+
+		if phase == "delete" {
+			fmt.Printf("%s      delete immediately\n", indent)
+		}
+
+		indent += "    "
+	}
+
+	return nil
+}
+
+func getIlmCurrentPhase(policy types.IlmPolicy, currentPhase string) *types.Phase {
+	switch currentPhase {
+	case "hot":
+		return policy.Phases.Hot
+	case "warm":
+		return policy.Phases.Warm
+	case "cold":
+		return policy.Phases.Cold
+	case "frozen":
+		return policy.Phases.Frozen
+	default:
+		return policy.Phases.Delete
+	}
+}
+
 func ilmPhaseString(phase *types.Phase, short bool) string {
 	if phase == nil {
 		return ""
@@ -220,6 +320,24 @@ func IlmExplain(conf *cfg.Config, index string) error {
 }
 
 func IlmCreate(conf *cfg.Config, policyname string) error {
+	var policy *types.IlmPolicy = nil
+
+	res, err := conf.DefaultCluster.ES().Ilm.GetLifecycle().
+		Policy(policyname).
+		Do(context.Background())
+	if err == nil {
+		if conf.Debug {
+			repr.Println(res)
+		}
+
+		ilm, exists := res[policyname]
+		if !exists {
+			return errors.New("no ilm policy retrieved")
+		}
+
+		policy = &ilm.Policy
+	}
+
 	ilm := conf.DefaultCluster.ES().Ilm.PutLifecycle(policyname)
 	cfg := conf.Ilm
 
@@ -231,6 +349,16 @@ func IlmCreate(conf *cfg.Config, policyname string) error {
 		rollover := &types.RolloverAction{}
 		haveroll := false
 
+		if policy != nil {
+			// update
+			actions = policy.Phases.Hot.Actions
+
+			if policy.Phases.Hot.Actions.Rollover != nil {
+				rollover = policy.Phases.Hot.Actions.Rollover
+				haveroll = true
+			}
+		}
+
 		if cfg.HotMinAge != "" {
 			hot.MinAge = cfg.HotMinAge
 		}
@@ -255,12 +383,22 @@ func IlmCreate(conf *cfg.Config, policyname string) error {
 
 		hot.Actions = actions.IlmActionsCaster()
 		phases.PhasesCaster().Hot = &hot
+	} else {
+		if policy != nil {
+			// update
+			phases.PhasesCaster().Hot = policy.Phases.Hot
+		}
 	}
 
 	if cfg.HaveWarm() {
 		warm := types.Phase{}
 		var actions types.IlmActionsVariant = esdsl.NewIlmActions()
 
+		if policy != nil {
+			// update
+			actions = policy.Phases.Warm.Actions
+		}
+
 		if cfg.WarmForceMerge != 0 {
 			actions.IlmActionsCaster().Forcemerge = &types.ForceMergeAction{MaxNumSegments: cfg.WarmForceMerge}
 		}
@@ -279,12 +417,22 @@ func IlmCreate(conf *cfg.Config, policyname string) error {
 
 		warm.Actions = actions.IlmActionsCaster()
 		phases.PhasesCaster().Warm = &warm
+	} else {
+		if policy != nil && policy.Phases.Warm != nil {
+			// update
+			phases.PhasesCaster().Warm = policy.Phases.Warm
+		}
 	}
 
 	if cfg.HaveCold() {
 		cold := types.Phase{}
 		var actions types.IlmActionsVariant = esdsl.NewIlmActions()
 
+		if policy != nil {
+			// update
+			actions = policy.Phases.Cold.Actions
+		}
+
 		if cfg.ColdForceMerge != 0 {
 			actions.IlmActionsCaster().Forcemerge = &types.ForceMergeAction{MaxNumSegments: cfg.ColdForceMerge}
 		}
@@ -304,12 +452,22 @@ func IlmCreate(conf *cfg.Config, policyname string) error {
 
 		cold.Actions = actions.IlmActionsCaster()
 		phases.PhasesCaster().Cold = &cold
+	} else {
+		if policy != nil && policy.Phases.Cold != nil {
+			// update
+			phases.PhasesCaster().Cold = policy.Phases.Cold
+		}
 	}
 
 	if cfg.HaveFrozen() {
 		froze := types.Phase{}
 		var actions types.IlmActionsVariant = esdsl.NewIlmActions()
 
+		if policy != nil {
+			// update
+			actions = policy.Phases.Frozen.Actions
+		}
+
 		if cfg.FrozenMinAge != "" {
 			froze.MinAge = cfg.FrozenMinAge
 		}
@@ -321,6 +479,11 @@ func IlmCreate(conf *cfg.Config, policyname string) error {
 
 		froze.Actions = actions.IlmActionsCaster()
 		phases.PhasesCaster().Frozen = &froze
+	} else {
+		if policy != nil && policy.Phases.Frozen != nil {
+			// update
+			phases.PhasesCaster().Frozen = policy.Phases.Frozen
+		}
 	}
 
 	if cfg.HaveDelete() {
@@ -328,6 +491,11 @@ func IlmCreate(conf *cfg.Config, policyname string) error {
 		delete := types.DeleteAction{}
 		var actions types.IlmActionsVariant = esdsl.NewIlmActions()
 
+		if policy != nil {
+			// update
+			actions = policy.Phases.Delete.Actions
+		}
+
 		if cfg.DeleteMinAge != "" {
 			del.MinAge = cfg.DeleteMinAge
 		}
@@ -339,16 +507,21 @@ func IlmCreate(conf *cfg.Config, policyname string) error {
 		actions.IlmActionsCaster().Delete = &delete
 		del.Actions = actions.IlmActionsCaster()
 		phases.PhasesCaster().Delete = &del
+	} else {
+		if policy != nil && policy.Phases.Delete != nil {
+			// update
+			phases.PhasesCaster().Delete = policy.Phases.Delete
+		}
 	}
 
 	put := &putlifecycle.Request{}
-	policy := &types.IlmPolicy{}
+	newpolicy := &types.IlmPolicy{}
-	policy.IlmPolicyCaster().Phases = *phases.PhasesCaster()
+	newpolicy.IlmPolicyCaster().Phases = *phases.PhasesCaster()
-	put.Policy = policy
+	put.Policy = newpolicy
 
 	ilm.Request(put)
 
-	_, err := ilm.Do(context.Background())
+	_, err = ilm.Do(context.Background())
 	if err != nil {
 		return fmt.Errorf("failed create ilm policy: %s", esErrorString(err))
 	}

pkg/es/ilm_forecast.go

@@ -190,9 +190,9 @@ func getIlmPhaseData(conf *cfg.Config) ([]PhaseData, error) {
 	wg := &sync.WaitGroup{}
 	wg.Add(3)
 
-	go getApiData(conf.DefaultCluster.ES(), wg, responses, "indicesbytes")
+	go getApiData(conf, conf.DefaultCluster.ES(), wg, responses, "indicesbytes")
-	go getApiData(conf.DefaultCluster.ES(), wg, responses, "explain")
+	go getApiData(conf, conf.DefaultCluster.ES(), wg, responses, "explain")
-	go getApiData(conf.DefaultCluster.ES(), wg, responses, "policies")
+	go getApiData(conf, conf.DefaultCluster.ES(), wg, responses, "policies")
 
 	wg.Wait()
 

pkg/es/index_template_settings.go

@@ -28,7 +28,7 @@ type Tpl struct {
 }
 
 func getIndexTemplateSettings(conf *cfg.Config, tplname string, table *printer.Table) error {
-	raw, err := CallAPI(conf, "GET", "/_index_template/"+tplname, "")
+	raw, err := CallAPI(conf, "GET", "/_index_template/"+tplname+"?flat_settings", "")
 	if err != nil {
 		return err
 	}
@@ -39,9 +39,11 @@ func getIndexTemplateSettings(conf *cfg.Config, tplname string, table *printer.T
 	}
 
 	if conf.Debug {
-		if err := prettyfiJson(conf, raw); err != nil {
+		output, err := prettyfiJson(conf, raw)
+		if err != nil {
 			return err
 		}
+		fmt.Println(output)
 	}
 
 	if len(data.IndexTemplates) == 0 {
@@ -50,11 +52,7 @@ func getIndexTemplateSettings(conf *cfg.Config, tplname string, table *printer.T
 
 	tpl := data.IndexTemplates[0].IndexTemplate.Template.Settings
 	for topic, val := range tpl {
-		paths := getJsonPath(map[string]string{}, val.(map[string]any), topic)
+		table.Entries = append(table.Entries, []string{topic, fmt.Sprintf("%v", val)})
-
-		for setting, value := range paths {
-			table.Entries = append(table.Entries, []string{setting, fmt.Sprintf("%v", value)})
-		}
 	}
 
 	return nil

pkg/es/license.go

@@ -0,0 +1,69 @@
+/*
+Copyright © 2026 Thomas von Dein
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+package es
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+
+	"codeberg.org/scip/esctl/pkg/cfg"
+	"codeberg.org/scip/esctl/pkg/printer"
+)
+
+func LicenseShow(conf *cfg.Config) error {
+	res, err := conf.DefaultCluster.ES().License.Get().
+		Do(context.Background())
+	if err != nil {
+		return fmt.Errorf("failed to get license: %s", esErrorString(err))
+	}
+
+	slog.Debug("license show", "license", res)
+
+	table := printer.NewTableEmpty(conf).WithHeaders("license setting", "value")
+
+	lic := res.License
+
+	var maxnodes = "infinite"
+	var maxunits = "infinite"
+	var expire = "never"
+
+	if lic.MaxNodes != nil {
+		maxnodes = fmt.Sprintf("%d", *lic.MaxNodes)
+	}
+
+	if lic.ExpiryDate != nil {
+		expire = lic.ExpiryDate.(string)
+	}
+
+	if lic.MaxResourceUnits != nil {
+		maxunits = fmt.Sprintf("%d", *lic.MaxResourceUnits)
+	}
+
+	table.Entries = [][]string{
+		{"UID", lic.Uid},
+		{"Issued to", lic.IssuedTo},
+		{"Expires", expire},
+		{"Issued", lic.IssueDate.(string)},
+		{"Max nodes", maxnodes},
+		{"Max resource units", maxunits},
+		{"Type", lic.Type.Name},
+		{"Status", lic.Status.Name},
+	}
+
+	return table.Print()
+}

pkg/es/node.go

@@ -20,9 +20,12 @@ import (
 	"context"
 	"fmt"
 	"log/slog"
+	"strings"
+	"time"
 
 	"codeberg.org/scip/esctl/pkg/cfg"
 	"codeberg.org/scip/esctl/pkg/printer"
+	"github.com/dustin/go-humanize"
 )
 
 func NodeList(conf *cfg.Config) error {
@@ -56,3 +59,145 @@ func NodeList(conf *cfg.Config) error {
 
 	return nil
 }
+
+func NodeNames(conf *cfg.Config) ([]string, error) {
+	nodes, err := conf.DefaultCluster.ES().Cat.Nodes().
+		Do(context.Background())
+	if err != nil {
+		return nil, fmt.Errorf("failed to get nodes: %s", esErrorString(err))
+	}
+
+	slog.Debug("ES result", "nodes", nodes)
+
+	nodelist := make([]string, len(nodes))
+
+	for idx, node := range nodes {
+		nodelist[idx] = *node.Name
+	}
+
+	return nodelist, err
+}
+
+// FIXME: adding the settings metric leads to json unmarshall error:
+// https://github.com/elastic/go-elasticsearch/issues/1524
+func NodeShow(conf *cfg.Config, nodename string) error {
+	res, err := conf.DefaultCluster.ES().Nodes.Info().
+		NodeId(nodename).
+		Metric("os, jvm, thread_pool, remote_cluster_server").
+		Do(context.Background())
+	if err != nil {
+		return fmt.Errorf("failed to get node info: %s", esErrorString(err))
+	}
+
+	slog.Debug("ES result", "node", res)
+
+	stats, err := conf.DefaultCluster.ES().Nodes.Stats().
+		NodeId(nodename).
+		Do(context.Background())
+	if err != nil {
+		return fmt.Errorf("failed to get node stats: %s", esErrorString(err))
+	}
+
+	slog.Debug("ES result", "stat", stats)
+
+	for id, info := range res.Nodes {
+		stat := stats.Nodes[id]
+
+		table := printer.NewTable(conf, 2, 0)
+		table.Addheaders(nodename+" property", "value")
+
+		roles := make([]string, len(info.Roles))
+		for idx, role := range info.Roles {
+			roles[idx] = role.Name
+		}
+
+		k8snode := info.Attributes["k8s_node_name"]
+
+		table.Entries = [][]string{
+			{"Id", id},
+			{"Name", nodename},
+			{"Kubernetes node", k8snode},
+			{"Ip address", info.Ip},
+			{"Node rank", *stat.AdaptiveSelection[id].Rank},
+			{"JVM", info.Jvm.VmName + " " + info.Jvm.Version},
+			{"JVM Started", time.UnixMilli(info.Jvm.StartTimeInMillis).String()},
+			{"OS", info.Os.PrettyName + " " + info.Os.Version},
+			{"Node roles", strings.Join(roles, ",")},
+			{"Node version", info.Version},
+			{"HTTP clients", fmt.Sprintf("%d", *stat.Http.CurrentOpen)},
+			{"CPUs", fmt.Sprintf("%d", *info.Os.AllocatedProcessors)},
+			{"Load 15m/5m/1m", fmt.Sprintf("%.2f/%.2f/%.2f",
+				stat.Os.Cpu.LoadAverage["15m"],
+				stat.Os.Cpu.LoadAverage["5m"],
+				stat.Os.Cpu.LoadAverage["1m"],
+			)},
+			{"Open FD's", fmt.Sprintf("%d", *stat.Process.OpenFileDescriptors)},
+			{"Response time avg", fmt.Sprintf("%dns", *stat.AdaptiveSelection[id].AvgResponseTimeNs)},
+			{"Memory usage (used/avail)",
+				humanize.Bytes(uint64(*stat.Os.Mem.UsedInBytes)) + " / " + humanize.Bytes(uint64(*stat.Os.Mem.TotalInBytes))},
+		}
+
+		if len(stat.Fs.Data) > 0 {
+			fs := stat.Fs.Data[0]
+			table.Entries = append(table.Entries, [][]string{
+				{"Storage usage (used/avail)",
+					humanize.Bytes(uint64(*fs.AvailableInBytes)) + " / " + humanize.Bytes(uint64(*fs.TotalInBytes))},
+				{"Storage mount", *fs.Mount},
+			}...)
+		}
+
+		if err := table.Print(); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func NodeClients(conf *cfg.Config, nodename string) error {
+	stats, err := conf.DefaultCluster.ES().Nodes.Stats().
+		NodeId(nodename).
+		Do(context.Background())
+	if err != nil {
+		return fmt.Errorf("failed to get node stats: %s", esErrorString(err))
+	}
+
+	slog.Debug("ES result", "stat", stats)
+
+	table := printer.NewTable(conf, 5, 0)
+	table.Addheaders("agent", "id", "when", "from host", "url")
+
+	for _, stat := range stats.Nodes {
+		for _, client := range stat.Http.Clients {
+			if client.ClosedTimeMillis == nil {
+				agent := ""
+				if client.Agent != nil {
+					agent = *client.Agent
+				}
+
+				uri := ""
+				if client.LastUri != nil {
+					uri = *client.LastUri
+
+					if !conf.All {
+						parts := strings.Split(uri, "?")
+						uri = parts[0]
+					}
+				}
+
+				table.AddRow(
+					agent,
+					fmt.Sprintf("%d", *client.Id),
+					time.UnixMilli(*client.LastRequestTimeMillis).String(),
+					*client.RemoteAddress,
+					uri,
+				)
+			}
+		}
+
+		break
+	}
+
+	table.Sort()
+	return table.Print()
+}

pkg/es/parallel.go

@@ -18,10 +18,10 @@ package es
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"sync"
 
+	"codeberg.org/scip/esctl/pkg/cfg"
 	"github.com/elastic/go-elasticsearch/v9"
 	"github.com/elastic/go-elasticsearch/v9/typedapi/cat/indices"
 	"github.com/elastic/go-elasticsearch/v9/typedapi/cat/tasks"
@@ -43,12 +43,14 @@ const (
 	ResponseTasks
 	ResponseExplain
 	ResponseLifecycle
+	ResponseHealthReport
 )
 
 type apiResponse struct {
 	error            error
 	info             *info.Response
 	health           *health.Response
+	healthreport     *HealthReport
 	ccr              *stats.Response
 	stats            *clusterstats.Response
 	indices          *indices.Response
@@ -59,12 +61,16 @@ type apiResponse struct {
 	which            int
 }
 
-func getApiData(es *elasticsearch.TypedClient, wg *sync.WaitGroup,
+func getApiData(
-	reschan chan apiResponse, which string) {
+	conf *cfg.Config,
+	es *elasticsearch.TypedClient,
+	wg *sync.WaitGroup,
+	reschan chan apiResponse,
+	which string) {
 	defer wg.Done()
 
 	ar := apiResponse{}
-	arerr := errors.New("")
+	var arerr error
 
 	switch which {
 	case "health":
@@ -75,6 +81,13 @@ func getApiData(es *elasticsearch.TypedClient, wg *sync.WaitGroup,
 		ar.which = ResponseHealth
 		arerr = err
 
+	case "healthreport":
+		report, err := getHealthReport(conf)
+
+		ar.healthreport = report
+		ar.which = ResponseHealthReport
+		arerr = err
+
 	case "info":
 		res, err := es.Info().
 			Do(context.Background())
@@ -83,7 +96,7 @@ func getApiData(es *elasticsearch.TypedClient, wg *sync.WaitGroup,
 		ar.which = ResponseInfo
 		arerr = err
 
-	case "ccrstats":
+	case "ccr":
 		res, err := es.Ccr.Stats().
 			Do(context.Background())
 

pkg/es/search.go

@@ -26,7 +26,6 @@ import (
 
 	"codeberg.org/scip/esctl/pkg/cfg"
 	"codeberg.org/scip/esctl/pkg/printer"
-	"github.com/alecthomas/repr"
 	"github.com/elastic/go-elasticsearch/v9/typedapi/core/search"
 	"github.com/elastic/go-elasticsearch/v9/typedapi/esdsl"
 	"github.com/elastic/go-elasticsearch/v9/typedapi/indices/validatequery"
@@ -149,25 +148,6 @@ func validateSearch(conf *cfg.Config, queries []string) error {
 	return nil
 }
 
-func Debug(conf *cfg.Config) error {
-	res, err := conf.DefaultCluster.ES().Search().
-		Index(conf.Index).
-		Size(0).
-		Aggregations(map[string]types.Aggregations{
-			"min_ts": *esdsl.NewMinAggregation().Field("@timestamp").AggregationsCaster(),
-			"max_ts": *esdsl.NewMaxAggregation().Field("@timestamp").AggregationsCaster(),
-		}).
-		Do(context.Background())
-
-	if err != nil {
-		return err
-	}
-
-	repr.Println(res)
-
-	return nil
-}
-
 func searchOnce(conf *cfg.Config, search *search.Search) error {
 	res, err := search.
 		From(conf.From).

pkg/printer/doc.go

@@ -52,6 +52,6 @@ func PrintDoc(conf *cfg.Config, hit types.Hit) {
 		value := gjson.Get(docjson, conf.Path)
 		fmt.Println(value.String())
 	} else {
-		fmt.Print(docjson)
+		fmt.Println(docjson)
 	}
 }

pkg/printer/table.go

@@ -28,6 +28,7 @@ import (
 	"github.com/olekukonko/tablewriter"
 	"github.com/olekukonko/tablewriter/renderer"
 	"github.com/olekukonko/tablewriter/tw"
+	"github.com/seeruk/go-wordwrap"
 	"gopkg.in/yaml.v3"
 )
 
@@ -38,10 +39,11 @@ type Table struct {
 
 	lenHeaders []int
 	alignInts  bool
+	maxwidth   int
 }
 
 func NewTable(conf *cfg.Config, columns, rows int) *Table {
-	table := Table{Mode: conf.Output}
+	table := Table{Mode: conf.Output, maxwidth: cfg.GetTermWidth()}
 
 	table.Headers = make([]string, columns)
 	table.Entries = make([][]string, rows)
@@ -51,6 +53,24 @@ func NewTable(conf *cfg.Config, columns, rows int) *Table {
 	return &table
 }
 
+func NewTableEmpty(conf *cfg.Config) *Table {
+	table := Table{Mode: conf.Output, maxwidth: cfg.GetTermWidth()}
+	table.alignInts = conf.AlignInts
+	return &table
+}
+
+func (table *Table) WithHeaders(headers ...string) *Table {
+	count := len(headers)
+
+	table.Entries = [][]string{}
+	table.lenHeaders = make([]int, count)
+	table.Headers = make([]string, count)
+
+	table.Addheaders(headers...)
+
+	return table
+}
+
 func (data *Table) Print() error {
 	switch data.Mode {
 	case "markdown", "md":
@@ -118,17 +138,31 @@ func (data *Table) PrintTSV() error {
 	}
 
 	for _, entries := range data.Entries {
+		currentWidth := 0
 		for idx, entry := range entries {
 			length := visibleLen(entry)
+
 			if data.lenHeaders[idx] < length {
+				if length > currentWidth+data.maxwidth {
+					data.lenHeaders[idx] = data.maxwidth - currentWidth
+				} else {
 					data.lenHeaders[idx] = length
 				}
 			}
+
+			currentWidth += data.lenHeaders[idx]
+		}
 	}
 
-	// output
+	// output headers
 	for idx, header := range data.Headers {
+		if idx+1 != len(data.Headers) {
 			fmt.Print(header, strings.Repeat(" ", data.lenHeaders[idx]-visibleLen(header)))
+		} else {
+			// no padding for last header
+			fmt.Print(header)
+		}
+
 		if idx < len(data.Headers)-1 {
 			fmt.Print("  ")
 		}
@@ -136,14 +170,37 @@ func (data *Table) PrintTSV() error {
 	fmt.Println()
 
 	for _, entries := range data.Entries {
+		currentWidth := 0
+
 		for idx, entry := range entries {
 			length := visibleLen(entry)
 
+			if length+currentWidth > data.maxwidth && data.maxwidth-currentWidth > 1 {
+				// text is too wide to be put into one line, wrap it
+				wrapper := wordwrap.Wrapper(data.maxwidth-currentWidth, false)
+				wrapped := wrapper(entry)
+
+				// and indent it
+				for idx, line := range strings.Split(wrapped, "\n") {
+					if idx == 0 {
+						entry = line
+					} else {
+						entry += "\n  " + strings.Repeat(" ", currentWidth) + line
+					}
+				}
+			}
+
+			currentWidth += data.lenHeaders[idx]
+
 			if isInt(entry) && data.alignInts {
 				// align right
 				fmt.Print(strings.Repeat(" ", data.lenHeaders[idx]-length), entry)
-			} else {
+			} else if length < data.lenHeaders[idx] && idx+1 != len(entries) {
+				// pad right, if required
 				fmt.Print(entry, strings.Repeat(" ", data.lenHeaders[idx]-length))
+			} else {
+				// no padding for last entry
+				fmt.Print(entry)
 			}
 
 			if idx < len(data.Headers)-1 {

t/.env

@@ -0,0 +1,6 @@
+ES_PASS=tuscador1
+STACK_VERSION=9.4.2
+CLUSTER_NAME=domdoc
+LICENSE=basic
+ES_PORT=9200
+MEM_LIMIT=1073741824

t/Makefile

@@ -0,0 +1,46 @@
+.PHONY: test clean cluster docs search up down delete
+
+# docker stuff
+up:
+	@echo "booting up docker containers"
+	docker compose up -d --remove-orphans
+
+waitup:
+	@echo "wait til es cluster is up and running"
+	mosscap msc-actions.yaml msc-docker.yaml -n -t 300
+
+down:
+	@echo "shutting down docker containers"
+	docker compose down
+
+delete:
+	@echo "delete docker volumes"
+	docker volume rm t_certs t_esdata01 t_esdata02 t_esdata03
+
+clean-docker: down delete
+
+
+# mosscap esctl stuff
+test: up waitup cluster docs wait search
+
+cluster:
+	@echo "setting up elastisearch cluster"
+	mosscap msc-actions.yaml msc-cluster.yaml -n
+
+docs:
+	@echo "put some docs"
+	mosscap msc-actions.yaml msc-docs.yaml -n -c 1000 -p 100
+
+wait:
+	sleep 10
+
+search:
+	@echo "make some searches"
+	mosscap msc-actions.yaml msc-search.yaml -n -t 60
+
+clean:
+	@echo "cleaning up cluster"
+	mosscap msc-actions.yaml msc-clean.yaml -n
+	rm -f *.state debug.log
+
+

t/README.md

@@ -0,0 +1,76 @@
+# Testing
+
+Building regular unit tests would require to create a mock
+elasticsearch cluster, which is too much a chrore for a one man
+show. Running a regular elasticsearch cluster on Codeberg CI is not
+economically reasonable.
+
+Therefore I run tests manually. Here's how.
+
+# Dependencies
+
+The following is required:
+
+- linux, any distro will do
+- docker in a decent version
+- [mosscap](https://codeberg.org/scip/mosscap) version `0.0.17` or higher.
+
+# Docker
+
+There's a ready to use docker compose file, which fires up an
+elasticsearch cluster with 3 nodes. To start it manually just execute
+`make up`.
+
+When it's up and running it should look like this:
+
+```console
+NAMES     STATUS                   PORTS
+t-es03-1  Up 16 minutes (healthy)  9200/tcp, 9300/tcp
+t-es02-1  Up 16 minutes (healthy)  9200/tcp, 9300/tcp
+t-es01-1  Up 16 minutes (healthy)  0.0.0.0:9200->9200/tcp, [::]:9200->9200/tcp, 9300/tcp
+```
+
+There's an esctl config file `cluster.yaml`, which you can use to
+access that elasticsearch cluster, e.g.:
+
+```console
+esctl -c cluster.yaml cluster status 
+DOCKER/DOMDOC              STATUS
+Cluster Name               domdoc
+ES Status                  green
+ES Version                 9.4.2
+Is Leader                  false
+Active Shards              8
+Active Primary Shards      4
+Unassigned Shards          0
+Unassigned Primary Shards  0
+Pending Tasks              0
+Nodes                      3
+Red Indices                0
+Long Running Tasks         0
+```
+
+# Automated Tests
+
+To execute the automated tests, just execute `make test`. This boots
+up the elasticsearch containers, waits until they are up and running,
+creates data on it and queries it.
+
+# Cleanup
+
+To just clean up the elasticsearch cluster run `make clean`.
+
+To remove everything, run `make clean-docker`.
+
+# Reference
+
+## Mosscap configs
+
+| CONFIG           | DESCRIPTION                                                          |
+|------------------|----------------------------------------------------------------------|
+| msc-actions.yaml | contains all actions, used by all item configs                       |
+| msc-docker.yaml  | single item: used to wait until elasticsearch containers are healthy |
+| msc-cluster.yaml | single item: create indices etc                                      |
+| msc-docs.yaml    | generator items: put docs into the index                             |
+| msc-search.yaml  | single item: do some searches                                        |
+| msc-clean.yaml   | single item: clean up the elasticsearch cluster                      |

t/cluster.yaml

@@ -0,0 +1,7 @@
+# esctl config
+clusters:
+    docker/domdoc:
+        uri: https://elastic:9200
+        user: elastic
+        token: ""
+        default: false

t/docker-compose.yaml

@@ -0,0 +1,207 @@
+services:
+  setup:
+    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
+    volumes:
+      - certs:/usr/share/elasticsearch/config/certs
+    user: "0"
+    command: >
+      bash -c '
+        if [ x${ES_PASS} == x ]; then
+          echo "Set the ES_PASS environment variable in the .env file";
+          exit 1;
+        fi;
+        if [ ! -f config/certs/ca.zip ]; then
+          echo "Creating CA";
+          bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
+          unzip config/certs/ca.zip -d config/certs;
+        fi;
+        if [ ! -f config/certs/certs.zip ]; then
+          echo "Creating certs";
+          echo -ne \
+          "instances:\n"\
+          "  - name: es01\n"\
+          "    dns:\n"\
+          "      - es01\n"\
+          "      - localhost\n"\
+          "    ip:\n"\
+          "      - 127.0.0.1\n"\
+          "  - name: es02\n"\
+          "    dns:\n"\
+          "      - es02\n"\
+          "      - localhost\n"\
+          "    ip:\n"\
+          "      - 127.0.0.1\n"\
+          "  - name: es03\n"\
+          "    dns:\n"\
+          "      - es03\n"\
+          "      - localhost\n"\
+          "    ip:\n"\
+          "      - 127.0.0.1\n"\
+          > config/certs/instances.yml;
+          bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
+          unzip config/certs/certs.zip -d config/certs;
+        fi;
+        echo "Setting file permissions"
+        chown -R root:root config/certs;
+        find . -type d -exec chmod 750 \{\} \;;
+        find . -type f -exec chmod 640 \{\} \;;
+        echo "Waiting for Elasticsearch availability";
+        until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
+        echo "All done!";
+      '
+    healthcheck:
+      test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
+      interval: 1s
+      timeout: 5s
+      retries: 120
+
+  es01:
+    depends_on:
+      setup:
+        condition: service_healthy
+    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
+    volumes:
+      - certs:/usr/share/elasticsearch/config/certs
+      - esdata01:/usr/share/elasticsearch/data
+    ports:
+      - ${ES_PORT}:9200
+    environment:
+      - node.name=es01
+      - cluster.name=${CLUSTER_NAME}
+      - cluster.initial_master_nodes=es01
+      - discovery.seed_hosts=es02
+      - ELASTIC_PASSWORD=${ES_PASS}
+      - bootstrap.memory_lock=true
+      - xpack.security.enabled=true
+      - xpack.security.http.ssl.enabled=true
+      - xpack.security.http.ssl.key=certs/es01/es01.key
+      - xpack.security.http.ssl.certificate=certs/es01/es01.crt
+      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
+      - xpack.security.transport.ssl.enabled=true
+      - xpack.security.transport.ssl.key=certs/es01/es01.key
+      - xpack.security.transport.ssl.certificate=certs/es01/es01.crt
+      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
+      - xpack.security.transport.ssl.verification_mode=certificate
+      - xpack.license.self_generated.type=${LICENSE}
+      - xpack.ml.use_auto_machine_memory_percent=true
+      - node_roles=data_content
+      - xpack.searchable.snapshot.shared_cache.size="1gb"
+      - cluster.routing.allocation.disk.threshold_enabled=false
+      - cluster.routing.allocation.disk.watermark.low="2gb"
+      - cluster.routing.allocation.disk.watermark.high="1gb"
+      - cluster.routing.allocation.disk.watermark.flood_stage="500mb"
+    mem_limit: ${MEM_LIMIT}
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
+        ]
+      interval: 10s
+      timeout: 10s
+      retries: 120
+
+  es02:
+    depends_on:
+      - es01
+    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
+    volumes:
+      - certs:/usr/share/elasticsearch/config/certs
+      - esdata02:/usr/share/elasticsearch/data
+    environment:
+      - node.name=es02
+      - cluster.name=${CLUSTER_NAME}
+      - cluster.initial_master_nodes=es01,es02
+      - discovery.seed_hosts=es01
+      - ELASTIC_PASSWORD=${ES_PASS}
+      - bootstrap.memory_lock=true
+      - xpack.security.enabled=true
+      - xpack.security.http.ssl.enabled=true
+      - xpack.security.http.ssl.key=certs/es02/es02.key
+      - xpack.security.http.ssl.certificate=certs/es02/es02.crt
+      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
+      - xpack.security.transport.ssl.enabled=true
+      - xpack.security.transport.ssl.key=certs/es02/es02.key
+      - xpack.security.transport.ssl.certificate=certs/es02/es02.crt
+      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
+      - xpack.security.transport.ssl.verification_mode=certificate
+      - xpack.license.self_generated.type=${LICENSE}
+      - xpack.ml.use_auto_machine_memory_percent=true
+      - cluster.routing.allocation.disk.threshold_enabled=false
+      - cluster.routing.allocation.disk.watermark.low="2gb"
+      - cluster.routing.allocation.disk.watermark.high="1gb"
+      - cluster.routing.allocation.disk.watermark.flood_stage="500mb"
+    mem_limit: ${MEM_LIMIT}
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
+        ]
+      interval: 10s
+      timeout: 10s
+      retries: 120
+
+
+  es03:
+    depends_on:
+      - es01
+    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
+    volumes:
+      - certs:/usr/share/elasticsearch/config/certs
+      - esdata03:/usr/share/elasticsearch/data
+    environment:
+      - node.name=es03
+      - cluster.name=${CLUSTER_NAME}
+      - cluster.initial_master_nodes=es01,es03
+      - discovery.seed_hosts=es01
+      - ELASTIC_PASSWORD=${ES_PASS}
+      - bootstrap.memory_lock=true
+      - xpack.security.enabled=true
+      - xpack.security.http.ssl.enabled=true
+      - xpack.security.http.ssl.key=certs/es03/es03.key
+      - xpack.security.http.ssl.certificate=certs/es03/es03.crt
+      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
+      - xpack.security.transport.ssl.enabled=true
+      - xpack.security.transport.ssl.key=certs/es03/es03.key
+      - xpack.security.transport.ssl.certificate=certs/es03/es03.crt
+      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
+      - xpack.security.transport.ssl.verification_mode=certificate
+      - xpack.license.self_generated.type=${LICENSE}
+      - xpack.ml.use_auto_machine_memory_percent=true
+      - cluster.routing.allocation.disk.threshold_enabled=false
+      - cluster.routing.allocation.disk.watermark.low="2gb"
+      - cluster.routing.allocation.disk.watermark.high="1gb"
+      - cluster.routing.allocation.disk.watermark.flood_stage="500mb"
+    mem_limit: ${MEM_LIMIT}
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
+        ]
+      interval: 10s
+      timeout: 10s
+      retries: 120
+      
+volumes:
+  certs:
+    driver: local
+  esdata01:
+    driver: local
+  esdata02:
+    driver: local
+  esdata03:
+    driver: local

t/msc-actions.yaml

@@ -0,0 +1,96 @@
+#
+# this is a mosscap action file, see: https://codeberg.org/scip/mosscap
+vars:
+  esctl: ../esctl -c cluster.yaml
+
+actions:
+  wait: |
+    sleep [[.time]]
+
+  docker_wait: |
+    for i in {1..20}; do
+      count=$(docker ps | grep healthy | wc -l)
+      if test $count -eq 3; then
+        echo "all containers are healthy"
+        exit
+      fi
+      sleep 5s
+    done
+
+    echo "not all containers are healthy"
+    docker ps
+    
+  cluster_ls: |
+    [[.esctl]] cluster ls | grep -E domdoc.*reachable.*yes
+  
+  cluster_status: |
+    [[.esctl]] cluster status | grep green
+
+  create_ilm: |
+    [[.esctl]] ilm create --hot-rollover-max-age 7d \
+                          --hot-rollover-max-primary-shard-size 25g \
+                          --warm-min-age 0 \
+                          --delete-min-age 14d [[.policy]]
+
+  describe_ilm: |
+    [[.esctl]] ilm show [[.policy]] -t | grep -E "rollover when storage > 25g"    
+
+  create_index_template: |
+    [[.esctl]] index template create [[.index]] -i [[.index]] \
+      -s number_of_shards:3 \
+      -s index.sort.field:@timestamp \
+      -r 1m --ilm-policy [[.policy]] \
+      user:keyword message:text @timestamp:date
+
+  describe_index_template: |
+    [[.esctl]] index template show [[.index]] | grep -E "index.sort.field.*@timestamp"
+    
+  create_index: |
+    [[.esctl]] index create -s 2 -r 2 [[.index]]
+
+  describe_index: |
+    [[.esctl]] index show [[.index]] | grep -E "fields.*@timestamp"
+
+  index_ls: |
+    [[.esctl]] index ls | grep [[.index]]
+
+  index_template_rm: |
+    [[.esctl]] index template rm [[.index]]
+    
+  index_rm: |
+    [[.esctl]] index rm [[.index]]
+
+  timestamp: |
+    date --iso-8601=second
+
+  date: |
+    date +%Y-%m-%d
+    
+  doc_add: |
+    [[.esctl]] doc add -i [[.index]] '{"user": "[[.user]]", "message":"[[.message]]", "@timestamp":"[[.ts]]"}'
+    
+  search_any: |
+    [[.esctl]] search -i [[.index]] -l 1 | jq .source.message
+
+  search_date: |
+    [[.esctl]] search -i [[.index]] -l 1 | jq '.source."@timestamp"' | grep [[.today]]
+
+  search_count: |
+    [[.esctl]] search -i [[.index]] -l 5000 | wc -l | grep 1000
+    
+  search_content: |
+    [[.esctl]] search -i [[.index]] [[.content_pattern]] -l 1 | jq .source.message
+
+  search_content_and: |
+    [[.esctl]] search -i [[.index]] [[.content_and_pattern]] -l 1 | jq .source.message
+
+  nodes: |
+    [[.esctl]] node ls | grep es01
+
+  describe_node: |
+    [[.esctl]] node show [[.node]] | grep "Node version"
+
+  license: |
+    [[.esctl]] license show | grep -E "Expires.*never"
+
+

t/msc-clean.yaml

@@ -0,0 +1,14 @@
+#
+# this is a mosscap item file, see: https://codeberg.org/scip/mosscap
+items:
+  - name: docker/domdoc
+
+vars:
+  index: test
+
+tasks:
+  - name: delete_index
+    action: index_rm
+
+  - name: delete_template
+    action: index_template_rm

t/msc-cluster.yaml

@@ -0,0 +1,48 @@
+#
+# this is a mosscap item file, see: https://codeberg.org/scip/mosscap
+statefile: cluster.state
+
+items:
+  - name: docker/domdoc
+
+vars:
+  index: test
+  node: es01
+  policy: testpolicy
+  
+tasks:
+  - name: ls
+    action: cluster_ls
+
+  - name: license
+    action: license
+    
+  - name: status
+    action: cluster_status
+
+  - name: ilm
+    action: create_ilm
+
+  - name: check_ilm
+    action: describe_ilm
+    
+  - name: template
+    action: create_index_template
+
+  - name: check_template
+    action: describe_index_template
+    
+  - name: index
+    action: create_index
+
+  - name: indexls
+    action: index_ls
+
+  - name: describe
+    action: describe_index
+
+  - name: nodes
+    action: nodes
+
+  - name: check_node
+    action: describe_node

t/msc-docker.yaml

@@ -0,0 +1,12 @@
+#
+# this is a mosscap item file, see: https://codeberg.org/scip/mosscap
+items:
+  - name: docker/domdoc
+
+vars:
+  index: test
+  time: 5s
+
+tasks:
+  - name: wait
+    action: docker_wait

t/msc-docs.yaml

@@ -0,0 +1,21 @@
+#
+# this is a mosscap generator item file, see: https://codeberg.org/scip/mosscap
+vars:
+  index: test
+
+tasks:
+  - name: ts
+    action: timestamp
+
+  - name: add
+    action: doc_add
+    args:
+      ts: ts.result
+  
+generate:
+  user: $name
+  # will become something like:
+  # "content34 iBAXQ2Va Ko5gMwwu FEH2O99B"
+  # so we can search for "content3*" and get multiple matches
+  message: "content$int8 $rand make$int8 $rand"
+  

t/msc-search.yaml

@@ -0,0 +1,31 @@
+#
+# this is a mosscap item file, see: https://codeberg.org/scip/mosscap
+items:
+  - name: docker/domdoc
+
+vars:
+  index: test
+  time: 10s
+  content_pattern: "content3*"
+  content_and_pattern: "content3* make"
+
+tasks:
+  - name: today
+    action: date
+
+  - name: search_any
+    action: search_any
+
+  - name: date
+    action: search_date
+    args:
+      today: today.result
+
+  - name: count
+    action: search_count
+
+  - name: content
+    action: search_content
+
+  - name: content_and
+    action: search_content_and