Compare commits

..

22 Commits

Author SHA1 Message Date
T. von Dein
b573d63c54 enhance cluster status, add word wrap to wide table rows (#62) 2026-07-02 12:16:59 +02:00
T. von Dein
e3b96b3e3b add node show + node clients (#61) 2026-07-01 13:13:16 +02:00
T. von Dein
6436bcfb22 ask for pass if missing or use env pass, add token support (#60) 2026-07-01 10:58:39 +02:00
T. von Dein
f5c8a23589 add cluster reroute commands, fix usage width bug (#59)
fixes #36
2026-06-29 13:08:05 +02:00
T. von Dein
f9eded5f92 enhance ilm show, add ilm update, rename modify to update everywhere (#58) 2026-06-29 11:48:29 +02:00
T. von Dein
b7a87051c0 add 'ilm show -t' to show a better visualization of a ilm policy (#57) 2026-06-29 10:11:50 +02:00
T. von Dein
81987b75f2 turn --show-command-tree into sub command, hide hidden commands (#56) 2026-06-29 08:38:03 +02:00
34bf4fbed9 add ilm forecast example 2026-06-26 11:41:55 +02:00
T. von Dein
a87f6d0c4f add --hidden flag (default off) to 'index ls' and 'ilm forecast ls' (#55) 2026-06-26 11:32:53 +02:00
T. von Dein
bbe72a474b fix #53: shardsize to string, fix nil pointer, add cold phase (#54) 2026-06-26 11:25:30 +02:00
e3aa50af37 remove debug print 2026-06-26 10:07:42 +02:00
T. von Dein
d2a0812579 feature add ilm-explain (#52) 2026-06-26 10:02:41 +02:00
T. von Dein
6200b5e7b0 enhance/ilm-forecast (#51) 2026-06-26 09:08:14 +02:00
b1b701303c optimzie virtualAge() a little 2026-06-25 20:30:42 +02:00
8f2c68b5b6 fix index response, fix linter issues 2026-06-25 14:19:43 +02:00
d60983411e refactored large func 2026-06-25 14:16:35 +02:00
e1291d1d92 add ilm forecast show 2026-06-25 13:29:24 +02:00
5f60696b01 add ilm forecast data collection, generalized parallel api calls 2026-06-25 11:54:06 +02:00
T. von Dein
825b3ce5d1 fix #49: add ilm policy to index+index-template create+modify (#50) 2026-06-25 07:52:54 +02:00
75a94fade3 fix doc 2026-06-24 10:30:53 +02:00
3e36f97e4b upd documentation 2026-06-24 10:29:17 +02:00
1b9309bca0 upd readme 2026-06-24 09:44:21 +02:00
32 changed files with 2764 additions and 612 deletions

612
README.md
View File

@@ -56,154 +56,6 @@ Features:
esctl doc add -i foo '{"title":"curry in a hurry", "message":"australian thai"}'
```
## Usage
Command tree:
```console
api - api access and documentation
list - list index of API calls
show - show an API doc
repl - interactive API repl
ccr - manage cross cluster replication
status - cross cluster replication status (yaml config with 2 clusters required)
pause - pause shard allocation
resume - resume shard allocation
follower - manage ccr follower indices
show - show ccr follower index details
add - add ccr follower index
delete - delete ccr follower index
unfollow - unfollow ccr follower index
pause - pause ccr index to follow
resume - resume ccr index to follow
renew - renew ccr follower index
info - show ccr remote info
cluster - manage cluster[s]
status - show cluster status
switch - set current elasticsearch cluster
list - list configured clusters
settings - cluster settings management
list - show cluster settings
set - set|update cluster settings
datastream - manage data streams
list - list indicies
show - show details about an data stream
create - create a new data stream
delete - delete a data stream
rollover - roll over a data stream
doc - manage documents
add - add JSON document index
show - show a JSON document
delete - delete JSON document[s] from index[es]
ilm - manage index lifecycle
retry - retry applying an ILM profile to an index
status - get the current index lifecycle management status
list - list index lifecycle policies
show - show details about an index lifecycle policy
create - create a index lifecycle policy
index - manage indicies
list - list indicies
show - show details about an index
create - create a new index
delete - delete an index
close - close an index
allocation - explain index allocation
modify - modify an index
fields - show info about field capabilities
ilm - show ilm status
alias - manage index aliases
create - create an index alias
list - list index aliases
delete - delete an index alias
rollover - roll over an index alias
template - manage index templates
list - list index templates
show - show details about an index template
create - create a new index template
modify - modify a new index template
delete - delete an index template
node - manage nodes
list - list nodes
show - show details about a node
role - manage roles
list - list roles
show - show details about a role
diff - show differences between roles and CSV baseline
search - search within an index
shard - manage shards
list - list shards
show - show details about a shard
snapshot - manage snapshots
list - list snapshots
show - show details about a snapshot
task - manage tasks
list - list tasks
cancel - cancel running task
version - show esctl version information
debug - developer only
help-jsonpath - show jsonpath help
completion - Output shell completion script for bash, zsh, fish, or Powershell
bash - Output bash completion script
zsh - Output zsh completion script
fish - Output fish completion script
pwsh - Output pwsh completion script
```
Configure `esctl` with environment variables:
- `ES_URI`: elasticsearch uri
- `ES_USER`: username
- `ES_PASS`: password
Or create a config file such as this:
```yaml
clusters:
foobar:
uri: https://es.foo.bar:9200/
user: elastic
pass: 123456
other:
uri: https://myes.foo:9200/
user: elastic
pass: asdasdasd
```
and specify it with `-c configfile`. You may also put clusters into a
default config file in `~/.config/esctl/config.yaml`. In this case you
can omit `-c ...`.
If you want to work on a specific cluster, you need to make it the
current default one. You can either manually configure it in the
config:
```yaml
clusters:
foobar:
uri: https://es.foo.bar:9200/
user: elastic
pass: 123456
default: true
```
or - if the cluster already exists in the config - issue this command:
```console
esctl cluster switch foobar
```
You may also temporary set a cluster as the current default with the
global `-C` option.
The following rules apply for default cluster selection:
1. If there's just one cluster configured (either via environment
variables or config), this one will be selected.
2. If there is one cluster configured with the `default` flag `true`,
use this one.
Use `esctl cluster ls` to check status and see which one would be used.
## Installation
The tool does not have any dependencies. Just download the binary for
@@ -254,6 +106,470 @@ make
sudo make install
```
## Configure
Configure `esctl` with environment variables:
- `ES_URI`: elasticsearch uri
- `ES_USER`: username
- `ES_PASS`: password
- `ES_TOKEN`: API token, instead of user+password
Or create a config file such as this:
```yaml
clusters:
foobar:
uri: https://es.foo.bar:9200/
user: elastic
pass: ******
other:
uri: https://myes.foo:9200/
token: ******
```
and specify it with `-c configfile`. You may also put clusters into a
default config file in `~/.config/esctl/config.yaml`. In this case you
can omit `-c ...`.
If you want to work on a specific cluster, you need to make it the
current default one. You can either manually configure it in the
config:
```yaml
clusters:
foobar:
uri: https://es.foo.bar:9200/
user: elastic
pass: 123456
default: true
```
or - if the cluster already exists in the config - issue this command:
```console
esctl cluster switch foobar
```
You may also temporary set a cluster as the current default with the
global `-C` option.
The following rules apply for default cluster selection:
1. If there's just one cluster configured (either via environment
variables or config), this one will be selected.
2. If a cluster have been specified with `-C <name>`, use this.
3. If there is one cluster configured with the `default` flag `true`,
use this one.
Use `esctl cluster ls` to check status and see which one would be
used.
Please be aware, that prior to use a cluster and send API requests to
it, `esctl` checks the connectivity to it first.
## Usage
Once you have configured one or more (and marked one as the default,
see above) elasticsearch clusters, you can start using `esctl`. Since
there are just too many things you can do with it, we'll just outline
a couple of examples here.
To see the status of a cluster:
```console
$ esctl cluster status
DEFAULT STATUS
Cluster Name eck-cluster-dev-s1
ES Status green
ES Version 9.4.2
Is Leader true
Active Shards 447
Active Primary Shards 221
Unassigned Shards 0
Unassigned Primary Shards 0
Pending Tasks 0
Nodes 3
Red Indices 0
Long Running Tasks 2
```
you may also add the `-v` flag to see more:
```console
$ esctl cluster status -v
DEFAULT STATUS
Cluster Name eck-cluster-dev-s1
ES Status green
ES Version 9.4.2
Is Leader true
Active Shards 447
Active Primary Shards 221
Unassigned Shards 0
Unassigned Primary Shards 0
Pending Tasks 0
Nodes 3
Red Indices 0
Long Running Tasks 2
Indicies 221
Docs 9854777
Total Size 3.4 GB
Total Queries 4210411
Shards Primaries 824642680872
Shards Total 824642680808
Storage 6.9 GB/6.9 GB
JVM Heap Memory 16 GB/39 GB
JVM Threads 458
JVM Version OpenJDK 64-Bit Server VM 26.0.1+8-34
CPUs 18
CPU Usage 0%
Open FDs 1423
```
View cluster settings:
```console
$ esctl cluster settings list
SETTING VALUE
cluster.remote.svct13.mode proxy
cluster.remote.svct13.proxy_address 100.127.161.24:9300
cluster.remote.svct13.proxy_socket_connections 18
cluster.remote.svct13.skip_unavailable false
cluster.routing.allocation.enable all
indices.recovery.max_bytes_per_sec 320mb
```
Settings can be changed, use json path to specify a setting:
```console
$ esctl cluster settings set indices.recovery.max_bytes_per_sec:480mb
```
Searching is pretty easy. First check which indices exist:
```console
$ esctl index ls
NAME SIZE DOCSCOUNT
[..]
foo 249b 0
foo2 31.4kb 14
mock 9.6kb 20
mr-strange 249b 0
silo18 5.2kb 1
```
Like a dataview in kibana you can specify an index pattern for search:
```console
$ esctl search -i foo*
{"id":"1779873299", "score":1.0000, "index":"foo2", "source":{"message":"foo bar"}}{"id":"1779870997", "score":1.0000, "index":"foo2", "source":{"message":"start","time":"27.05.2026T10:55:33+0200","title":"zeitbuchung"}}{"id":"1779870750", "score":1.0000, "index":"foo2", "source":{"message":"pause","time":"27.05.2026T10:33:45+0200","title":"zeitbuchung"}}{"id":"1779870103", "score":1.0000, "index":"foo2", "source":{"message":"punktlich gebucht","time":"27.05.2026T10:20:01+0200","title":"zeitbuchuung"}}{"id":"1779432806", "score":1.0000, "index":"foo2", "source":{"message":"strange compound 2"}}{"id":"1779432750", "score":1.0000, "index":"foo2", "source":{"message":"strange compound 1"}}{"id":"1778600947", "score":1.0000, "index":"foo2", "source":{"id":"d8d8d","user":"scip"}}{"id":"1778600945", "score":1.0000, "index":"foo2", "source":{"id":"d8d8d","user":"scip"}}{"id":"1778600857", "score":1.0000, "index":"foo2", "source":{"id":"d8d8d","user":"scip"}}{"id":"1778601788", "score":1.0000, "index":"foo2", "source":{"id":"new1","message":"live long and prosper"}}{"id":"1778601422", "score":1.0000, "index":"foo2", "source":{"id":"new1","message":"love long and prosper"}}{"id":"1779447242", "score":1.0000, "index":"foo2", "source":{"message":"australian thai","title":"curry in a hurry"}}{"id":"1779447284", "score":1.0000, "index":"foo2", "source":{"message":"german killer food","title":"wuerger burger"}}{"id":"1779364394", "score":1.0000, "index":"foo2", "source":{"id":"1","message":"non-dead human looks curious at TargetOne"}}
```
Oops, that looks gibberish. It's because `esctl` just prints the raw
JSON search results. There are multiple ways to post-process them:
You can use `jq` (here, we're using `-l 1` to limit the output to 1 document):
```console
$ esctl search -i foo* -l 1 | jq
{
"id": "1779364394",
"score": 1.0000,
"index": "foo2",
"source": {
"id": "1",
"message": "non-dead human looks curious at TargetOne"
}
}
```
However, you might as well use internal filtering using jsonpaths:
```console
$ esctl search -i foo* -l 1 -p source.message
non-dead human looks curious at TargetOne
```
To get more help about internal jsonpath filtering, run `esctl
help-jsonpath`.
A full text search looks like this:
```console
$ esctl search -i foo* thai | jq
{
"id": "1779447242",
"score": 2.3842,
"index": "foo2",
"source": {
"message": "australian thai",
"title": "curry in a hurry"
}
}
```
Instead of a costly full text you might as well specify the field to
search within:
```console
$ esctl search -i foo* message=thai | jq
{
"id": "1779447242",
"score": 2.3842,
"index": "foo2",
"source": {
"message": "australian thai",
"title": "curry in a hurry"
}
}
```
Of course you can search for multiple fields (or multiple terms in
full text search). By default search terms are applied using logical
AND, which you can change with the option `--or`.
You can also change the sorting (`--sort-by`) and the time range
(`--timerange`).
To further narrow down search results you can add filters, which must
match literally:
```console
$ esctl search -i foo* -F title=zeitbuchung message=pause | jq
{
"id": "1779870750",
"score": 2.8426,
"index": "foo2",
"source": {
"message": "pause",
"time": "27.05.2026T10:33:45+0200",
"title": "zeitbuchung"
}
}
```
To check which field mappings are available for an index:
```console
$ esctl index show foo2
INDEX-PROPERTY VALUE
name foo2
replicas 1
shards 1
created 2026-05-12 17:47:37
uuid XmRtlh_JSreIzTB7jJNNJA
fields id,message,time,title,user
```
Now, let's create a new index:
```console
$ esctl index create hyperdrive @timestamp:date message:keyword tag:text
esctl index show hyperdrive
INDEX-PROPERTY VALUE
name hyperdrive
replicas 1
shards 1
created 2026-06-24 10:16:23
uuid gYfUj8TiR6aMB_BHmMYr3Q
fields @timestamp,message,tag
```
Once we have an index, we can insert documents into it (you'll need
[gfn](https://codeberg.org/scip/gfn) for this):
```console
$ for name in $(gfn JapaneseNamesDiverse -n 20); do \
esctl doc add -i hyperdrive \
"{\"message\":\"$name\",\"tag\":\"loop\",\"@timestamp\":\"$(date
--iso-8601=second)\"}"; \
sleep 1; \
done
3644770736267480972
1249004675657072369
5499556813583921546
947619563748335783
6804953496467561176
1716813071323879201
463762522269463473
4674152788118344945
7741776772191747414
8531321287239151007
3335734000533108391
7020487792409206103
4926995290680601261
374509677635815298
49895974689085266
7603943644788310
2102931816420108217
696238814931581311
2463257155346426021
596556586001412147
```
Now, let's search them:
```console
$ esctl search -i hyperdrive -l 2 | jq
{
"id": "2463257155346426021",
"score": 0.0000,
"index": "hyperdrive",
"source": {
"@timestamp": "2026-06-24T10:25:19+02:00",
"message": "inenen",
"tag": "loop"
}
}
{
"id": "596556586001412147",
"score": 0.0000,
"index": "hyperdrive",
"source": {
"@timestamp": "2026-06-24T10:25:20+02:00",
"message": "ominuhen",
"tag": "loop"
}
}
```
There are also some uniq features which are not directly available via
the ES API or GUI. Here's one example: say you have a number of
indices with ILM policies each. At some day there were a surge in
incoming data in some of them and you want to know, when the excess
data will be rolled over to cold storage:
```console
$ esctl ilm forecast sh -f warm -w 3d
1.1 TB bytes of data in warm phase will be rolled within 72h0m0s to the next phase
```
You may also look at a detailed ilm forecast list:
```console
$ esctl ilm forecast ls -f warm -w 3d
INDEX CURRENT-SIZE CURRENT-AGE VIRTUAL-AGE MIN-AGE MIN-SIZE CURRENT-PHASE NEXT-PHASE
foobar-n1-p01-elastic-000126 12 GB 2d:8h:12m 3d:5h:16m 7d 25 GB hot warm
foobar-n1-p01-misc-000070 16 GB 5d:16h:31m 4d:10h:31m 7d 25 GB hot warm
foobar-n1-q01-elastic-000122 23 GB 5d:0h:11m 6d:10h:33m 7d 25 GB hot warm
foobar-n1-q01-misc-000072 18 GB 6d:15h:21m 4d:21h:36m 7d 25 GB hot warm
delaware-f1-p01-kafka-000023 1.3 MB 15h:12m 0s 7d 25 GB hot warm
delaware-f1-p01-misc-000024 17 GB 6d:16h:1m 4d:16h:33m 7d 25 GB hot warm
delaware-f1-q01-kafka-000023 1.5 MB 15h:12m 0s 7d 25 GB hot warm
delaware-f1-q01-misc-000024 16 GB 6d:15h:31m 4d:13h:52m 7d 25 GB hot warm
delaware-n1-p01-elastic-000417 16 GB 12h:42m 4d:10h:31m 7d 25 GB hot warm
```
So you can see, which index will be rolled over when. Note the
**VIRTUAL-AGE** field however: it is calculated from the current
storage usage of the index in relation to rollover max shard size. So
you can see, when an index will be rolled over either because it aged
out or because its storage exceeded the limit.
---
Please note, that `esctl` is still in its early stages and things are
changing heavily every now and then. New commands are being added
constantly as well.
### Command tree:
```console
api - api access and documentation
list - list index of API calls
show - show an API doc
repl - interactive API repl
ccr - manage cross cluster replication
status - cross cluster replication status (yaml config with 2 clusters required)
pause - pause shard allocation
resume - resume shard allocation
follower - manage ccr follower indices
show - show ccr follower index details
add - add ccr follower index
delete - delete ccr follower index
unfollow - unfollow ccr follower index
pause - pause ccr index to follow
resume - resume ccr index to follow
renew - renew ccr follower index
info - show ccr remote info
cluster - manage cluster[s]
status - show cluster status
switch - set current elasticsearch cluster
list - list configured clusters
settings - cluster settings management
list - show cluster settings
set - set|update cluster settings
reroute - manually change the allocation of individual shards in the cluster.
move - move shard to another node
allocate-replica - allocate-replica replica to another node
cancel - cancel a reroute operation
allocate-empty-primary - allocate an empty primary shard to a node
allocate-stale-primary - allocate a stale primary shard to a node
datastream - manage data streams
list - list indicies
show - show details about an data stream
create - create a new data stream
delete - delete a data stream
rollover - roll over a data stream
doc - manage documents
add - add JSON document index
show - show a JSON document
delete - delete JSON document[s] from index[es]
ilm - manage index lifecycle
retry - retry applying an ILM profile to an index
status - get the current index lifecycle management status
list - list index lifecycle policies
show - show details about an index lifecycle policy
create - create a new lifecycle policy
update - update an lifecycle policy
forecast - calculate index phase movements
list - list index rollover config
show - show rollover forecast over all indices
explain - explain ilm condition of an index
index - manage indicies
list - list indicies
show - show details about an index
create - create a new index
update - update an index
delete - delete an index
close - close an index
allocation - explain index allocation
fields - show info about field capabilities
ilm - show ilm status
alias - manage index aliases
create - create an index alias
list - list index aliases
delete - delete an index alias
rollover - roll over an index alias
template - manage index templates
list - list index templates
show - show details about an index template
create - create a new index template
update - update a new index template
delete - delete an index template
node - manage nodes
list - list nodes
show - show details about a node
clients - show node http clients
role - manage roles
list - list roles
show - show details about a role
diff - show differences between roles and CSV baseline
search - search within an index
shard - manage shards
list - list shards
show - show details about a shard
snapshot - manage snapshots
list - list snapshots
show - show details about a snapshot
task - manage tasks
list - list tasks
cancel - cancel running task
version - show esctl version information
debug - developer only
help-jsonpath - show jsonpath help
help-command-overview - show overview of all available commands
```
# Development
## To test completion

View File

@@ -37,6 +37,7 @@ func Cluster(conf *cfg.Config) *cli.Command {
ClusterSwitch(conf),
ClusterList(conf),
ClusterSettings(conf),
ClusterReroute(conf),
},
}
}
@@ -60,12 +61,6 @@ func ClusterStatus(conf *cfg.Config) *cli.Command {
Aliases: []string{"s"},
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "all",
Usage: "show status of all clusters",
Destination: &conf.All,
Aliases: []string{"a"},
},
&cli.BoolFlag{
Name: "verbose",
Usage: "include verbose statistics",

225
cmd/cluster_reroute.go Normal file
View File

@@ -0,0 +1,225 @@
/*
Copyright © 2026 Thomas von Dein
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package cmd
import (
"context"
"errors"
"codeberg.org/scip/esctl/pkg/cfg"
"codeberg.org/scip/esctl/pkg/es"
"github.com/urfave/cli/v3"
)
func ClusterReroute(conf *cfg.Config) *cli.Command {
return &cli.Command{
Name: "reroute",
Usage: "manually change the allocation of individual shards in the cluster.",
UsageText: "reroute <cluster-name>",
Aliases: []string{"ctx"},
Commands: []*cli.Command{
ClusterRerouteMove(conf),
ClusterRerouteAllocateReplica(conf),
ClusterRerouteCancel(conf),
ClusterRerouteAllocatePrimary(conf, false),
ClusterRerouteAllocatePrimary(conf, true),
},
}
}
func ClusterRerouteMove(conf *cfg.Config) *cli.Command {
return &cli.Command{
Name: "move",
Usage: "move shard to another node",
UsageText: "move [options] <index>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cindex)
},
Flags: []cli.Flag{
&cli.IntFlag{
Name: "shard",
Usage: "shard to move",
Destination: &conf.Shards,
Aliases: []string{"s"},
Required: true,
},
&cli.StringFlag{
Name: "from-node",
Usage: "current node",
Destination: &conf.FromNode,
Aliases: []string{"f"},
Required: true,
},
&cli.StringFlag{
Name: "to-node",
Usage: "node to move to",
Destination: &conf.ToNode,
Aliases: []string{"t"},
Required: true,
},
},
Action: func(ctx context.Context, cmd *cli.Command) error {
index := cmd.Args().Get(0)
if index == "" {
return errors.New("no index specified")
}
return es.ClusterRerouteMove(conf, index)
},
}
}
func ClusterRerouteAllocateReplica(conf *cfg.Config) *cli.Command {
return &cli.Command{
Name: "allocate-replica",
Usage: "allocate-replica replica to another node",
UsageText: "allocate-replica [options] <index>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cindex)
},
Flags: []cli.Flag{
&cli.IntFlag{
Name: "shard",
Usage: "shard to move",
Destination: &conf.Shards,
Aliases: []string{"s"},
Required: true,
},
&cli.StringFlag{
Name: "to-node",
Usage: "node to move to",
Destination: &conf.ToNode,
Aliases: []string{"t"},
Required: true,
},
},
Action: func(ctx context.Context, cmd *cli.Command) error {
index := cmd.Args().Get(0)
if index == "" {
return errors.New("no index specified")
}
return es.ClusterRerouteAllocateReplica(conf, index)
},
}
}
func ClusterRerouteCancel(conf *cfg.Config) *cli.Command {
return &cli.Command{
Name: "cancel",
Usage: "cancel a reroute operation",
UsageText: "cancel [options] <index>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cindex)
},
Flags: []cli.Flag{
&cli.IntFlag{
Name: "shard",
Usage: "shard to move",
Destination: &conf.Shards,
Aliases: []string{"s"},
Required: true,
},
&cli.StringFlag{
Name: "to-node",
Usage: "node to move to",
Destination: &conf.ToNode,
Aliases: []string{"t"},
Required: true,
},
&cli.BoolFlag{
Name: "allow-primary",
Usage: "allow primary shard to cancel",
Destination: &conf.AllowPrimary,
Aliases: []string{"a"},
},
},
Action: func(ctx context.Context, cmd *cli.Command) error {
index := cmd.Args().Get(0)
if index == "" {
return errors.New("no index specified")
}
return es.ClusterRerouteCancel(conf, index)
},
}
}
func ClusterRerouteAllocatePrimary(conf *cfg.Config, stale bool) *cli.Command {
name := "allocate-empty-primary"
usage := "allocate an empty primary shard to a node"
if stale {
name = "allocate-stale-primary"
usage = "allocate a stale primary shard to a node"
}
return &cli.Command{
Name: name,
Usage: usage,
UsageText: name + " [options] <index>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cindex)
},
Flags: []cli.Flag{
&cli.IntFlag{
Name: "shard",
Usage: "shard to move",
Destination: &conf.Shards,
Aliases: []string{"s"},
Required: true,
},
&cli.StringFlag{
Name: "to-node",
Usage: "node to move to",
Destination: &conf.ToNode,
Aliases: []string{"t"},
Required: true,
},
&cli.BoolFlag{
Name: "accept-data-loss",
Usage: "",
Destination: &conf.AcceptDataLoss,
Aliases: []string{"a"},
Required: true,
},
},
Action: func(ctx context.Context, cmd *cli.Command) error {
index := cmd.Args().Get(0)
if index == "" {
return errors.New("no index specified")
}
return es.ClusterRerouteAllocatePrimary(conf, index, stale)
},
}
}

View File

@@ -32,6 +32,7 @@ const (
Ccluster
Capi
Cilm
Cnode
)
func complete(cmd *cli.Command, what int) {
@@ -64,6 +65,8 @@ func complete(cmd *cli.Command, what int) {
list = es.ApiPathNames()
case Cilm:
list, err = es.IlmNames(conf)
case Cnode:
list, err = es.NodeNames(conf)
}
if err != nil {

View File

@@ -36,7 +36,10 @@ func Ilm(conf *cfg.Config) *cli.Command {
IlmStatus(conf),
IlmList(conf),
IlmShow(conf),
IlmCreate(conf),
IlmCreate(conf, false),
IlmCreate(conf, true),
IlmForecast(conf),
IlmExplain(conf),
},
}
}
@@ -53,6 +56,23 @@ func IlmRetry(conf *cfg.Config) *cli.Command {
}
}
func IlmExplain(conf *cfg.Config) *cli.Command {
return &cli.Command{
Name: "explain",
Usage: "explain ilm condition of an index",
UsageText: "explain <index>",
Action: func(ctx context.Context, cmd *cli.Command) error {
index := cmd.Args().Get(0)
if index == "" {
return errors.New("no index specified")
}
return es.IlmExplain(conf, index)
},
}
}
func IlmStatus(conf *cfg.Config) *cli.Command {
return &cli.Command{
Name: "status",
@@ -84,6 +104,15 @@ func IlmShow(conf *cfg.Config) *cli.Command {
Usage: "show details about an index lifecycle policy",
UsageText: "show <policy>",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "tree",
Usage: "display policy as a tree",
Destination: &conf.Ilm.Tree,
Aliases: []string{"t"},
},
},
Action: func(ctx context.Context, cmd *cli.Command) error {
policy := cmd.Args().Get(0)
if policy == "" {
@@ -99,72 +128,22 @@ func IlmShow(conf *cfg.Config) *cli.Command {
}
}
/*
*
{
"policy": {
"phases": {
"hot": {
"min_age": "0ms",
"actions": {
"rollover": {
"max_primary_shard_size": "50gb",
"max_age": "1d",
"max_docs": 100000000
},
"set_priority": {
"priority": 100
}
}
},
"warm": {
"min_age": "7d",
"actions": {
"migrate": {},
"shrink": {
"number_of_shards": 1
},
"forcemerge": {
"max_num_segments": 1
},
"set_priority": {
"priority": 50
}
}
},
"cold": {
"min_age": "30d",
"actions": {
"migrate": {},
"searchable_snapshot": {
"snapshot_repository": "s3-logs-archive",
"force_merge_index": true
},
"set_priority": {
"priority": 0
}
}
},
"delete": {
"min_age": "90d",
"actions": {
"delete": {
"delete_searchable_snapshot": true
}
}
}
}
}
}
func IlmCreate(conf *cfg.Config, modify bool) *cli.Command {
name := "create"
alias := "+"
usage := "create a new lifecycle policy"
*/
if modify {
name = "update"
usage = "update an lifecycle policy"
alias = "upd"
}
func IlmCreate(conf *cfg.Config) *cli.Command {
return &cli.Command{
Name: "create",
Aliases: []string{"+"},
Usage: "create a index lifecycle policy",
UsageText: "create [options] <policy>",
Name: name,
Aliases: []string{alias},
Usage: usage,
UsageText: name + " [options] <policy>",
Flags: []cli.Flag{
&cli.StringFlag{
@@ -185,7 +164,7 @@ func IlmCreate(conf *cfg.Config) *cli.Command {
Usage: "hot phase: rollover after",
Destination: &conf.Ilm.HotRolloverMaxAge,
},
&cli.Int64Flag{
&cli.StringFlag{
Name: "hot-rollover-max-primary-shard-size",
Usage: "hot phase: max primary shard size",
Destination: &conf.Ilm.HotRolloverMaxPrimaryShardSize,
@@ -251,6 +230,11 @@ func IlmCreate(conf *cfg.Config) *cli.Command {
Usage: "frozen phase: min age",
Destination: &conf.Ilm.FrozenMinAge,
},
&cli.StringFlag{
Name: "frozen-searchable-snapshot-repo",
Usage: "frozen phase: searchable snapshot repo",
Destination: &conf.Ilm.FrozenSearchableSnapshotRepo,
},
// delete
&cli.StringFlag{

139
cmd/ilm_forecast.go Normal file
View File

@@ -0,0 +1,139 @@
/*
Copyright © 2026 Thomas von Dein
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package cmd
import (
"context"
"errors"
"slices"
"codeberg.org/scip/esctl/pkg/cfg"
"codeberg.org/scip/esctl/pkg/es"
"github.com/urfave/cli/v3"
)
func IlmForecast(conf *cfg.Config) *cli.Command {
return &cli.Command{
Name: "forecast",
Usage: "calculate index phase movements",
Commands: []*cli.Command{
IlmForecastList(conf),
IlmForecastShow(conf),
},
}
}
const IlmForeCastListReference = `<duration> must be a string which consists one or more of these elements:
d - days
h - hours
m - minutes
s - seconds
Examples:
58d12h - 58 days and 12 hours
122h30m - 122 hours and 30 minutes
Specs may be mixed:
8h12d - 12 days and 8 hours`
func IlmForecastList(conf *cfg.Config) *cli.Command {
return &cli.Command{
Name: "list",
Aliases: []string{"ls"},
Usage: "list index rollover config",
UsageText: "list [options] [<filter>]",
CustomHelpTemplate: addReference(IlmForeCastListReference),
Flags: []cli.Flag{
&cli.StringFlag{
Name: "from-phase",
Usage: "which ilm phase to forecast from",
Destination: &conf.Ilm.FromPhase,
Aliases: []string{"f"},
},
&cli.StringFlag{
Name: "within",
Usage: "<duration> withing which to forecast",
Destination: &conf.Ilm.Within,
Aliases: []string{"w"},
},
&cli.StringFlag{
Name: "min-age",
Usage: "show only indices older than <duration>",
Destination: &conf.Ilm.MinAge,
Aliases: []string{"m"},
},
&cli.BoolFlag{
Name: "verbose",
Usage: "show ilm policy as well",
Destination: &conf.Verbose,
Aliases: []string{"v"},
},
&cli.BoolFlag{
Name: "hidden",
Usage: "include hidden indicies",
Destination: &conf.Hidden,
Aliases: []string{"H"},
},
},
Action: func(ctx context.Context, cmd *cli.Command) error {
valid := []string{"hot", "warm", "cold", "frozen"}
if !slices.Contains(valid, conf.Ilm.FromPhase) {
return errors.New("invalid from phase, allowed: hot, warm, cold, frozen")
}
return es.IlmForecastList(conf, cmd.Args().Get(0))
},
}
}
func IlmForecastShow(conf *cfg.Config) *cli.Command {
return &cli.Command{
Name: "show",
Aliases: []string{"sh"},
Usage: "show rollover forecast over all indices",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "from-phase",
Usage: "which ilm phase to forecast from",
Destination: &conf.Ilm.FromPhase,
Aliases: []string{"f"},
},
&cli.StringFlag{
Name: "within",
Usage: "duration withing which to forecast",
Destination: &conf.Ilm.Within,
Aliases: []string{"w"},
},
},
Action: func(ctx context.Context, cmd *cli.Command) error {
valid := []string{"hot", "warm", "cold", "frozen"}
if !slices.Contains(valid, conf.Ilm.FromPhase) {
return errors.New("invalid from phase, allowed: hot, warm, cold, frozen")
}
return es.IlmForecastShow(conf)
},
}
}

View File

@@ -36,11 +36,11 @@ func Index(conf *cfg.Config) *cli.Command {
Commands: []*cli.Command{
IndexList(conf),
IndexShow(conf),
IndexCreate(conf),
IndexCreate(conf, false),
IndexCreate(conf, true),
IndexDelete(conf),
IndexClose(conf),
IndexAllocation(conf),
IndexModify(conf),
IndexFields(conf),
IndexIlm(conf),
@@ -70,6 +70,12 @@ func IndexList(conf *cfg.Config) *cli.Command {
Destination: &conf.Partials,
Aliases: []string{"p"},
},
&cli.BoolFlag{
Name: "hidden",
Usage: "include hidden indicies",
Destination: &conf.Hidden,
Aliases: []string{"H"},
},
&cli.BoolFlag{
Name: "reds",
Usage: "include only red failed indicies",
@@ -143,12 +149,20 @@ func IndexShow(conf *cfg.Config) *cli.Command {
}
}
func IndexCreate(conf *cfg.Config) *cli.Command {
func IndexCreate(conf *cfg.Config, modify bool) *cli.Command {
name := "create"
usage := "create a new index"
if modify {
name = "update"
usage = "update an index"
}
return &cli.Command{
Name: "create",
Name: name,
Aliases: []string{"+"},
Usage: "create a new index",
UsageText: `create <name> <fieldmapping:type>...
Usage: usage,
UsageText: name + ` <name> <fieldmapping:type>...
Valid field mapping types: integer, text, date, keyword`,
Flags: []cli.Flag{
@@ -171,6 +185,12 @@ Valid field mapping types: integer, text, date, keyword`,
Aliases: []string{"r"},
Value: 1,
},
&cli.StringFlag{
Name: "ilm-policy",
Usage: "ilm policy to apply",
Destination: &conf.Policy,
Aliases: []string{"p"},
},
},
Action: func(ctx context.Context, cmd *cli.Command) error {
@@ -229,37 +249,6 @@ func IndexClose(conf *cfg.Config) *cli.Command {
}
}
func IndexModify(conf *cfg.Config) *cli.Command {
return &cli.Command{
Name: "modify",
Usage: "modify an index",
UsageText: "modify <index[,index,...]|_all>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cindex)
},
Flags: []cli.Flag{
&cli.IntFlag{
Name: "replicas",
Usage: "number of replicas",
Destination: &conf.Replicas,
Aliases: []string{"r"},
Value: 1,
},
},
Action: func(ctx context.Context, cmd *cli.Command) error {
index := cmd.Args().Get(0)
if index == "" {
return errors.New("no index specified")
}
return es.IndexModify(conf, index)
},
}
}
func IndexFields(conf *cfg.Config) *cli.Command {
return &cli.Command{
Name: "fields",

View File

@@ -83,8 +83,8 @@ func IndexTemplateCreate(conf *cfg.Config, modify bool) *cli.Command {
required := true
if modify {
name = "modify"
alias = "mod"
name = "update"
alias = "upd"
required = false
}
@@ -169,6 +169,12 @@ https://www.elastic.co/docs/reference/elasticsearch/index-settings
Destination: &conf.Aliases,
Aliases: []string{"A"},
},
&cli.StringFlag{
Name: "ilm-policy",
Usage: "ilm policy to apply",
Destination: &conf.Policy,
Aliases: []string{"p"},
},
},
Action: func(ctx context.Context, cmd *cli.Command) error {

View File

@@ -18,6 +18,7 @@ package cmd
import (
"context"
"errors"
"codeberg.org/scip/esctl/pkg/cfg"
"codeberg.org/scip/esctl/pkg/es"
@@ -34,6 +35,7 @@ func Node(conf *cfg.Config) *cli.Command {
Commands: []*cli.Command{
NodeList(conf),
NodeShow(conf),
NodeClients(conf),
},
}
}
@@ -57,10 +59,47 @@ func NodeShow(conf *cfg.Config) *cli.Command {
Usage: "show details about a node",
UsageText: "show [options] <node>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cnode)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
// FIXME: implement es.NodeShow()
// return es.NodeShow(conf, cmd.Args().Get(0))
return nil
node := cmd.Args().Get(0)
if node == "" {
return errors.New("no node specified")
}
return es.NodeShow(conf, node)
},
}
}
func NodeClients(conf *cfg.Config) *cli.Command {
return &cli.Command{
Name: "clients",
Usage: "show node http clients",
UsageText: "clients [options] <node>",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "query",
Usage: "include query parameters",
Destination: &conf.All,
Aliases: []string{"q"},
},
},
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cnode)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
node := cmd.Args().Get(0)
if node == "" {
return errors.New("no node specified")
}
return es.NodeClients(conf, node)
},
}
}

View File

@@ -42,7 +42,6 @@ func Finish(err error) int {
func Main() int {
conf := cfg.NewConfig()
tree := false
cmd := &cli.Command{
Name: "esctl",
@@ -64,13 +63,6 @@ func Main() int {
Usage: "enable HTTP debugging",
Destination: &conf.DebugHTTP,
},
&cli.BoolFlag{
Name: "show-command-tree",
Value: false,
Usage: "generate a command tree",
Destination: &tree,
Hidden: true,
},
&cli.BoolFlag{
Name: "align-ints",
Aliases: []string{"I"},
@@ -125,17 +117,10 @@ func Main() int {
Version(conf),
Debug(conf),
HelpJsonPath(conf),
HelpUsage(conf),
},
Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) {
if tree {
if err := Tree(cmd); err != nil {
return nil, err
}
os.Exit(0)
}
if err := conf.Init(); err != nil {
if len(os.Args) > 1 {
return nil, err
@@ -261,10 +246,42 @@ func Debug(conf *cfg.Config) *cli.Command {
}
}
func Tree(cmd *cli.Command) error {
max := 20
func HelpUsage(conf *cfg.Config) *cli.Command {
return &cli.Command{
Name: "help-command-overview",
Usage: "show overview of all available commands",
Aliases: []string{"usage"},
return cmd.Walk(func(cmd *cli.Command) error {
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "hidden",
Usage: "include hidden commands",
Destination: &conf.Hidden,
Aliases: []string{"H"},
},
},
Action: func(ctx context.Context, cmd *cli.Command) error {
maxCommandWidth := 0
// first pass, determine max command width
if err := walkVisible(conf, cmd.Root(), func(cmd *cli.Command) error {
path := cmd.Path()
size := len(path[len(path)-1])
if size > maxCommandWidth {
maxCommandWidth = size
}
return nil
}); err != nil {
return err
}
maxCommandWidth += 4 // account for indent width
// second pass, build tree
return walkVisible(conf, cmd.Root(), func(cmd *cli.Command) error {
path := cmd.Path()
command := path[len(path)-1]
@@ -273,10 +290,37 @@ func Tree(cmd *cli.Command) error {
}
indent := strings.Repeat(" ", len(path[1:])-1)
space := strings.Repeat(" ", max-(len(command)+len(indent)))
space := strings.Repeat(" ", maxCommandWidth-(len(command)+len(indent)))
fmt.Printf("%s%s %s - %s\n", indent, command, space, cmd.Usage)
return nil
})
},
}
}
// copy of cmd.Walk() with the exception to skip hidden commands and its siblings
// see: https://github.com/urfave/cli/issues/2372
func walkVisible(conf *cfg.Config, cmd *cli.Command, fn func(*cli.Command) error) error {
if fn == nil {
return nil
}
if !conf.Hidden && cmd.Hidden {
return nil
}
if err := fn(cmd); err != nil {
return err
}
for _, sub := range cmd.Commands {
if err := walkVisible(conf, sub, fn); err != nil {
return err
}
}
return nil
}

1
go.mod
View File

@@ -85,6 +85,7 @@ require (
github.com/olekukonko/errors v1.2.0 // indirect
github.com/olekukonko/ll v0.1.8 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/seeruk/go-wordwrap v0.0.0-20191208221741-14ec4aac9550 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect

2
go.sum
View File

@@ -171,6 +171,8 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
github.com/seeruk/go-wordwrap v0.0.0-20191208221741-14ec4aac9550 h1:C3CfUXH/qmWuQFRqnPm3Sx8PFxa+pqACjhV5CaNO8pw=
github.com/seeruk/go-wordwrap v0.0.0-20191208221741-14ec4aac9550/go.mod h1:Sl541M2Em6rRG3V9WObycR7MYFZiERVkd/TJg0Gt0U4=
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

View File

@@ -17,26 +17,32 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package cfg
import (
"context"
"crypto/tls"
"errors"
"fmt"
"net/http"
"os"
"strings"
"syscall"
"time"
"github.com/elastic/elastic-transport-go/v8/elastictransport"
"github.com/elastic/go-elasticsearch/v9"
"golang.org/x/term"
"gopkg.in/yaml.v3"
)
// used in general config struct
type Cluster struct {
Uri, User, Pass string
Name, Uri, User, Pass, Token string
client *elasticsearch.TypedClient
Default bool
Default, DebugHTTP bool
}
// used just for writing back to the config file
type ClusterConfig struct {
Uri, User, Pass string
Uri, User, Pass, Token string
Default bool
}
@@ -45,17 +51,96 @@ type WriteConfig struct {
Clusters map[string]*ClusterConfig
}
func (cluster *Cluster) SetClient(client *elasticsearch.TypedClient) {
cluster.client = client
}
func (cluster *Cluster) getTransport() elastictransport.Option {
transport := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
if cluster.DebugHTTP {
return elastictransport.WithTransport(
&DebugTransport{Transport: transport},
)
}
return elastictransport.WithTransport(transport)
}
func (cluster *Cluster) getDefaultOptions() []elasticsearch.Option {
// These headers are not needed with ES 9, but with ES 8, we set
// them here so every API call uses it. The only exception being
// the api repl, which does it on its own.
headers := http.Header{}
headers.Add("content-type", "application/json")
headers.Add("Accept", "application/json")
return []elasticsearch.Option{
elasticsearch.WithAddresses(cluster.Uri),
elasticsearch.WithTransportOptions(
cluster.getTransport(),
elastictransport.WithHeader(headers),
),
}
}
// return the go-elasticsearch client object but before doing that,
// check if we need to tune auth
func (cluster *Cluster) ES() *elasticsearch.TypedClient {
if cluster.client == nil {
fmt.Println("no current cluster, use 'esctl cluster switch <name>' to set one")
os.Exit(1)
}
if err := cluster.CheckAuth(); err != nil {
fmt.Printf("Error: %s", err)
os.Exit(1)
}
return cluster.client
}
func (cluster *Cluster) SetClient(client *elasticsearch.TypedClient) {
cluster.client = client
// add authentication to es client, if not yet done
func (cluster *Cluster) CheckAuth() error {
if cluster.Pass == "" && cluster.User != "" && cluster.Token == "" && cluster.Default {
// no token - user is set, but no password.
// check if the env var is set
pass := os.Getenv("ES_PASS")
if pass != "" {
cluster.Pass = pass
} else {
// k, try interactively
fmt.Printf("Enter password for elasticsearch user %s@%s: ", cluster.User, cluster.Name)
pass, err := term.ReadPassword(int(syscall.Stdin))
if err != nil {
return err
}
passwd := strings.TrimSpace(string(pass))
if passwd == "" {
return errors.New("password empty")
}
cluster.Pass = string(pass)
fmt.Println()
}
opts := cluster.getDefaultOptions()
opts = append(opts, elasticsearch.WithBasicAuth(cluster.User, cluster.Pass))
es, err := elasticsearch.NewTyped(opts...)
if err != nil {
return fmt.Errorf("failed to setup elasticsearch connection: %w", err)
}
cluster.SetClient(es)
}
return nil
}
// set Default=true for the given cluster in the config (if exists)
@@ -72,6 +157,7 @@ func (conf *Config) SwitchCluster(name string) error {
Uri: cluster.Uri,
User: cluster.User,
Pass: cluster.Pass,
Token: cluster.Token,
Default: false,
}
@@ -97,24 +183,62 @@ func (conf *Config) SwitchCluster(name string) error {
return nil
}
func (conf *Config) SetupES() error {
// These headers are not needed with ES 9, but with ES 8, we set
// them here so every API call uses it. The only exception being
// the api repl, which does it on its own.
headers := http.Header{}
headers.Add("content-type", "application/json")
headers.Add("Accept", "application/json")
// We do NOT use go-elasticsearch to check for cluster reachability,
// because at this stage, auth may not have been configured. So
// instead we just connect to the cluster using plan net/http, ignore
// HTTP response status and return true if we could just reach ith
func (cluster *Cluster) IsReachable() (bool, error) {
ctx, cancel := context.WithTimeout(
context.Background(),
time.Duration(500)*time.Millisecond)
defer cancel()
for _, cluster := range conf.Clusters {
es, err := elasticsearch.NewTyped(
elasticsearch.WithAddresses(cluster.Uri),
elasticsearch.WithBasicAuth(cluster.User, cluster.Pass),
elasticsearch.WithTransportOptions(
conf.getTransport(),
elastictransport.WithHeader(headers),
),
req, err := http.NewRequestWithContext(
ctx,
"GET",
cluster.Uri,
nil,
)
if err != nil {
return false, err
}
client := &http.Client{Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}}
resp, err := client.Do(req)
if err != nil {
return false, err
}
if resp != nil {
// at this stage we do not care if the elasticsearch cluster
// accepts our request or if it's misconfigured in some way
return true, nil
}
return false, nil
}
func (conf *Config) SetupES() error {
for name, cluster := range conf.Clusters {
cluster.Name = name
cluster.DebugHTTP = conf.DebugHTTP
opts := cluster.getDefaultOptions()
switch {
case cluster.Pass != "" && cluster.User != "":
opts = append(opts, elasticsearch.WithBasicAuth(cluster.User, cluster.Pass))
case cluster.Token != "":
opts = append(opts, elasticsearch.WithAPIKey(cluster.Token))
}
es, err := elasticsearch.NewTyped(opts...)
if err != nil {
return fmt.Errorf("failed to setup elasticsearch connection: %w", err)
}

View File

@@ -28,7 +28,7 @@ import (
)
const (
Version string = `v0.0.21`
Version string = `v0.0.24`
)
var (
@@ -51,6 +51,7 @@ type Config struct {
Failed, Partials bool // index: flags
Shards, Replicas int // index create+allocation: -s -r
Wait bool // index create: -w
Policy string // index create: -p
Primary bool // index allocation: -p
Searchable bool // index fields: -s
Aggretable bool // index fields: -a
@@ -102,6 +103,9 @@ type Config struct {
Tag string // api ls: -t
Ilm Ilm // ilm create
FromNode, ToNode string // cluster reroute move: -f + -t
AllowPrimary, AcceptDataLoss bool // cluster reroute cancel: -p,-a
}
func NewConfig() *Config {
@@ -212,15 +216,14 @@ func (conf *Config) LoadEnv() error {
Uri: os.Getenv("ES_URI"),
User: os.Getenv("ES_USER"),
Pass: os.Getenv("ES_PASS"),
Token: os.Getenv("ES_TOKEN"),
}
switch {
case cluster.Uri == "":
return errors.New("ES_URI unset")
case cluster.User == "":
return errors.New("ES_USER unset")
case cluster.Pass == "":
return errors.New("ES_PASS unset")
case cluster.User == "" || cluster.Token == "":
return errors.New("ES_USER and ES_TOKEN unset")
}
conf.Clusters["default"] = &cluster

View File

@@ -21,7 +21,7 @@ type Ilm struct {
HotMinAge string
HotRolloverMaxAge string
HotRolloverMaxPrimaryShardSize int64
HotRolloverMaxPrimaryShardSize string
HotRolloverMaxDocs int64
HotReadonly bool
@@ -36,16 +36,20 @@ type Ilm struct {
ColdPriority int
FrozenMinAge string
FrozenSearchableSnapshotRepo string
DeleteMinAge string
DeleteSearchableSnapshots bool
MinAge, FromPhase, Within string // forecast
Tree bool
}
func (cfg *Ilm) HaveHot() bool {
return cfg.HotMinAge != "" ||
cfg.HotRolloverMaxAge != "" ||
cfg.HotRolloverMaxDocs > 0 ||
cfg.HotRolloverMaxPrimaryShardSize > 0 ||
cfg.HotRolloverMaxPrimaryShardSize != "" ||
cfg.HotReadonly
}

38
pkg/cfg/term.go Normal file
View File

@@ -0,0 +1,38 @@
/*
Copyright © 2026 Thomas von Dein
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package cfg
import (
"os"
"golang.org/x/term"
)
const (
DefaultMargin = 4
)
func GetTermWidth() int {
if term.IsTerminal(int(os.Stdout.Fd())) {
width, _, err := term.GetSize(int(os.Stdout.Fd()))
if err == nil {
return width - DefaultMargin
}
}
return 80
}

View File

@@ -18,13 +18,10 @@ package cfg
import (
"bytes"
"crypto/tls"
"encoding/json"
"fmt"
"log/slog"
"net/http"
"github.com/elastic/elastic-transport-go/v8/elastictransport"
)
// used to print uri, path and body of a request made by the go-client
@@ -62,17 +59,3 @@ func (t *DebugTransport) RoundTrip(req *http.Request) (*http.Response, error) {
return t.Transport.RoundTrip(req)
}
func (conf *Config) getTransport() elastictransport.Option {
transport := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
if conf.DebugHTTP {
return elastictransport.WithTransport(
&DebugTransport{Transport: transport},
)
}
return elastictransport.WithTransport(transport)
}

View File

@@ -41,11 +41,9 @@ import (
"github.com/charmbracelet/lipgloss"
"github.com/chzyer/readline"
"github.com/go-openapi/spec"
"golang.org/x/term"
)
const (
DefaultMargin = 4
intro = `Input format: verb path [data]"
Example:
@@ -172,7 +170,17 @@ func CallAPI(conf *cfg.Config, verb, path, data string) ([]byte, error) {
req.Header.Add("Content-Type", "application/json")
req.Header.Add("accept", "application/json")
// make sure we have got all we need
if err := conf.DefaultCluster.CheckAuth(); err != nil {
return nil, err
}
if conf.DefaultCluster.Token != "" {
req.Header.Add("Authorization", "APIKey "+conf.DefaultCluster.Token)
} else {
req.Header.Add("Authorization", "Basic "+encodeAuth(conf.DefaultCluster.User, conf.DefaultCluster.Pass))
}
// actually execute the request
resp, err := client.Do(req)
@@ -318,19 +326,19 @@ func ApiShow(conf *cfg.Config, showpath, verb string) error {
cleanMarkup := regexp.MustCompile(`<[^<>]+>`)
width := getTermWidth()
width := cfg.GetTermWidth()
params := getApiParameters(op, showpath, width)
sample := getApiExample(op)
description := markdown.Render(cleanMarkup.ReplaceAllString(op.Op.Description, ""), width, DefaultMargin)
description := markdown.Render(cleanMarkup.ReplaceAllString(op.Op.Description, ""), width, cfg.DefaultMargin)
var bold = lipgloss.NewStyle().
Bold(true)
var paragraph = lipgloss.NewStyle().
MarginBottom(1).
MarginLeft(DefaultMargin)
MarginLeft(cfg.DefaultMargin)
var boldparagraph = lipgloss.NewStyle().
MarginBottom(1).
MarginLeft(DefaultMargin).
MarginLeft(cfg.DefaultMargin).
Bold(true)
var indentparagraph = lipgloss.NewStyle().
MarginBottom(1).
@@ -413,7 +421,7 @@ func getApiParameters(op *Op, path string, width int) Params {
}
} else {
par := Param{
Description: string(markdown.Render(param.Description, width, DefaultMargin)),
Description: string(markdown.Render(param.Description, width, cfg.DefaultMargin)),
Param: param.Name,
}
@@ -517,14 +525,3 @@ func findOperation(item spec.PathItemProps) []*Op {
return ops
}
func getTermWidth() int {
if term.IsTerminal(int(os.Stdout.Fd())) {
width, _, err := term.GetSize(int(os.Stdout.Fd()))
if err == nil {
return width - DefaultMargin
}
}
return 80
}

View File

@@ -0,0 +1,59 @@
/*
Copyright © 2026 Thomas von Dein
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package es
import (
"encoding/json"
"fmt"
"codeberg.org/scip/esctl/pkg/cfg"
)
type HealthReport struct {
Indicators map[string]HealthReportIndicator
}
type HealthReportIndicator struct {
Status, Symptom string
Diagnosis []HealthReportDiagnosis
}
type HealthReportDiagnosis struct {
Id, Cause, Action string
AffectedResources map[string][]string `json:"affected_resources"`
}
// we do not use the go-elasticsearch client API here but call the ES
// API directly, because the returned structure (a
// healthreport.Response) is not iterable, you'd have to explicitly
// call every indicator type and every cause etc which also have
// different types each. To check which is !green would result in a
// gigantic function.
func getHealthReport(conf *cfg.Config) (*HealthReport, error) {
raw, err := CallAPI(conf, "GET", "/_health_report", "")
if err != nil {
return nil, err
}
report := HealthReport{}
if err := json.Unmarshal(raw, &report); err != nil {
return nil, fmt.Errorf("failed to unmarshal healthreport response: %w", err)
}
return &report, nil
}

View File

@@ -17,7 +17,6 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package es
import (
"context"
"fmt"
"log/slog"
"strings"
@@ -26,64 +25,40 @@ import (
"codeberg.org/scip/esctl/pkg/cfg"
"codeberg.org/scip/esctl/pkg/printer"
"github.com/dustin/go-humanize"
"github.com/elastic/go-elasticsearch/v9/typedapi/cat/indices"
"github.com/elastic/go-elasticsearch/v9/typedapi/cat/tasks"
"github.com/elastic/go-elasticsearch/v9/typedapi/ccr/stats"
"github.com/elastic/go-elasticsearch/v9/typedapi/cluster/health"
clusterstats "github.com/elastic/go-elasticsearch/v9/typedapi/cluster/stats"
"github.com/elastic/go-elasticsearch/v9/typedapi/core/info"
"github.com/elastic/go-elasticsearch/v9/typedapi/types"
)
const (
ResponseHealth = iota
ResponseInfo
ResponseCcr
ResponseStats
ResponseIndices
ResponseTasks
)
type ClusterIndices map[string]map[string]*types.IndicesRecord
type apiResponse struct {
error error
info *info.Response
health *health.Response
ccr *stats.Response
stats *clusterstats.Response
indices *indices.Response
tasks *tasks.Response
which int
}
func ClusterList(conf *cfg.Config) error {
table := printer.NewTable(conf, 4, len(conf.Clusters))
table := printer.NewTable(conf, 5, len(conf.Clusters))
table.Addheaders("cluster", "uri", "reachable", "current")
table.Addheaders("cluster", "uri", "reachable", "current", "error")
idx := 0
for name, cluster := range conf.Clusters {
reachable := "no"
current := "no"
errmsg := ""
_, err := cluster.ES().Cluster.Health().
Do(context.Background())
online, err := cluster.IsReachable()
if err == nil {
if online {
reachable = printer.Colorize(conf, "green", "reachable")
}
if cluster.Default {
current = printer.Colorize(conf, "green", "yes")
if err != nil {
if !online {
reachable = printer.Colorize(conf, "red", "no")
errmsg = err.Error()
}
}
table.Entries[idx] = []string{name, cluster.Uri, reachable, current}
table.Entries[idx] = []string{name, cluster.Uri, reachable, current, errmsg}
idx++
}
@@ -98,90 +73,84 @@ func ClusterList(conf *cfg.Config) error {
// We're using goroutines here to parallelize API requests, since we
// have to do 3 of'em for each cluster. This speeds things up.
func ClusterStatus(conf *cfg.Config) error {
clusters := []string{}
gocount := 5
func getClusterStatus(conf *cfg.Config) (*apiResponse, error) {
gocount := 6
if conf.Verbose {
gocount++
}
if conf.All {
for key := range conf.Clusters {
clusters = append(clusters, key)
}
} else {
clusters = []string{"default"}
}
for _, cluster := range clusters {
es := conf.DefaultCluster.ES()
if cluster != "default" {
es = conf.Clusters[cluster].ES()
}
responses := make(chan apiResponse, gocount)
wg := &sync.WaitGroup{}
wg.Add(gocount)
go getClusterData(es, wg, responses, "health")
go getClusterData(es, wg, responses, "info")
go getClusterData(es, wg, responses, "ccrstats")
go getClusterData(es, wg, responses, "indices")
go getClusterData(es, wg, responses, "tasks")
go getApiData(conf, es, wg, responses, "health")
go getApiData(conf, es, wg, responses, "healthreport")
go getApiData(conf, es, wg, responses, "info")
go getApiData(conf, es, wg, responses, "ccr")
go getApiData(conf, es, wg, responses, "indices")
go getApiData(conf, es, wg, responses, "tasks")
if conf.Verbose {
go getClusterData(es, wg, responses, "stats")
go getApiData(conf, es, wg, responses, "stats")
}
wg.Wait()
var clusterhealth *health.Response
var info *info.Response
var ccrstats *stats.Response
var clusterstats *clusterstats.Response
var indexstats *indices.Response
var taskstatus *tasks.Response
all := apiResponse{}
for i := 0; i < gocount; i++ {
r := <-responses
if r.error != nil {
return r.error
return nil, r.error
}
switch r.which {
case ResponseHealth:
clusterhealth = r.health
all.health = r.health
case ResponseCcr:
ccrstats = r.ccr
all.ccr = r.ccr
case ResponseInfo:
info = r.info
all.info = r.info
case ResponseStats:
clusterstats = r.stats
all.stats = r.stats
case ResponseIndices:
indexstats = r.indices
all.indices = r.indices
case ResponseTasks:
taskstatus = r.tasks
all.tasks = r.tasks
case ResponseHealthReport:
all.healthreport = r.healthreport
}
}
slog.Debug("ES result", "cluster health", clusterhealth)
return &all, nil
}
isleader := len(ccrstats.AutoFollowStats.AutoFollowedClusters) == 0
func ClusterStatus(conf *cfg.Config) error {
res, err := getClusterStatus(conf)
if err != nil {
return err
}
slog.Debug("ES result", "cluster health", res.health)
isleader := len(res.ccr.AutoFollowStats.AutoFollowedClusters) == 0
ccrfollowing := ""
if len(ccrstats.AutoFollowStats.AutoFollowedClusters) > 0 {
if len(res.ccr.AutoFollowStats.AutoFollowedClusters) > 0 {
// is following another cluster
ccrfollowing = fmt.Sprintf("%s (%d/%d)",
ccrstats.AutoFollowStats.AutoFollowedClusters[0].ClusterName,
ccrstats.AutoFollowStats.NumberOfSuccessfulFollowIndices,
ccrstats.AutoFollowStats.NumberOfFailedFollowIndices,
res.ccr.AutoFollowStats.AutoFollowedClusters[0].ClusterName,
res.ccr.AutoFollowStats.NumberOfSuccessfulFollowIndices,
res.ccr.AutoFollowStats.NumberOfFailedFollowIndices,
)
}
// look for red indices, if any
redindices := 0
for _, index := range *indexstats {
for _, index := range *res.indices {
if *index.Health == "red" {
redindices++
}
@@ -189,26 +158,26 @@ func ClusterStatus(conf *cfg.Config) error {
// look for long running tasks
longtasks := 0
for _, task := range *taskstatus {
for _, task := range *res.tasks {
if strings.Contains(*task.RunningTime, "d") {
longtasks++
}
}
table := printer.NewTable(conf, 2, 7)
table.Addheaders(cluster, "status")
table.Addheaders(conf.DefaultCluster.Name, "status")
table.Entries = [][]string{
{"Cluster Name", clusterhealth.ClusterName},
{"ES Status", printer.Colorize(conf, clusterhealth.Status.Name, clusterhealth.Status.Name)},
{"ES Version", info.Version.Int},
{"Cluster Name", res.health.ClusterName},
{"ES Status", printer.Colorize(conf, res.health.Status.Name, res.health.Status.Name)},
{"ES Version", res.info.Version.Int},
{"Is Leader", fmt.Sprintf("%t", isleader)},
{"Active Shards", fmt.Sprintf("%d", clusterhealth.ActiveShards)},
{"Active Primary Shards", fmt.Sprintf("%d", clusterhealth.ActivePrimaryShards)},
{"Unassigned Shards", fmt.Sprintf("%d", clusterhealth.UnassignedShards)},
{"Unassigned Primary Shards", fmt.Sprintf("%d", clusterhealth.UnassignedPrimaryShards)},
{"Pending Tasks", fmt.Sprintf("%d", clusterhealth.NumberOfPendingTasks)},
{"Nodes", fmt.Sprintf("%d", clusterhealth.NumberOfNodes)},
{"Active Shards", fmt.Sprintf("%d", res.health.ActiveShards)},
{"Active Primary Shards", fmt.Sprintf("%d", res.health.ActivePrimaryShards)},
{"Unassigned Shards", fmt.Sprintf("%d", res.health.UnassignedShards)},
{"Unassigned Primary Shards", fmt.Sprintf("%d", res.health.UnassignedPrimaryShards)},
{"Pending Tasks", fmt.Sprintf("%d", res.health.NumberOfPendingTasks)},
{"Nodes", fmt.Sprintf("%d", res.health.NumberOfNodes)},
{"Red Indices", fmt.Sprintf("%d", redindices)},
{"Long Running Tasks", fmt.Sprintf("%d", longtasks)},
}
@@ -216,18 +185,35 @@ func ClusterStatus(conf *cfg.Config) error {
if !isleader {
table.Entries = append(table.Entries, [][]string{
{"AutoFollow (success/failed indices)", ccrfollowing},
{"Followed Indices", fmt.Sprintf("%d", len(ccrstats.FollowStats.Indices))},
{"Followed Indices", fmt.Sprintf("%d", len(res.ccr.FollowStats.Indices))},
}...)
}
if conf.Verbose {
table = gatherClusterStats(conf, clusterstats, table)
table = gatherClusterStats(conf, res.stats, table)
}
if res.health.Status.Name != "green" {
for name, indicator := range res.healthreport.Indicators {
if indicator.Status != "green" {
table.Entries = append(table.Entries, []string{
printer.Colorize(conf, indicator.Status, "Bad health "+name), indicator.Symptom,
})
for _, diag := range indicator.Diagnosis {
table.Entries = append(table.Entries, []string{" -> cause", diag.Cause})
for resource, items := range diag.AffectedResources {
table.Entries = append(table.Entries, []string{" -> affected " + resource, strings.Join(items, ",")})
}
}
}
}
}
if err := table.Print(); err != nil {
return err
}
}
return nil
}

125
pkg/es/cluster_reroute.go Normal file
View File

@@ -0,0 +1,125 @@
/*
Copyright © 2026 Thomas von Dein
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package es
import (
"context"
"fmt"
"codeberg.org/scip/esctl/pkg/cfg"
"github.com/elastic/go-elasticsearch/v9/typedapi/esdsl"
"github.com/elastic/go-elasticsearch/v9/typedapi/types"
)
func ClusterRerouteMove(conf *cfg.Config, index string) error {
move := conf.DefaultCluster.ES().Cluster.Reroute()
commands := esdsl.NewCommand()
moveCommand := &types.CommandMoveAction{
Shard: conf.Shards,
FromNode: conf.FromNode,
ToNode: conf.ToNode,
Index: index,
}
commands.CommandCaster().Move = moveCommand
move.Commands(commands)
_, err := move.Do(context.Background())
if err != nil {
return fmt.Errorf("failed to reroute move: %w", err)
}
return nil
}
func ClusterRerouteAllocateReplica(conf *cfg.Config, index string) error {
move := conf.DefaultCluster.ES().Cluster.Reroute()
commands := esdsl.NewCommand()
allocCommand := &types.CommandAllocateReplicaAction{
Shard: conf.Shards,
Node: conf.ToNode,
Index: index,
}
commands.CommandCaster().AllocateReplica = allocCommand
move.Commands(commands)
_, err := move.Do(context.Background())
if err != nil {
return fmt.Errorf("failed to allocate a replica shard: %w", err)
}
return nil
}
func ClusterRerouteCancel(conf *cfg.Config, index string) error {
move := conf.DefaultCluster.ES().Cluster.Reroute()
commands := esdsl.NewCommand()
cancelCommand := &types.CommandCancelAction{
Shard: conf.Shards,
Node: conf.ToNode,
Index: index,
AllowPrimary: &conf.AllowPrimary,
}
commands.CommandCaster().Cancel = cancelCommand
move.Commands(commands)
_, err := move.Do(context.Background())
if err != nil {
return fmt.Errorf("failed to cancel a reroute process: %w", err)
}
return nil
}
func ClusterRerouteAllocatePrimary(conf *cfg.Config, index string, stale bool) error {
move := conf.DefaultCluster.ES().Cluster.Reroute()
commands := esdsl.NewCommand()
allocCommand := &types.CommandAllocatePrimaryAction{
Shard: conf.Shards,
Node: conf.ToNode,
Index: index,
AcceptDataLoss: conf.AcceptDataLoss,
}
if stale {
commands.CommandCaster().AllocateStalePrimary = allocCommand
} else {
commands.CommandCaster().AllocateEmptyPrimary = allocCommand
}
move.Commands(commands)
_, err := move.Do(context.Background())
if err != nil {
which := "empty"
if stale {
which = "stale"
}
return fmt.Errorf("failed to allocate %s primary shard: %w", which, err)
}
return nil
}

View File

@@ -18,16 +18,13 @@ package es
import (
"context"
"errors"
"fmt"
"regexp"
"strconv"
"strings"
"sync"
"codeberg.org/scip/esctl/pkg/cfg"
"codeberg.org/scip/esctl/pkg/printer"
"github.com/elastic/go-elasticsearch/v9"
"github.com/elastic/go-elasticsearch/v9/typedapi/cluster/health"
"github.com/elastic/go-elasticsearch/v9/typedapi/types"
)
@@ -334,69 +331,6 @@ func splitArg(arg string) (string, string) {
}
}
func getClusterData(es *elasticsearch.TypedClient, wg *sync.WaitGroup, reschan chan apiResponse, which string) {
defer wg.Done()
ar := apiResponse{}
arerr := errors.New("")
switch which {
case "health":
res, err := es.Cluster.Health().
Do(context.Background())
ar.health = res
ar.which = ResponseHealth
arerr = err
case "info":
res, err := es.Info().
Do(context.Background())
ar.info = res
ar.which = ResponseInfo
arerr = err
case "ccrstats":
res, err := es.Ccr.Stats().
Do(context.Background())
ar.ccr = res
ar.which = ResponseCcr
arerr = err
case "stats":
res, err := es.Cluster.Stats().
Do(context.Background())
ar.stats = res
ar.which = ResponseStats
arerr = err
case "indices":
res, err := es.Cat.Indices().
Do(context.Background())
ar.indices = &res
ar.which = ResponseIndices
arerr = err
case "tasks":
res, err := es.Cat.Tasks().
Do(context.Background())
ar.tasks = &res
ar.which = ResponseTasks
arerr = err
}
if arerr != nil {
ar.error = fmt.Errorf("failed to get cluster health: %s", arerr)
}
reschan <- ar
}
// recursively traverse the raw settings hash and build a flat map
// consisting of the translated path and its value.
//

48
pkg/es/debug.go Normal file
View File

@@ -0,0 +1,48 @@
/*
Copyright © 2026 Thomas von Dein
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package es
import (
"codeberg.org/scip/esctl/pkg/cfg"
"github.com/alecthomas/repr"
)
func Debug(conf *cfg.Config) error {
report, err := getHealthReport(conf)
if err != nil {
return err
}
repr.Println(report)
/*
res, err := conf.DefaultCluster.ES().Search().
Index(conf.Index).
Size(0).
Aggregations(map[string]types.Aggregations{
"min_ts": *esdsl.NewMinAggregation().Field("@timestamp").AggregationsCaster(),
"max_ts": *esdsl.NewMaxAggregation().Field("@timestamp").AggregationsCaster(),
}).
Do(context.Background())
if err != nil {
return err
}
repr.Println(res)
*/
return nil
}

126
pkg/es/duration.go Normal file
View File

@@ -0,0 +1,126 @@
/*
Copyright © 2026 Thomas von Dein
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package es
import (
"fmt"
"math"
"regexp"
"strconv"
"time"
)
var (
matchDuration = regexp.MustCompile(`(\d+)([dhms])`)
)
type esDuration struct {
days, hours, minutes, seconds int
}
/*
We could use time.ParseDuration(), but this doesn't support days.
We could also use github.com/xhit/go-str2duration/v2, which does
the job, but it's just another dependency, just for this little
gem. And we don't need a time.Time value. And int is good enough
for duration comparison.
Convert a duration into an integer. Valid time units are "s", "m", "h" and "d".
via https://codeberg.org/scip/tablizer/src/branch/main/lib/sort.go#L113
*/
func parseDuration(duration string) time.Duration {
seconds := 0
for _, match := range matchDuration.FindAllStringSubmatch(duration, -1) {
if len(match) == 3 {
durationvalue, _ := strconv.Atoi(match[1])
switch match[2][0] {
case 'd':
seconds += durationvalue * 86400
case 'h':
seconds += durationvalue * 3600
case 'm':
seconds += durationvalue * 60
case 's':
seconds += durationvalue
}
}
}
return time.Duration(seconds) * time.Second
}
// pretty much the same as above, but return a struct filled with int duration values
func extractDuration(duration string) esDuration {
esd := esDuration{}
for _, match := range matchDuration.FindAllStringSubmatch(duration, -1) {
if len(match) == 3 {
durationvalue, _ := strconv.Atoi(match[1])
switch match[2][0] {
case 'd':
esd.days = durationvalue
case 'h':
esd.hours = durationvalue
case 'm':
esd.minutes = durationvalue
case 's':
esd.seconds = durationvalue
}
}
}
return esd
}
// convert a duration like 156h6m15.095s-6d/12h/6m => 6d:12h:6m
func formatDuration(val time.Duration) string {
esd := extractDuration(val.String())
daysF := float64(esd.hours) / 24.0
days := math.Floor(daysF)
hoursLeft := (daysF - days) * 24
switch {
case val.Minutes() < 1:
return fmt.Sprintf("%ds", int64(esd.seconds))
case val.Hours() < 1:
return fmt.Sprintf("%dm", int64(esd.minutes))
case val.Hours() < 24:
if esd.minutes == 0 {
return fmt.Sprintf("%dh", esd.hours)
}
return fmt.Sprintf("%dh:%dm", esd.hours, int64(esd.minutes))
default:
if esd.minutes == 0 && hoursLeft == 0 {
return fmt.Sprintf("%dd", int64(days))
}
return fmt.Sprintf("%dd:%dh:%dm",
int64(days),
int64(hoursLeft),
esd.minutes,
)
}
}

View File

@@ -18,6 +18,7 @@ package es
import (
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
@@ -26,6 +27,7 @@ import (
"codeberg.org/scip/esctl/pkg/cfg"
"codeberg.org/scip/esctl/pkg/printer"
"github.com/alecthomas/repr"
"github.com/charmbracelet/lipgloss"
"github.com/elastic/go-elasticsearch/v9/typedapi/esdsl"
"github.com/elastic/go-elasticsearch/v9/typedapi/ilm/putlifecycle"
"github.com/elastic/go-elasticsearch/v9/typedapi/types"
@@ -119,6 +121,10 @@ func IlmShow(conf *cfg.Config, policy string) error {
return errors.New("no ilm policy retrieved")
}
if conf.Ilm.Tree {
return IlmShowTree(conf, ilm.Policy)
}
table := printer.NewTable(conf, 2, 5)
table.Addheaders("ilm policy setting", "value")
@@ -133,6 +139,101 @@ func IlmShow(conf *cfg.Config, policy string) error {
return table.Print()
}
// Visualize phases or a ILM policy, taking into account that the
// minAge for the current phases is set in the next phase
func IlmShowTree(conf *cfg.Config, ilm types.IlmPolicy) error {
indent := ""
table := printer.NewTable(conf, 4, 0)
table.Addheaders("phase", "min age", "min size", "snapshot repo")
for _, phase := range IlmPhaseOrder {
if phase == "hot" {
fmt.Printf("%s%s phase:\n%s rollover after %s\n",
indent, phase,
indent, formatDuration(parseDuration(ilm.Phases.Hot.Actions.Rollover.MaxAge.(string))),
)
if ilm.Phases.Hot.Actions.Rollover.MaxPrimaryShardSize != "" {
fmt.Printf("%s rollover when storage > %s\n",
indent, ilm.Phases.Hot.Actions.Rollover.MaxPrimaryShardSize)
}
if ilm.Phases.Hot.Actions.Rollover.MaxDocs != nil {
fmt.Printf("%s rollover docs > %d\n",
indent, *ilm.Phases.Hot.Actions.Rollover.MaxDocs)
}
indent += " "
continue
}
current := getIlmCurrentPhase(ilm, phase)
if current == nil {
continue
}
next := findNextPhase(ilm, phase)
fmt.Printf("%s%s phase:\n", indent, phase)
if next != nil {
fmt.Printf("%s rollover after %s\n",
indent, formatDuration(next.minage),
)
if current.Actions.SearchableSnapshot != nil {
fmt.Printf("%s roll to snapshot repo: %s\n",
indent, current.Actions.SearchableSnapshot.SnapshotRepository)
}
if current.Actions.Forcemerge != nil {
fmt.Printf("%s force merge segments: %d\n",
indent, current.Actions.Forcemerge.MaxNumSegments)
}
if current.Actions.Shrink != nil {
fmt.Printf("%s shrink shards: %d\n",
indent, *current.Actions.Shrink.NumberOfShards)
}
if current.Actions.SetPriority != nil {
fmt.Printf("%s priority: %d\n",
indent, *current.Actions.SetPriority.Priority)
}
if current.Actions.Delete != nil && current.Actions.Delete.DeleteSearchableSnapshot != nil {
fmt.Printf("%s delete searchable snapshots: %t\n",
indent, *current.Actions.Delete.DeleteSearchableSnapshot)
}
}
if phase == "delete" {
fmt.Printf("%s delete immediately\n", indent)
}
indent += " "
}
return nil
}
func getIlmCurrentPhase(policy types.IlmPolicy, currentPhase string) *types.Phase {
switch currentPhase {
case "hot":
return policy.Phases.Hot
case "warm":
return policy.Phases.Warm
case "cold":
return policy.Phases.Cold
case "frozen":
return policy.Phases.Frozen
default:
return policy.Phases.Delete
}
}
func ilmPhaseString(phase *types.Phase, short bool) string {
if phase == nil {
return ""
@@ -183,21 +284,59 @@ func IlmExplain(conf *cfg.Config, index string) error {
table := printer.NewTable(conf, 2, 0)
table.Addheaders("ilm status field", "value")
info := ""
err = json.Unmarshal(ilm.StepInfo["reason"], &info)
if err != nil {
return fmt.Errorf("failed to unmarshal step info: %w", err)
}
table.Entries = [][]string{
{"index", ilm.Index},
{"ilm policy", *ilm.Policy},
{"action", *ilm.Action},
{"step", *ilm.Step},
{"age", fmt.Sprintf("%s", ilm.Age)},
{"managed", fmt.Sprintf("%t", ilm.Managed)},
{"phase", *ilm.Phase},
{"phase execution", ilmPhaseString(ilm.PhaseExecution.PhaseDefinition, false)},
{"step", *ilm.Step},
{"failed step", *ilm.FailedStep},
{"failed step retry count", fmt.Sprintf("%d", ilm.FailedStepRetryCount)},
}
return table.Print()
if err := table.Print(); err != nil {
return err
}
if info != "" {
var bold = lipgloss.NewStyle().Bold(true)
fmt.Printf("\n%s:\n%s\n",
bold.Render("Step Reason"),
info)
}
return nil
}
func IlmCreate(conf *cfg.Config, policyname string) error {
var policy *types.IlmPolicy = nil
res, err := conf.DefaultCluster.ES().Ilm.GetLifecycle().
Policy(policyname).
Do(context.Background())
if err == nil {
if conf.Debug {
repr.Println(res)
}
ilm, exists := res[policyname]
if !exists {
return errors.New("no ilm policy retrieved")
}
policy = &ilm.Policy
}
ilm := conf.DefaultCluster.ES().Ilm.PutLifecycle(policyname)
cfg := conf.Ilm
@@ -209,6 +348,16 @@ func IlmCreate(conf *cfg.Config, policyname string) error {
rollover := &types.RolloverAction{}
haveroll := false
if policy != nil {
// update
actions = policy.Phases.Hot.Actions
if policy.Phases.Hot.Actions.Rollover != nil {
rollover = policy.Phases.Hot.Actions.Rollover
haveroll = true
}
}
if cfg.HotMinAge != "" {
hot.MinAge = cfg.HotMinAge
}
@@ -223,7 +372,7 @@ func IlmCreate(conf *cfg.Config, policyname string) error {
haveroll = true
}
if cfg.HotRolloverMaxPrimaryShardSize != 0 {
if cfg.HotRolloverMaxPrimaryShardSize != "" {
rollover.MaxPrimaryShardSize = &cfg.HotRolloverMaxPrimaryShardSize
}
@@ -233,12 +382,22 @@ func IlmCreate(conf *cfg.Config, policyname string) error {
hot.Actions = actions.IlmActionsCaster()
phases.PhasesCaster().Hot = &hot
} else {
if policy != nil {
// update
phases.PhasesCaster().Hot = policy.Phases.Hot
}
}
if cfg.HaveWarm() {
warm := types.Phase{}
var actions types.IlmActionsVariant = esdsl.NewIlmActions()
if policy != nil {
// update
actions = policy.Phases.Warm.Actions
}
if cfg.WarmForceMerge != 0 {
actions.IlmActionsCaster().Forcemerge = &types.ForceMergeAction{MaxNumSegments: cfg.WarmForceMerge}
}
@@ -257,21 +416,85 @@ func IlmCreate(conf *cfg.Config, policyname string) error {
warm.Actions = actions.IlmActionsCaster()
phases.PhasesCaster().Warm = &warm
} else {
if policy != nil && policy.Phases.Warm != nil {
// update
phases.PhasesCaster().Warm = policy.Phases.Warm
}
}
if cfg.HaveCold() {
cold := types.Phase{}
var actions types.IlmActionsVariant = esdsl.NewIlmActions()
if policy != nil {
// update
actions = policy.Phases.Cold.Actions
}
if cfg.ColdForceMerge != 0 {
actions.IlmActionsCaster().Forcemerge = &types.ForceMergeAction{MaxNumSegments: cfg.ColdForceMerge}
}
if cfg.ColdMinAge != "" {
cold.MinAge = cfg.ColdMinAge
}
if cfg.ColdPriority != 0 {
actions.IlmActionsCaster().SetPriority = &types.SetPriorityAction{Priority: &cfg.ColdPriority}
}
if cfg.ColdSearchableSnapshotRepo != "" {
actions.IlmActionsCaster().SearchableSnapshot =
&types.SearchableSnapshotAction{SnapshotRepository: cfg.ColdSearchableSnapshotRepo}
}
cold.Actions = actions.IlmActionsCaster()
phases.PhasesCaster().Cold = &cold
} else {
if policy != nil && policy.Phases.Cold != nil {
// update
phases.PhasesCaster().Cold = policy.Phases.Cold
}
}
if cfg.HaveFrozen() {
froze := phases.PhasesCaster().Frozen
froze := types.Phase{}
var actions types.IlmActionsVariant = esdsl.NewIlmActions()
if policy != nil {
// update
actions = policy.Phases.Frozen.Actions
}
if cfg.FrozenMinAge != "" {
froze.MinAge = cfg.FrozenMinAge
}
if cfg.FrozenSearchableSnapshotRepo != "" {
actions.IlmActionsCaster().SearchableSnapshot =
&types.SearchableSnapshotAction{SnapshotRepository: cfg.FrozenSearchableSnapshotRepo}
}
froze.Actions = actions.IlmActionsCaster()
phases.PhasesCaster().Frozen = &froze
} else {
if policy != nil && policy.Phases.Frozen != nil {
// update
phases.PhasesCaster().Frozen = policy.Phases.Frozen
}
}
if cfg.HaveDelete() {
del := phases.PhasesCaster().Delete
del := types.Phase{}
delete := types.DeleteAction{}
var actions types.IlmActionsVariant = esdsl.NewIlmActions()
if policy != nil {
// update
actions = policy.Phases.Delete.Actions
}
if cfg.DeleteMinAge != "" {
del.MinAge = cfg.DeleteMinAge
}
@@ -282,17 +505,22 @@ func IlmCreate(conf *cfg.Config, policyname string) error {
actions.IlmActionsCaster().Delete = &delete
del.Actions = actions.IlmActionsCaster()
phases.PhasesCaster().Delete = del
phases.PhasesCaster().Delete = &del
} else {
if policy != nil && policy.Phases.Delete != nil {
// update
phases.PhasesCaster().Delete = policy.Phases.Delete
}
}
put := &putlifecycle.Request{}
policy := &types.IlmPolicy{}
policy.IlmPolicyCaster().Phases = *phases.PhasesCaster()
put.Policy = policy
newpolicy := &types.IlmPolicy{}
newpolicy.IlmPolicyCaster().Phases = *phases.PhasesCaster()
put.Policy = newpolicy
ilm.Request(put)
_, err := ilm.Do(context.Background())
_, err = ilm.Do(context.Background())
if err != nil {
return fmt.Errorf("failed create ilm policy: %s", esErrorString(err))
}

391
pkg/es/ilm_forecast.go Normal file
View File

@@ -0,0 +1,391 @@
/*
Copyright © 2026 Thomas von Dein
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package es
import (
"fmt"
"regexp"
"strconv"
"strings"
"sync"
"time"
"codeberg.org/scip/esctl/pkg/cfg"
"codeberg.org/scip/esctl/pkg/printer"
"github.com/dustin/go-humanize"
"github.com/elastic/go-elasticsearch/v9/typedapi/cat/indices"
"github.com/elastic/go-elasticsearch/v9/typedapi/ilm/explainlifecycle"
"github.com/elastic/go-elasticsearch/v9/typedapi/ilm/getlifecycle"
"github.com/elastic/go-elasticsearch/v9/typedapi/types"
)
var (
IlmPhaseOrder = []string{"hot", "warm", "cold", "frozen", "delete"}
)
type NextPhase struct {
minage time.Duration
minsize int64
phase string
previousHot bool
}
type PhaseData struct {
index string
policy string
size int64
age time.Duration
minage time.Duration
minsize int64
currentPhase, nextPhase string
}
/*
ilm stages:
types.Lifecycle{
Policy: types.IlmPolicy{
Phases: types.Phases{
Hot: &types.Phase{
Actions: &types.IlmActions{
Rollover: &types.RolloverAction{
MaxAge: "7d",
MaxPrimaryShardSize: "25gb",
},
},
MinAge: "0ms",
},
Warm: &types.Phase{
MinAge: "0d",
},
Frozen: &types.Phase{
MinAge: "14d",
},
Delete: &types.Phase{
MinAge: "60d",
},
},
},
}
hot for 7 days or 25Gig, then Rollover => warm 14 days => frozen 60 days => delete
*/
func IlmForecastList(conf *cfg.Config, filter string) error {
phaseData, err := getIlmPhaseData(conf)
if err != nil {
return err
}
flt := regexp.MustCompile(filter)
minage := time.Duration(0)
if conf.Ilm.MinAge != "" {
minage = parseDuration(conf.Ilm.MinAge)
}
headers := []string{"index", "current size", "current age",
"virtual age", "min age", "min size", "current phase", "next phase"}
if conf.Verbose {
headers = append(headers, "ilm policy")
}
table := printer.NewTable(conf, len(headers), 0)
table.Addheaders(headers...)
for _, phase := range phaseData {
if filter != "" && !flt.MatchString(phase.index) {
continue
}
if conf.Ilm.MinAge != "" && phase.age < minage {
continue
}
virtualAge := virtualAge(&phase)
row := []string{
phase.index,
humanize.Bytes(uint64(phase.size)),
formatDuration(phase.age),
formatDuration(virtualAge),
formatDuration(phase.minage),
humanize.Bytes(uint64(phase.minsize)),
phase.currentPhase,
phase.nextPhase,
}
if conf.Verbose {
row = append(row, phase.policy)
}
table.AddRow(row...)
}
table.Sort()
return table.Print()
}
// Loop over all current index phases of the current phase
// (conf.Ilm.FromPhase). Calculate virtualAge if phase.minsize >0. If
// the actual age is smaller than the virtual age, use this as a base,
// otherwise use virtual age. Then look if the rollover would happen
// within the phase.minage window (conf.Ilm.Within + current age) and
// add the size.
func IlmForecastShow(conf *cfg.Config) error {
phaseData, err := getIlmPhaseData(conf)
if err != nil {
return err
}
within := parseDuration(conf.Ilm.Within)
var toBeFreed int64 = 0
for _, phase := range phaseData {
age := virtualAge(&phase)
if age < phase.age {
age = phase.age
}
if age+within >= phase.minage {
toBeFreed += phase.size
}
}
fmt.Printf("%s bytes of data in %s phase will be rolled within %s to the next phase\n",
humanize.Bytes(uint64(toBeFreed)),
conf.Ilm.FromPhase,
within)
return nil
}
func virtualAge(phase *PhaseData) time.Duration {
if phase.minsize == 0 {
return time.Duration(0)
}
return phase.minage * time.Duration(100*phase.size/phase.minsize) / 100
}
// Retrieve all index, ilm-explain and ilm-policies in parallel
func getIlmPhaseData(conf *cfg.Config) ([]PhaseData, error) {
responses := make(chan apiResponse, 3)
wg := &sync.WaitGroup{}
wg.Add(3)
go getApiData(conf, conf.DefaultCluster.ES(), wg, responses, "indicesbytes")
go getApiData(conf, conf.DefaultCluster.ES(), wg, responses, "explain")
go getApiData(conf, conf.DefaultCluster.ES(), wg, responses, "policies")
wg.Wait()
var ilmdetails *explainlifecycle.Response
var indicesres *indices.Response
var ilmpolicies getlifecycle.Response
for i := 0; i < 3; i++ {
r := <-responses
if r.error != nil {
return nil, r.error
}
switch r.which {
case ResponseExplain:
ilmdetails = r.explainlifecycle
case ResponseIndices:
indicesres = r.indicesbytes
case ResponseLifecycle:
ilmpolicies = *r.lifecycle
}
}
list := []PhaseData{}
indices := make(map[string]types.IndicesRecord)
for _, index := range *indicesres {
indices[*index.Index] = index
}
for indexName, explain := range ilmdetails.Indices {
if !conf.Hidden && strings.HasPrefix(indexName, ".") {
continue
}
// avoid types.LifecycleExplainUnmanaged casting error
switch explain.(type) {
case *types.LifecycleExplainManaged:
default:
continue
}
explain := explain.(*types.LifecycleExplainManaged)
if explain.Phase == nil {
continue
}
phase := *explain.Phase
if phase != conf.Ilm.FromPhase {
continue
}
size, err := strconv.ParseInt(*indices[indexName].DatasetSize, 10, 64)
if err != nil {
return nil, fmt.Errorf("failed to convert index.DatasetSize to int: %w", err)
}
age := time.Duration(*explain.AgeInMillis * int64(time.Millisecond))
policy := ilmpolicies[*explain.Policy].Policy
nextPhase := findNextPhase(policy, phase)
if nextPhase == nil {
continue
}
list = append(list, PhaseData{
index: indexName,
policy: *explain.Policy,
size: size,
age: age,
minage: nextPhase.minage,
minsize: nextPhase.minsize,
currentPhase: conf.Ilm.FromPhase,
nextPhase: nextPhase.phase,
})
}
return list, nil
}
func registerPhases(policy types.IlmPolicy, currentPhase string) (map[string]*types.Phase, int) {
// register phases configured in the policy
phases := map[string]*types.Phase{
"hot": nil,
"warm": nil,
"cold": nil,
"frozen": nil,
"delete": nil,
}
// looking for a defined previous phase
start := 0
// register phases
for idx, phase := range IlmPhaseOrder {
switch phase {
case "hot":
if policy.Phases.Hot != nil {
phases[phase] = policy.Phases.Hot
}
case "warm":
if policy.Phases.Warm != nil {
phases[phase] = policy.Phases.Warm
}
case "cold":
if policy.Phases.Cold != nil {
phases[phase] = policy.Phases.Cold
}
case "frozen":
if policy.Phases.Frozen != nil {
phases[phase] = policy.Phases.Frozen
}
case "delete":
if policy.Phases.Delete != nil {
phases[phase] = policy.Phases.Delete
}
}
if phase == currentPhase {
start = idx
}
}
return phases, start
}
func findNextPhase(policy types.IlmPolicy, currentPhase string) *NextPhase {
// phase list to determine which comes next
phases, start := registerPhases(policy, currentPhase)
nextPhase := &NextPhase{}
// finally determine which phase comes next
// exception: hot, where we look for rollover rules
for idx, phase := range IlmPhaseOrder {
if idx < start {
continue
}
if phase == "hot" {
// if we're starting here, do not look for the next phase
maxage := parseDuration(phases[phase].Actions.Rollover.MaxAge.(string))
var maxsize int64 = 0
switch val := phases[phase].Actions.Rollover.MaxPrimaryShardSize.(type) {
case int64:
maxsize = val
case string:
size, err := humanize.ParseBytes(val)
if err != nil {
panic(err)
}
maxsize = int64(size)
}
nextPhase.minage = maxage
nextPhase.minsize = maxsize
nextPhase.previousHot = true
continue
}
if phase == currentPhase {
continue // use the next one
}
if phases[phase] == nil {
continue // skip it, since empty
}
nextPhase.phase = phase
if nextPhase.previousHot {
return nextPhase
}
minage := parseDuration(phases[phase].MinAge.(string))
nextPhase.minage = minage
nextPhase.minsize = 0
return nextPhase
}
return nil
}

View File

@@ -53,11 +53,16 @@ func IndexNames(conf *cfg.Config) ([]string, error) {
func filterIndices(conf *cfg.Config, list indices.Response) indices.Response {
// apply partials filter first
selectedlist := indices.Response{}
for _, index := range list {
if !conf.Partials && strings.HasPrefix(*index.Index, "partial-") {
continue
}
if !conf.Hidden && strings.HasPrefix(*index.Index, ".") {
continue
}
if strings.HasPrefix(*index.Index, ".ds-") {
// ignore data stream backing indicies
continue
@@ -131,6 +136,8 @@ func IndexShow(conf *cfg.Config, indexpattern string) error {
return fmt.Errorf("failed to get index: %s", esErrorString(err))
}
slog.Debug("index show", "index", res)
for name, index := range res {
fields := make([]string, len(index.Mappings.Properties))
idx := 0
@@ -139,7 +146,7 @@ func IndexShow(conf *cfg.Config, indexpattern string) error {
idx++
}
table := printer.NewTable(conf, 2, 5)
table := printer.NewTable(conf, 2, 7)
table.Addheaders("index property", "value")
ts, err := strconv.ParseInt(index.Settings.Index.CreationDate.(string), 10, 64)
@@ -155,9 +162,17 @@ func IndexShow(conf *cfg.Config, indexpattern string) error {
{"shards", *index.Settings.Index.NumberOfShards},
{"created", created.Format("2006-01-02 15:04:05")},
{"uuid", *index.Settings.Index.Uuid},
{"version", *index.Settings.Index.Version.Created},
{"fields", strings.Join(fields, ",")},
}
if index.Settings.Index.Lifecycle != nil {
table.Entries = append(table.Entries, [][]string{
{"ilm policy", *index.Settings.Index.Lifecycle.Name},
{"ilm rollover alias", *index.Settings.Index.Lifecycle.RolloverAlias},
}...)
}
if err := table.Print(); err != nil {
return err
}
@@ -178,10 +193,10 @@ func IndexCreate(conf *cfg.Config, index string, mappings []string) error {
}
if conf.Shards > 0 {
settings = settings.NumberOfShards(strconv.Itoa(conf.Shards))
settings.NumberOfShards(strconv.Itoa(conf.Shards))
}
if conf.Replicas > 0 {
settings = settings.NumberOfReplicas(strconv.Itoa(conf.Replicas))
settings.NumberOfReplicas(strconv.Itoa(conf.Replicas))
}
if len(mappings) > 0 {
@@ -208,6 +223,13 @@ func IndexCreate(conf *cfg.Config, index string, mappings []string) error {
create.Mappings(maps)
}
if conf.Policy != "" {
settings.Lifecycle(
esdsl.NewIndexSettingsLifecycle().
Name(conf.Policy).
RolloverAlias(index))
}
_, err := create.Settings(settings).
Do(context.Background())
@@ -281,6 +303,7 @@ func IndexAllocation(conf *cfg.Config, index string) error {
return nil
}
/*
func IndexModify(conf *cfg.Config, index string) error {
settings := esdsl.NewIndexSettings().NumberOfReplicas(strconv.Itoa(conf.Replicas))
@@ -294,6 +317,7 @@ func IndexModify(conf *cfg.Config, index string) error {
return nil
}
*/
func IndexFields(conf *cfg.Config, index string) error {
res, err := conf.DefaultCluster.ES().FieldCaps().

View File

@@ -157,11 +157,17 @@ func IndexTemplateCreate(conf *cfg.Config, name string, mappings []string) error
create := conf.DefaultCluster.ES().Indices.PutIndexTemplate(name)
if conf.Shards > 0 {
settings = settings.NumberOfShards(strconv.Itoa(conf.Shards))
settings.NumberOfShards(strconv.Itoa(conf.Shards))
}
if conf.Replicas > 0 {
settings = settings.NumberOfReplicas(strconv.Itoa(conf.Replicas))
settings.NumberOfReplicas(strconv.Itoa(conf.Replicas))
}
if conf.Policy != "" {
settings.Lifecycle(
esdsl.NewIndexSettingsLifecycle().
Name(conf.Policy))
}
if conf.Stream {
@@ -284,6 +290,10 @@ func IndexTemplateModify(conf *cfg.Config, name string, mappings []string) error
*settings.Mode = conf.Mode
}
if conf.Policy != "" {
settings.Lifecycle.Name = &conf.Policy
}
if len(mappings) > 0 {
typemaps, err := modMappings(mappings)
if err != nil {

View File

@@ -20,9 +20,12 @@ import (
"context"
"fmt"
"log/slog"
"strings"
"time"
"codeberg.org/scip/esctl/pkg/cfg"
"codeberg.org/scip/esctl/pkg/printer"
"github.com/dustin/go-humanize"
)
func NodeList(conf *cfg.Config) error {
@@ -56,3 +59,145 @@ func NodeList(conf *cfg.Config) error {
return nil
}
func NodeNames(conf *cfg.Config) ([]string, error) {
nodes, err := conf.DefaultCluster.ES().Cat.Nodes().
Do(context.Background())
if err != nil {
return nil, fmt.Errorf("failed to get nodes: %s", esErrorString(err))
}
slog.Debug("ES result", "nodes", nodes)
nodelist := make([]string, len(nodes))
for idx, node := range nodes {
nodelist[idx] = *node.Name
}
return nodelist, err
}
// FIXME: adding the settings metric leads to json unmarshall error:
// https://github.com/elastic/go-elasticsearch/issues/1524
func NodeShow(conf *cfg.Config, nodename string) error {
res, err := conf.DefaultCluster.ES().Nodes.Info().
NodeId(nodename).
Metric("os, jvm, thread_pool, remote_cluster_server").
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get node info: %s", esErrorString(err))
}
slog.Debug("ES result", "node", res)
stats, err := conf.DefaultCluster.ES().Nodes.Stats().
NodeId(nodename).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get node stats: %s", esErrorString(err))
}
slog.Debug("ES result", "stat", stats)
for id, info := range res.Nodes {
stat := stats.Nodes[id]
table := printer.NewTable(conf, 2, 0)
table.Addheaders(nodename+" property", "value")
roles := make([]string, len(info.Roles))
for idx, role := range info.Roles {
roles[idx] = role.Name
}
k8snode := info.Attributes["k8s_node_name"]
table.Entries = [][]string{
{"Id", id},
{"Name", nodename},
{"Kubernetes node", k8snode},
{"Ip address", info.Ip},
{"Node rank", *stat.AdaptiveSelection[id].Rank},
{"JVM", info.Jvm.VmName + " " + info.Jvm.Version},
{"JVM Started", time.UnixMilli(info.Jvm.StartTimeInMillis).String()},
{"OS", info.Os.PrettyName + " " + info.Os.Version},
{"Node roles", strings.Join(roles, ",")},
{"Node version", info.Version},
{"HTTP clients", fmt.Sprintf("%d", *stat.Http.CurrentOpen)},
{"CPUs", fmt.Sprintf("%d", *info.Os.AllocatedProcessors)},
{"Load 15m/5m/1m", fmt.Sprintf("%.2f/%.2f/%.2f",
stat.Os.Cpu.LoadAverage["15m"],
stat.Os.Cpu.LoadAverage["5m"],
stat.Os.Cpu.LoadAverage["1m"],
)},
{"Open FD's", fmt.Sprintf("%d", *stat.Process.OpenFileDescriptors)},
{"Response time avg", fmt.Sprintf("%dns", *stat.AdaptiveSelection[id].AvgResponseTimeNs)},
{"Memory usage (used/avail)",
humanize.Bytes(uint64(*stat.Os.Mem.UsedInBytes)) + " / " + humanize.Bytes(uint64(*stat.Os.Mem.TotalInBytes))},
}
if len(stat.Fs.Data) > 0 {
fs := stat.Fs.Data[0]
table.Entries = append(table.Entries, [][]string{
{"Storage usage (used/avail)",
humanize.Bytes(uint64(*fs.AvailableInBytes)) + " / " + humanize.Bytes(uint64(*fs.TotalInBytes))},
{"Storage mount", *fs.Mount},
}...)
}
if err := table.Print(); err != nil {
return err
}
}
return nil
}
func NodeClients(conf *cfg.Config, nodename string) error {
stats, err := conf.DefaultCluster.ES().Nodes.Stats().
NodeId(nodename).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get node stats: %s", esErrorString(err))
}
slog.Debug("ES result", "stat", stats)
table := printer.NewTable(conf, 5, 0)
table.Addheaders("agent", "id", "when", "from host", "url")
for _, stat := range stats.Nodes {
for _, client := range stat.Http.Clients {
if client.ClosedTimeMillis == nil {
agent := ""
if client.Agent != nil {
agent = *client.Agent
}
uri := ""
if client.LastUri != nil {
uri = *client.LastUri
if !conf.All {
parts := strings.Split(uri, "?")
uri = parts[0]
}
}
table.AddRow(
agent,
fmt.Sprintf("%d", *client.Id),
time.UnixMilli(*client.LastRequestTimeMillis).String(),
*client.RemoteAddress,
uri,
)
}
}
break
}
table.Sort()
return table.Print()
}

165
pkg/es/parallel.go Normal file
View File

@@ -0,0 +1,165 @@
/*
Copyright © 2026 Thomas von Dein
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package es
import (
"context"
"fmt"
"sync"
"codeberg.org/scip/esctl/pkg/cfg"
"github.com/elastic/go-elasticsearch/v9"
"github.com/elastic/go-elasticsearch/v9/typedapi/cat/indices"
"github.com/elastic/go-elasticsearch/v9/typedapi/cat/tasks"
"github.com/elastic/go-elasticsearch/v9/typedapi/ccr/stats"
"github.com/elastic/go-elasticsearch/v9/typedapi/cluster/health"
clusterstats "github.com/elastic/go-elasticsearch/v9/typedapi/cluster/stats"
"github.com/elastic/go-elasticsearch/v9/typedapi/core/info"
"github.com/elastic/go-elasticsearch/v9/typedapi/ilm/explainlifecycle"
"github.com/elastic/go-elasticsearch/v9/typedapi/ilm/getlifecycle"
"github.com/elastic/go-elasticsearch/v9/typedapi/types/enums/bytes"
)
const (
ResponseHealth = iota
ResponseInfo
ResponseCcr
ResponseStats
ResponseIndices
ResponseTasks
ResponseExplain
ResponseLifecycle
ResponseHealthReport
)
type apiResponse struct {
error error
info *info.Response
health *health.Response
healthreport *HealthReport
ccr *stats.Response
stats *clusterstats.Response
indices *indices.Response
indicesbytes *indices.Response
tasks *tasks.Response
lifecycle *getlifecycle.Response
explainlifecycle *explainlifecycle.Response
which int
}
func getApiData(
conf *cfg.Config,
es *elasticsearch.TypedClient,
wg *sync.WaitGroup,
reschan chan apiResponse,
which string) {
defer wg.Done()
ar := apiResponse{}
var arerr error
switch which {
case "health":
res, err := es.Cluster.Health().
Do(context.Background())
ar.health = res
ar.which = ResponseHealth
arerr = err
case "healthreport":
report, err := getHealthReport(conf)
ar.healthreport = report
ar.which = ResponseHealthReport
arerr = err
case "info":
res, err := es.Info().
Do(context.Background())
ar.info = res
ar.which = ResponseInfo
arerr = err
case "ccr":
res, err := es.Ccr.Stats().
Do(context.Background())
ar.ccr = res
ar.which = ResponseCcr
arerr = err
case "stats":
res, err := es.Cluster.Stats().
Do(context.Background())
ar.stats = res
ar.which = ResponseStats
arerr = err
case "indices":
res, err := es.Cat.Indices().
Do(context.Background())
ar.indices = &res
ar.which = ResponseIndices
arerr = err
case "indicesbytes":
res, err := es.Cat.
Indices().
Bytes(bytes.Bytes{Name: "b"}).
Do(context.Background())
ar.indicesbytes = &res
ar.which = ResponseIndices
arerr = err
case "tasks":
res, err := es.Cat.Tasks().
Do(context.Background())
ar.tasks = &res
ar.which = ResponseTasks
arerr = err
case "explain":
res, err := es.Ilm.
ExplainLifecycle("_all").
Do(context.Background())
ar.explainlifecycle = res
ar.which = ResponseExplain
arerr = err
case "policies":
res, err := es.Ilm.
GetLifecycle().
Do(context.Background())
ar.lifecycle = &res
ar.which = ResponseLifecycle
arerr = err
}
if arerr != nil {
ar.error = fmt.Errorf("failed to get data from API: %s", arerr)
}
reschan <- ar
}

View File

@@ -26,7 +26,6 @@ import (
"codeberg.org/scip/esctl/pkg/cfg"
"codeberg.org/scip/esctl/pkg/printer"
"github.com/alecthomas/repr"
"github.com/elastic/go-elasticsearch/v9/typedapi/core/search"
"github.com/elastic/go-elasticsearch/v9/typedapi/esdsl"
"github.com/elastic/go-elasticsearch/v9/typedapi/indices/validatequery"
@@ -149,25 +148,6 @@ func validateSearch(conf *cfg.Config, queries []string) error {
return nil
}
func Debug(conf *cfg.Config) error {
res, err := conf.DefaultCluster.ES().Search().
Index(conf.Index).
Size(0).
Aggregations(map[string]types.Aggregations{
"min_ts": *esdsl.NewMinAggregation().Field("@timestamp").AggregationsCaster(),
"max_ts": *esdsl.NewMaxAggregation().Field("@timestamp").AggregationsCaster(),
}).
Do(context.Background())
if err != nil {
return err
}
repr.Println(res)
return nil
}
func searchOnce(conf *cfg.Config, search *search.Search) error {
res, err := search.
From(conf.From).

View File

@@ -28,6 +28,7 @@ import (
"github.com/olekukonko/tablewriter"
"github.com/olekukonko/tablewriter/renderer"
"github.com/olekukonko/tablewriter/tw"
"github.com/seeruk/go-wordwrap"
"gopkg.in/yaml.v3"
)
@@ -38,10 +39,11 @@ type Table struct {
lenHeaders []int
alignInts bool
maxwidth int
}
func NewTable(conf *cfg.Config, columns, rows int) *Table {
table := Table{Mode: conf.Output}
table := Table{Mode: conf.Output, maxwidth: cfg.GetTermWidth()}
table.Headers = make([]string, columns)
table.Entries = make([][]string, rows)
@@ -118,17 +120,31 @@ func (data *Table) PrintTSV() error {
}
for _, entries := range data.Entries {
currentWidth := 0
for idx, entry := range entries {
length := visibleLen(entry)
if data.lenHeaders[idx] < length {
if length > currentWidth+data.maxwidth {
data.lenHeaders[idx] = data.maxwidth - currentWidth
} else {
data.lenHeaders[idx] = length
}
}
currentWidth += data.lenHeaders[idx]
}
}
// output
// output headers
for idx, header := range data.Headers {
if idx+1 != len(data.Headers) {
fmt.Print(header, strings.Repeat(" ", data.lenHeaders[idx]-visibleLen(header)))
} else {
// no padding for last header
fmt.Print(header)
}
if idx < len(data.Headers)-1 {
fmt.Print(" ")
}
@@ -136,14 +152,37 @@ func (data *Table) PrintTSV() error {
fmt.Println()
for _, entries := range data.Entries {
currentWidth := 0
for idx, entry := range entries {
length := visibleLen(entry)
if length+currentWidth > data.maxwidth {
// text is too wide to be put into one line, wrap it
wrapper := wordwrap.Wrapper(data.maxwidth-currentWidth, false)
wrapped := wrapper(entry)
// and indent it
for idx, line := range strings.Split(wrapped, "\n") {
if idx == 0 {
entry = line
} else {
entry += "\n " + strings.Repeat(" ", currentWidth) + line
}
}
}
currentWidth += data.lenHeaders[idx]
if isInt(entry) && data.alignInts {
// align right
fmt.Print(strings.Repeat(" ", data.lenHeaders[idx]-length), entry)
} else {
} else if length < data.lenHeaders[idx] && idx+1 != len(entries) {
// pad right, if required
fmt.Print(entry, strings.Repeat(" ", data.lenHeaders[idx]-length))
} else {
// no padding for last entry
fmt.Print(entry)
}
if idx < len(data.Headers)-1 {