scip/esctl
1
0
Fork 0
mirror of https://codeberg.org/scip/esctl.git synced 2026-08-25 01:34:19 +02:00
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: scip:0.0.26
Branches Tags
scip:main scip:table scip:feature/reindex scip:hide-ds-indices scip:fix/lintmap scip:mapper scip:feature/index-templates-filter scip:fix/crashes scip:enhance/go scip:feature/esql scip:fix/api-repl-headers scip:feature/node-usage scip:feature/api-repl-human-cat scip:fix/node-show-crash scip:feature/enhance-node-status scip:feature/config-load-test scip:feature/health scip:fix/es8-headers scip:fix/releaser scip:intern/delint scip:internal/gofix scip:feature/allocation-explain scip:internal/rework-table scip:feature/catch-license-errors scip:feature/automate
scip:0.0.27 scip:0.0.26 scip:0.0.25 scip:0.0.24 scip:0.0.23 scip:0.0.22 scip:0.0.21 scip:0.0.20 scip:0.0.19 scip:0.0.18 scip:0.0.17 scip:0.0.16 scip:0.0.15 scip:0.0.14 scip:0.0.13 scip:0.0.12 scip:0.0.11 scip:0.0.10 scip:0.0.9 scip:0.0.8 scip:0.0.7 scip:0.0.6 scip:0.0.5 scip:0.0.4 scip:0.0.3 scip:0.0.2
..
compare: scip:feature/automate
Branches Tags
scip:main scip:table scip:feature/reindex scip:hide-ds-indices scip:fix/lintmap scip:mapper scip:feature/index-templates-filter scip:fix/crashes scip:enhance/go scip:feature/esql scip:fix/api-repl-headers scip:feature/node-usage scip:feature/api-repl-human-cat scip:fix/node-show-crash scip:feature/enhance-node-status scip:feature/config-load-test scip:feature/health scip:fix/es8-headers scip:fix/releaser scip:intern/delint scip:internal/gofix scip:feature/allocation-explain scip:internal/rework-table scip:feature/catch-license-errors scip:feature/automate
scip:0.0.27 scip:0.0.26 scip:0.0.25 scip:0.0.24 scip:0.0.23 scip:0.0.22 scip:0.0.21 scip:0.0.20 scip:0.0.19 scip:0.0.18 scip:0.0.17 scip:0.0.16 scip:0.0.15 scip:0.0.14 scip:0.0.13 scip:0.0.12 scip:0.0.11 scip:0.0.10 scip:0.0.9 scip:0.0.8 scip:0.0.7 scip:0.0.6 scip:0.0.5 scip:0.0.4 scip:0.0.3 scip:0.0.2

1 Commits
0.0.26 ... feature/au

Author SHA1 Message Date
Thomas von Dein
489074da01 started implementing auto config loader 2026-07-02 13:49:32 +02:00
82 changed files with 1334 additions and 4241 deletions
Expand all files Collapse all files

3
.gitignore vendored
View File

@@ -32,6 +32,3 @@ esctl
*.log
cpu.profile
t
single

110
.golangci.yml
View File

@@ -1,110 +0,0 @@
version: "2"
linters:
enable:
- errcheck
- govet
- ineffassign
- staticcheck
- unused
- arangolint
- asasalint
- asciicheck
- bidichk
- bodyclose
- canonicalheader
- clickhouselint
- containedctx
#- contextcheck
- copyloopvar
#- cyclop
- decorder
- dogsled
- dupword
- durationcheck
- embeddedstructfieldcheck
- errchkjson
- errname
#- errorlint
- exhaustive
- exptostd
- fatcontext
- funcorder
#- funlen
- ginkgolinter
- gocheckcompilerdirectives
- gochecknoinits
- gochecksumtype
#- gocognit
- gocritic
#- gocyclo
#- godoclint
- goheader
- gomoddirectives
- gomodguard_v2
- goprintffuncname
#- gosec
- gosmopolitan
- grouper
- iface
- importas
- inamedparam
- interfacebloat
- intrange
- iotamixing
- lll
- loggercheck
- makezero
- misspell
- modernize
- nakedret
#- nestif
- nilerr
- nilnesserr
- nlreturn
- nonamedreturns
- nosprintfhostport
- paralleltest
- perfsprint
- prealloc
#- predeclared
- promlinter
- protogetter
- reassign
- recvcheck
#- revive
- rowserrcheck
- sloglint
- spancheck
- sqlclosecheck
- tagalign
- testableexamples
- testifylint
- testpackage
- thelper
- tparallel
- unconvert
- unparam
- unqueryvet
- usestdlibvars
- usetesting
- varnamelen
- wastedassign
- whitespace
- wsl_v5
- zerologlint
settings:
varnamelen:
ignore-names:
- err
- wg
- mu
- ts
- to
- es
- op
- fd
- id
- fn

4
.woodpecker/build.yaml
View File

@@ -2,7 +2,7 @@ matrix:
platform:
- linux/amd64
goversion:
- 1.26.4
- 1.25.8
labels:
platform: ${platform}
@@ -21,6 +21,6 @@ steps:
event: [push,manual]
image: golang:${goversion}
commands:
- curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.12.2
- curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.5.0
- golangci-lint --version
- golangci-lint run ./...

13
Makefile
View File

@@ -62,16 +62,17 @@ install: buildlocal
clean:
rm -rf $(tool) coverage.out testdata t/out pkg/es/openspec.go
test: clean buildlocal
make -C t test
test: clean
mkdir -p t/out
go test ./... $(ARGS)
testlint: test lint
lint-basic:
golangci-lint run --enable-only errcheck,govet,ineffassign,staticcheck,unused
lint:
golangci-lint run --show-stats=false
golangci-lint run
lint-full:
golangci-lint run --enable-all --exclude-use-default --disable exhaustivestruct,exhaustruct,depguard,interfacer,deadcode,golint,structcheck,scopelint,varcheck,ifshort,maligned,nosnakecase,godot,funlen,gofumpt,cyclop,noctx,gochecknoglobals,paralleltest
testfuzzy: clean
go test -fuzz ./... $(ARGS)

40
README.md
View File

@@ -23,13 +23,10 @@ Features:
- Shell completion support (bash, zsh and fish). Put this into your
rc: `source <(esctl completion bash)`.
- Cluster settings can be viewed and modified.
- Comprehensive cluster status.
- Cluster reroute support.
- Search: you can search indices using full text or by fields, select
logical condition (OR, AND), use PIT, limit datetime (ES date math
can be used), etc. It is however not yet possible to create
recursive searches like: `(cond1 AND cond2) OR (cond3 OR cond4)`.
- Search using ES|QL language: `esctl searchql`.
- Cross cluster replication (ccr): view, pause, resume, delete
replication. You can also manage follower configuration.
- Index management: manage aliases, create, modify, delete indices,
@@ -50,9 +47,7 @@ Features:
elasticsearch API. You can run API calls on the current selected
cluster w/o the hassle to specify the whole url, credentials etc. It
has line editing and history support. If `jq` is installed output
JSON will be syntax highlighted. A simple internal pager will be
used if output exceeds the terminal height. You can tweak this using
the `$ES_JSON_PAGER` environment variable (I'd recommend [fx](https://fx.wtf/)).
JSON will be syntax highlighted.
- Doc support. You can put, delete and show docs for an index. Very
handy if you want to play with it. Just create a new index:
`esctl index create foo` and then insert docs into it for search
@@ -353,30 +348,6 @@ $ esctl search -i foo* -F title=zeitbuchung message=pause | jq
}
```
You can also search using [ES|QL](https://www.elastic.co/docs/reference/query-languages/esql/esql-getting-started):
```console
$ esctl searchql "from hyperdrive | sort @timestamp | limit 5"
@TIMESTAMP MESSAGE TAG
2026-06-24T08:24:56.000Z arosu loop
2026-06-24T08:24:58.000Z hami loop
2026-06-24T08:24:59.000Z ishininu loop
2026-06-24T08:25:00.000Z uyomoruron loop
2026-06-24T08:25:02.000Z ishimime loop
```
There are several output modes (json, yaml, csv), to get esql output as CSV:
```console
$ esctl searchql "from hyperdrive | sort @timestamp | limit 5" -o csv
@timestamp,message,tag
2026-06-24T08:24:56.000Z,arosu,loop
2026-06-24T08:24:58.000Z,hami,loop
2026-06-24T08:24:59.000Z,ishininu,loop
2026-06-24T08:25:00.000Z,uyomoruron,loop
2026-06-24T08:25:02.000Z,ishimime,loop
```
To check which field mappings are available for an index:
```console
$ esctl index show foo2
@@ -561,9 +532,9 @@ index - manage indicies
update - update an index
delete - delete an index
close - close an index
allocation - explain index allocation
fields - show info about field capabilities
ilm - show ilm status
du - show index disk usage
alias - manage index aliases
create - create an index alias
list - list index aliases
@@ -575,13 +546,10 @@ index - manage indicies
create - create a new index template
update - update a new index template
delete - delete an index template
license - manage cluster license
show - show details about the cluster license
node - manage nodes
list - list nodes
show - show details about a node
clients - show node http clients
usage - show node usage stats
role - manage roles
list - list roles
show - show details about a role
@@ -590,7 +558,6 @@ search - search within an index
shard - manage shards
list - list shards
show - show details about a shard
allocation - explain shard allocation
snapshot - manage snapshots
list - list snapshots
show - show details about a snapshot
@@ -600,8 +567,7 @@ task - manage tasks
version - show esctl version information
debug - developer only
help-jsonpath - show jsonpath help
help-usage - show overview of all available commands
help-esql - show esql help
help-command-overview - show overview of all available commands
```
# Development

4
TODO.md
View File

@@ -3,6 +3,10 @@
- index show: add more details, see screenshots
- add shard explain, aka:
get /_cluster/allocation/explain {"index":"yourindex", "primary": true, "shard":0}
- add datastream support:
https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-get-data-stream

12
assets/assets.go
View File

@@ -24,11 +24,10 @@ import (
"github.com/go-openapi/swag/loading"
)
//go:embed *.json *.md
//go:embed *.json
var AssetFS embed.FS
var OpenAPI *loads.Document
var EsQlCheatSheet string
func LoadAssetOpenApi() {
doc, err := loads.Spec(
@@ -42,12 +41,3 @@ func LoadAssetOpenApi() {
OpenAPI = doc
}
func LoadEsql() {
md, err := AssetFS.ReadFile("esql.md")
if err != nil {
panic(err)
}
EsQlCheatSheet = string(md)
}

951
assets/esql.md
View File

@@ -1,951 +0,0 @@
<!-- courtesy https://github.com/linkan-per/ES-QL-Cheat-Sheet by |linkan-per -->
# ES|QL (Elasticsearch Query Language) Cheat Sheet
## Table of Contents
1. [Introduction](#introduction)
2. [Basic Query Structure](#basic-query-structure)
3. [Source Commands](#source-commands)
4. [Processing Commands](#processing-commands)
5. [Data Selection & Filtering](#data-selection--filtering)
6. [Aggregations & Statistics](#aggregations--statistics)
7. [String Functions](#string-functions)
8. [Mathematical Functions](#mathematical-functions)
9. [Date/Time Functions](#datetime-functions)
10. [Type Conversion Functions](#type-conversion-functions)
11. [Conditional Functions](#conditional-functions)
12. [Array Functions](#array-functions)
13. [Join Operations (LOOKUP)](#join-operations-lookup)
14. [Sorting & Limiting](#sorting--limiting)
15. [Grouping & Aggregating](#grouping--aggregating)
16. [Advanced Patterns](#advanced-patterns)
---
## Introduction
ES|QL is Elasticsearch's new query language designed for data exploration, analysis, and transformation. It uses a pipe (`|`) syntax to chain commands together.
**Basic Syntax:**
```
FROM <data-source>
| <processing-command>
| <processing-command>
| ...
```
---
## Basic Query Structure
### Simple Query
```esql
FROM logs-*
| LIMIT 10
```
### Query with Multiple Commands
```esql
FROM employees
| WHERE department == "Engineering"
| KEEP name, salary, hire_date
| SORT salary DESC
| LIMIT 5
```
---
## Source Commands
### FROM - Specify Data Source
```esql
// From a single index
FROM logs-2024
// From multiple indices with wildcard
FROM logs-*, metrics-*
// From specific indices
FROM index1, index2, index3
// With metadata
FROM logs-* METADATA _id, _index
```
### ROW - Generate Inline Data
```esql
// Create a single row
ROW name = "John", age = 30, city = "NYC"
// Multiple rows
ROW a = 1, b = "x"
| EVAL c = a * 10
```
---
## Processing Commands
### KEEP - Select Specific Fields
```esql
FROM employees
| KEEP name, department, salary
// Keep with pattern
FROM logs-*
| KEEP @timestamp, message, host.*
```
### DROP - Remove Specific Fields
```esql
FROM employees
| DROP password, ssn, internal_notes
// Drop with pattern
FROM logs-*
| DROP *.keyword
```
### RENAME - Rename Fields
```esql
FROM employees
| RENAME emp_name AS name, emp_dept AS department
// Multiple renames
FROM logs-*
| RENAME source.ip AS src_ip, destination.ip AS dst_ip
```
---
## Data Selection & Filtering
### WHERE - Filter Rows
```esql
// Equality
FROM employees
| WHERE department == "Sales"
// Comparison operators
FROM products
| WHERE price > 100 AND stock < 50
// IS NULL / IS NOT NULL
FROM logs-*
| WHERE error_code IS NOT NULL
// IN operator
FROM employees
| WHERE department IN ("Sales", "Marketing", "HR")
// LIKE operator (wildcards)
FROM logs-*
| WHERE message LIKE "*error*"
// RLIKE operator (regex)
FROM logs-*
| WHERE message RLIKE "error|exception|failure"
// NOT operator
FROM employees
| WHERE NOT department == "IT"
// Multiple conditions
FROM orders
| WHERE status == "completed"
AND total_amount > 1000
AND order_date >= "2024-01-01"
```
---
## Aggregations & Statistics
### STATS - Aggregate Functions
#### COUNT
```esql
// Count all rows
FROM logs-*
| STATS count = COUNT()
// Count distinct
FROM employees
| STATS unique_departments = COUNT_DISTINCT(department)
// Count by group
FROM logs-*
| STATS event_count = COUNT() BY log_level
```
#### SUM, AVG, MIN, MAX
```esql
FROM sales
| STATS
total_revenue = SUM(amount),
avg_sale = AVG(amount),
min_sale = MIN(amount),
max_sale = MAX(amount)
// With grouping
FROM sales
| STATS
total = SUM(amount),
average = AVG(amount)
BY product_category
```
#### MEDIAN, PERCENTILE
```esql
FROM response_times
| STATS
median_time = MEDIAN(duration),
p95 = PERCENTILE(duration, 95),
p99 = PERCENTILE(duration, 99)
```
#### Multiple Aggregations
```esql
FROM orders
| STATS
order_count = COUNT(),
total_revenue = SUM(amount),
avg_order_value = AVG(amount),
unique_customers = COUNT_DISTINCT(customer_id)
BY region, product_category
```
---
## String Functions
### CONCAT - Concatenate Strings
```esql
FROM employees
| EVAL full_name = CONCAT(first_name, " ", last_name)
// With separator
FROM logs-*
| EVAL log_info = CONCAT(level, ": ", message)
```
### SUBSTRING - Extract Substring
```esql
FROM employees
| EVAL first_initial = SUBSTRING(first_name, 0, 1)
// Extract with length
FROM products
| EVAL short_code = SUBSTRING(product_id, 0, 5)
```
### LENGTH - String Length
```esql
FROM messages
| EVAL message_length = LENGTH(message)
| WHERE message_length > 100
```
### TRIM, LTRIM, RTRIM - Remove Whitespace
```esql
FROM user_input
| EVAL cleaned = TRIM(input_field)
| EVAL left_trimmed = LTRIM(input_field)
| EVAL right_trimmed = RTRIM(input_field)
```
### UPPER, LOWER - Case Conversion
```esql
FROM employees
| EVAL name_upper = UPPER(name)
| EVAL email_lower = LOWER(email)
```
### REPLACE - Replace String
```esql
FROM logs-*
| EVAL cleaned_message = REPLACE(message, "ERROR", "Warning")
```
### SPLIT - Split String into Array
```esql
FROM logs-*
| EVAL tags_array = SPLIT(tags, ",")
```
### STARTS_WITH, ENDS_WITH
```esql
FROM files
| WHERE STARTS_WITH(filename, "log_")
| WHERE ENDS_WITH(filename, ".txt")
```
---
## Mathematical Functions
### Basic Operations
```esql
FROM sales
| EVAL
total = price * quantity,
discount_price = price * 0.9,
tax = price * 0.08
// Multiple operations
FROM metrics
| EVAL
sum_val = field1 + field2,
diff_val = field1 - field2,
product = field1 * field2,
ratio = field1 / field2,
remainder = field1 % field2
```
### ABS - Absolute Value
```esql
FROM transactions
| EVAL abs_amount = ABS(transaction_amount)
```
### ROUND, FLOOR, CEIL
```esql
FROM measurements
| EVAL
rounded = ROUND(value, 2),
floored = FLOOR(value),
ceiled = CEIL(value)
```
### POW - Power
```esql
FROM data
| EVAL squared = POW(value, 2)
| EVAL cubed = POW(value, 3)
```
### SQRT - Square Root
```esql
FROM measurements
| EVAL sqrt_value = SQRT(value)
```
### LOG, LOG10
```esql
FROM data
| EVAL
natural_log = LOG(value),
log_base_10 = LOG10(value)
```
### GREATEST, LEAST
```esql
FROM comparisons
| EVAL
max_val = GREATEST(val1, val2, val3),
min_val = LEAST(val1, val2, val3)
```
---
## Date/Time Functions
### NOW - Current Timestamp
```esql
FROM logs-*
| EVAL current_time = NOW()
```
### DATE_EXTRACT - Extract Date Parts
```esql
FROM events
| EVAL
year = DATE_EXTRACT("year", @timestamp),
month = DATE_EXTRACT("month", @timestamp),
day = DATE_EXTRACT("day", @timestamp),
hour = DATE_EXTRACT("hour", @timestamp),
minute = DATE_EXTRACT("minute", @timestamp),
day_of_week = DATE_EXTRACT("day_of_week", @timestamp)
```
### DATE_FORMAT - Format Date
```esql
FROM events
| EVAL formatted_date = DATE_FORMAT("yyyy-MM-dd", @timestamp)
| EVAL custom_format = DATE_FORMAT("MMM dd, yyyy HH:mm", @timestamp)
```
### DATE_TRUNC - Truncate Date
```esql
FROM logs-*
| EVAL
hour_bucket = DATE_TRUNC("hour", @timestamp),
day_bucket = DATE_TRUNC("day", @timestamp),
month_bucket = DATE_TRUNC("month", @timestamp)
```
### DATE_DIFF - Date Difference
```esql
FROM orders
| EVAL days_since_order = DATE_DIFF("days", order_date, NOW())
| EVAL hours_to_delivery = DATE_DIFF("hours", order_date, delivery_date)
```
### DATE_PARSE - Parse String to Date
```esql
FROM data
| EVAL parsed_date = DATE_PARSE("yyyy-MM-dd", date_string)
```
---
## Type Conversion Functions
### TO_STRING - Convert to String
```esql
FROM data
| EVAL id_string = TO_STRING(id)
| EVAL amount_string = TO_STRING(amount)
```
### TO_INTEGER, TO_LONG - Convert to Integer
```esql
FROM data
| EVAL age_int = TO_INTEGER(age_string)
| EVAL id_long = TO_LONG(id_string)
```
### TO_DOUBLE - Convert to Double
```esql
FROM data
| EVAL price_double = TO_DOUBLE(price_string)
```
### TO_BOOLEAN - Convert to Boolean
```esql
FROM data
| EVAL is_active = TO_BOOLEAN(active_string)
```
### TO_DATETIME - Convert to DateTime
```esql
FROM data
| EVAL timestamp = TO_DATETIME(date_string)
```
### TO_IP - Convert to IP Address
```esql
FROM logs-*
| EVAL ip_address = TO_IP(ip_string)
```
---
## Conditional Functions
### CASE - Conditional Logic
```esql
FROM employees
| EVAL salary_grade = CASE(
salary < 50000, "Entry",
salary < 80000, "Mid",
salary < 120000, "Senior",
"Executive"
)
// With multiple conditions
FROM orders
| EVAL order_status = CASE(
status == "pending" AND days_old > 7, "Overdue",
status == "pending", "Processing",
status == "shipped", "In Transit",
status == "delivered", "Completed",
"Unknown"
)
```
### COALESCE - Return First Non-Null Value
```esql
FROM data
| EVAL display_name = COALESCE(nickname, first_name, username, "Unknown")
```
### IF - Simple Conditional
```esql
FROM products
| EVAL stock_status =
CASE(stock > 0, "Available", "Out of Stock")
// Nested conditions
FROM employees
| EVAL bonus = CASE(
performance_rating >= 4.5, salary * 0.15,
performance_rating >= 3.5, salary * 0.10,
performance_rating >= 2.5, salary * 0.05,
0
)
```
---
## Array Functions
### MV_COUNT - Count Array Elements
```esql
FROM logs-*
| EVAL tag_count = MV_COUNT(tags)
| WHERE tag_count > 3
```
### MV_AVG, MV_SUM, MV_MIN, MV_MAX - Array Aggregations
```esql
FROM metrics
| EVAL
avg_value = MV_AVG(values),
total = MV_SUM(values),
min_value = MV_MIN(values),
max_value = MV_MAX(values)
```
### MV_CONCAT - Concatenate Array Elements
```esql
FROM logs-*
| EVAL all_tags = MV_CONCAT(tags, ", ")
```
### MV_DEDUPE - Remove Duplicates from Array
```esql
FROM data
| EVAL unique_values = MV_DEDUPE(values)
```
### MV_FIRST, MV_LAST - Get First/Last Element
```esql
FROM logs-*
| EVAL first_tag = MV_FIRST(tags)
| EVAL last_tag = MV_LAST(tags)
```
### MV_SLICE - Extract Array Slice
```esql
FROM data
| EVAL first_three = MV_SLICE(values, 0, 3)
```
---
## Join Operations (LOOKUP)
ES|QL uses ENRICH (similar to LOOKUP/JOIN) to join data from enrich policies.
### Prerequisites: Create Enrich Policy
First, create an enrich policy in Kibana Dev Tools:
```json
PUT /_enrich/policy/user_lookup
{
"match": {
"indices": "users",
"match_field": "user_id",
"enrich_fields": ["username", "email", "department"]
}
}
POST /_enrich/policy/user_lookup/_execute
```
### ENRICH - Join/Lookup Data
```esql
FROM logs-*
| ENRICH user_lookup ON user_id
| KEEP @timestamp, user_id, username, email, message
// With field renaming
FROM transactions
| ENRICH product_lookup ON product_id WITH product_name, category, price
| KEEP transaction_id, product_name, category, quantity, price
// Multiple enrichments
FROM orders
| ENRICH customer_lookup ON customer_id WITH customer_name, customer_tier
| ENRICH product_lookup ON product_id WITH product_name, product_category
| KEEP order_id, customer_name, product_name, order_amount
```
### Complex Join Example
```esql
FROM orders
| ENRICH customer_lookup ON customer_id
WITH customer_name, customer_email, customer_segment
| ENRICH product_lookup ON product_id
WITH product_name, product_category, product_price
| EVAL total_price = quantity * product_price
| WHERE customer_segment == "Premium"
| STATS
total_orders = COUNT(),
total_revenue = SUM(total_price)
BY customer_name, product_category
| SORT total_revenue DESC
```
---
## Sorting & Limiting
### SORT - Order Results
```esql
// Ascending order (default)
FROM employees
| SORT salary
// Descending order
FROM employees
| SORT salary DESC
// Multiple fields
FROM employees
| SORT department ASC, salary DESC
// With nulls first/last
FROM data
| SORT value DESC NULLS FIRST
```
### LIMIT - Limit Results
```esql
// Get first 10 rows
FROM logs-*
| LIMIT 10
// Top 5 highest salaries
FROM employees
| SORT salary DESC
| LIMIT 5
// Pagination (skip and limit)
FROM products
| SORT price
| LIMIT 20 // Results 0-19
```
### HEAD - Get First N Rows (Alias for LIMIT)
```esql
FROM logs-*
| HEAD 100
```
---
## Grouping & Aggregating
### GROUP BY with STATS
```esql
// Single field grouping
FROM sales
| STATS total_sales = SUM(amount) BY region
// Multiple field grouping
FROM orders
| STATS
order_count = COUNT(),
total_revenue = SUM(amount)
BY region, product_category, sales_rep
// Time-based grouping
FROM logs-*
| EVAL hour = DATE_TRUNC("hour", @timestamp)
| STATS event_count = COUNT() BY hour, log_level
| SORT hour DESC
```
### Complex Aggregation Example
```esql
FROM sales_data
| WHERE order_date >= "2024-01-01"
| EVAL month = DATE_TRUNC("month", order_date)
| STATS
total_orders = COUNT(),
total_revenue = SUM(amount),
avg_order_value = AVG(amount),
unique_customers = COUNT_DISTINCT(customer_id),
max_order = MAX(amount),
min_order = MIN(amount)
BY month, region, product_category
| EVAL revenue_per_customer = total_revenue / unique_customers
| WHERE total_orders > 100
| SORT month DESC, total_revenue DESC
| LIMIT 50
```
---
## Advanced Patterns
### Window Functions Pattern
```esql
// Running total by group
FROM sales
| SORT date
| STATS
daily_sales = SUM(amount),
order_count = COUNT()
BY date, region
| SORT region, date
```
### Pivoting Data
```esql
// Count by status and priority
FROM tickets
| STATS ticket_count = COUNT() BY status, priority
| SORT status, priority
```
### Finding Duplicates
```esql
FROM users
| STATS count = COUNT() BY email
| WHERE count > 1
| SORT count DESC
```
### Time Series Analysis
```esql
FROM metrics-*
| EVAL
hour = DATE_TRUNC("hour", @timestamp),
day = DATE_EXTRACT("day", @timestamp)
| STATS
avg_cpu = AVG(cpu_percent),
max_cpu = MAX(cpu_percent),
avg_memory = AVG(memory_percent)
BY hour, host
| WHERE avg_cpu > 80
| SORT hour DESC
```
### Percentage Calculations
```esql
FROM sales
| STATS
total_sales = SUM(amount),
count = COUNT()
BY product_category
| EVAL percentage = ROUND(total_sales / SUM(total_sales) * 100, 2)
| SORT percentage DESC
```
### Top N per Group
```esql
// Top 3 products per category by sales
FROM sales
| STATS total_sales = SUM(amount) BY product_category, product_name
| SORT product_category, total_sales DESC
// Note: ES|QL doesn't have native PARTITION BY,
// so you may need to process this in multiple queries or use aggregations
```
### Data Cleaning
```esql
FROM raw_data
| EVAL
// Clean whitespace
cleaned_name = TRIM(name),
// Standardize case
email_lower = LOWER(email),
// Replace values
status = REPLACE(status, "N/A", "Unknown"),
// Handle nulls
age = COALESCE(age, 0),
// Validate ranges
valid_age = CASE(age < 0 OR age > 150, NULL, age)
| WHERE cleaned_name IS NOT NULL
| DROP name, email
| RENAME cleaned_name AS name, email_lower AS email
```
### Cohort Analysis
```esql
FROM user_events
| EVAL
signup_month = DATE_TRUNC("month", signup_date),
event_month = DATE_TRUNC("month", event_date)
| STATS
active_users = COUNT_DISTINCT(user_id)
BY signup_month, event_month
| SORT signup_month, event_month
```
### Anomaly Detection Pattern
```esql
FROM metrics-*
| EVAL hour = DATE_TRUNC("hour", @timestamp)
| STATS
avg_value = AVG(value),
stddev = SQRT(AVG(POW(value - AVG(value), 2)))
BY hour
| EVAL
upper_bound = avg_value + (2 * stddev),
lower_bound = avg_value - (2 * stddev)
```
---
## Complete Real-World Examples
### Example 1: User Activity Dashboard
```esql
FROM user_logs-*
| WHERE @timestamp >= NOW() - 7 days
| ENRICH user_lookup ON user_id WITH username, user_tier
| EVAL day = DATE_TRUNC("day", @timestamp)
| STATS
daily_active_users = COUNT_DISTINCT(user_id),
total_sessions = COUNT(),
avg_session_duration = AVG(session_duration)
BY day, user_tier
| EVAL avg_duration_minutes = ROUND(avg_session_duration / 60, 2)
| SORT day DESC, user_tier
```
### Example 2: E-commerce Sales Report
```esql
FROM orders
| WHERE order_date >= "2024-01-01"
| ENRICH customer_lookup ON customer_id
WITH customer_name, customer_segment
| ENRICH product_lookup ON product_id
WITH product_name, product_category, cost_price
| EVAL
profit = (price - cost_price) * quantity,
month = DATE_TRUNC("month", order_date)
| STATS
total_orders = COUNT(),
total_revenue = SUM(price * quantity),
total_profit = SUM(profit),
avg_order_value = AVG(price * quantity),
unique_customers = COUNT_DISTINCT(customer_id)
BY month, product_category, customer_segment
| EVAL profit_margin = ROUND(total_profit / total_revenue * 100, 2)
| WHERE total_revenue > 10000
| SORT month DESC, total_revenue DESC
| LIMIT 100
```
### Example 3: Security Log Analysis
```esql
FROM security-logs-*
| WHERE @timestamp >= NOW() - 24 hours
| WHERE event_type IN ("login_failed", "suspicious_activity")
| EVAL hour = DATE_TRUNC("hour", @timestamp)
| STATS
event_count = COUNT(),
unique_ips = COUNT_DISTINCT(source_ip),
unique_users = COUNT_DISTINCT(username)
BY hour, event_type, country
| WHERE event_count > 100
| SORT hour DESC, event_count DESC
```
### Example 4: Application Performance Monitoring
```esql
FROM apm-*
| WHERE @timestamp >= NOW() - 1 hour
| EVAL
response_category = CASE(
response_time < 100, "Fast",
response_time < 500, "Medium",
response_time < 1000, "Slow",
"Very Slow"
),
minute = DATE_TRUNC("minute", @timestamp)
| STATS
request_count = COUNT(),
avg_response = AVG(response_time),
p95_response = PERCENTILE(response_time, 95),
p99_response = PERCENTILE(response_time, 99),
error_count = COUNT() WHERE status_code >= 400
BY minute, endpoint, response_category
| EVAL error_rate = ROUND(error_count / request_count * 100, 2)
| WHERE request_count > 10
| SORT minute DESC, avg_response DESC
```
---
## Tips & Best Practices
1. **Use KEEP instead of SELECT** - More explicit about which fields to retain
2. **Filter early with WHERE** - Reduce data processing by filtering before aggregations
3. **Use DATE_TRUNC for time bucketing** - Essential for time series analysis
4. **Leverage ENRICH for joins** - Pre-create enrich policies for frequently joined data
5. **Use EVAL for calculated fields** - Create derived fields before aggregation
6. **Combine multiple conditions in WHERE** - More efficient than multiple WHERE clauses
7. **Use STATS with BY for grouping** - Replaces traditional GROUP BY
8. **Sort after aggregation** - More efficient than sorting before
9. **Use LIMIT to control output size** - Especially important for large datasets
10. **Use metadata fields when needed** - Access _id, _index with METADATA keyword
---
## Common Patterns Cheat Sheet
```esql
// Count by field
FROM index | STATS count = COUNT() BY field
// Top N
FROM index | STATS value = SUM(amount) BY category | SORT value DESC | LIMIT 10
// Time series
FROM index | EVAL bucket = DATE_TRUNC("hour", @timestamp) | STATS count = COUNT() BY bucket
// Percentage of total
FROM index | STATS total = SUM(amount) BY category | EVAL pct = total / SUM(total) * 100
// Filter nulls
FROM index | WHERE field IS NOT NULL
// String matching
FROM index | WHERE field LIKE "*pattern*"
// Date range
FROM index | WHERE @timestamp >= NOW() - 7 days
// Multiple aggregations
FROM index | STATS count = COUNT(), sum = SUM(val), avg = AVG(val) BY group
// Conditional aggregation
FROM index | STATS error_count = COUNT() WHERE status == "error" BY service
```
---
## Comparison with Traditional SQL
| SQL | ES|QL |
|-----|-------|
| SELECT * | FROM index |
| SELECT field1, field2 | FROM index \| KEEP field1, field2 |
| WHERE condition | WHERE condition (same) |
| GROUP BY field | STATS ... BY field |
| ORDER BY field | SORT field |
| LIMIT 10 | LIMIT 10 (same) |
| COUNT(*) | STATS count = COUNT() |
| SUM(field) | STATS total = SUM(field) |
| AVG(field) | STATS avg = AVG(field) |
| JOIN | ENRICH (using enrich policies) |
| CASE WHEN | CASE(...) |
| CONCAT(a, b) | CONCAT(a, b) (same) |
---
## Resources
- Official ES|QL Documentation: https://www.elastic.co/guide/en/elasticsearch/reference/current/esql.html
- ES|QL Functions Reference: https://www.elastic.co/guide/en/elasticsearch/reference/current/esql-functions.html
- Enrich Processor: https://www.elastic.co/guide/en/elasticsearch/reference/current/enrich-processor.html
---
*Last Updated: 2024*
*ES|QL is actively evolving - check official documentation for latest features*

18
cmd/api.go
View File

@@ -72,7 +72,7 @@ func ApiShow(conf *cfg.Config) *cli.Command {
},
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Capi)
complete(cmd, Capi)
},
}
}
@@ -83,22 +83,6 @@ func ApiRepl(conf *cfg.Config) *cli.Command {
Aliases: []string{"shell"},
Usage: "interactive API repl",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "pager",
Usage: "external viewer program (default:internal)",
Destination: &conf.Pager,
Aliases: []string{"p"},
Sources: cli.EnvVars("PAGER", "ES_JSON_PAGER"),
},
&cli.BoolFlag{
Name: "human-readable-cat",
Usage: "enable human readable /_cat output",
Destination: &conf.HumanCat,
Aliases: []string{"H"},
},
},
Action: func(ctx context.Context, cmd *cli.Command) error {
return es.ApiRepl(conf)
},

4
cmd/ccr.go
View File

@@ -59,7 +59,7 @@ func CcrStatus(conf *cfg.Config) *cli.Command {
},
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Ccluster)
complete(cmd, Ccluster)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
@@ -115,7 +115,7 @@ func CcrRemoteInfo(conf *cfg.Config) *cli.Command {
UsageText: "info [options] [<index>]",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cindex)
complete(cmd, Cindex)
},
Action: func(ctx context.Context, cmd *cli.Command) error {

14
cmd/ccr_follower.go
View File

@@ -60,7 +60,7 @@ func CcrFollowerRenew(conf *cfg.Config) *cli.Command {
},
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cindex)
complete(cmd, Cindex)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
@@ -82,7 +82,7 @@ func CcrFollowerResume(conf *cfg.Config) *cli.Command {
UsageText: "resume [options] <index>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cindex)
complete(cmd, Cindex)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
@@ -104,7 +104,7 @@ func CcrFollowerPause(conf *cfg.Config) *cli.Command {
UsageText: "pause [options] <index>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cindex)
complete(cmd, Cindex)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
@@ -126,7 +126,7 @@ func CcrFollowerUnfollow(conf *cfg.Config) *cli.Command {
UsageText: "unfollow [options] <index>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cindex)
complete(cmd, Cindex)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
@@ -158,7 +158,7 @@ func CcrFollowerAdd(conf *cfg.Config) *cli.Command {
},
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cindex)
complete(cmd, Cindex)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
@@ -181,7 +181,7 @@ func CcrFollowerDelete(conf *cfg.Config) *cli.Command {
UsageText: "delete <index>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cindex)
complete(cmd, Cindex)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
@@ -205,7 +205,7 @@ func CcrFollowerShow(conf *cfg.Config) *cli.Command {
UsageText: "show <index>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cindex)
complete(cmd, Cindex)
},
Action: func(ctx context.Context, cmd *cli.Command) error {

2
cmd/cluster.go
View File

@@ -83,7 +83,7 @@ func ClusterSwitch(conf *cfg.Config) *cli.Command {
Aliases: []string{"ctx"},
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Ccluster)
complete(cmd, Ccluster)
},
Action: func(ctx context.Context, cmd *cli.Command) error {

9
cmd/cluster_reroute.go
View File

@@ -50,7 +50,7 @@ func ClusterRerouteMove(conf *cfg.Config) *cli.Command {
UsageText: "move [options] <index>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cindex)
complete(cmd, Cindex)
},
Flags: []cli.Flag{
@@ -95,7 +95,7 @@ func ClusterRerouteAllocateReplica(conf *cfg.Config) *cli.Command {
UsageText: "allocate-replica [options] <index>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cindex)
complete(cmd, Cindex)
},
Flags: []cli.Flag{
@@ -133,7 +133,7 @@ func ClusterRerouteCancel(conf *cfg.Config) *cli.Command {
UsageText: "cancel [options] <index>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cindex)
complete(cmd, Cindex)
},
Flags: []cli.Flag{
@@ -177,6 +177,7 @@ func ClusterRerouteAllocatePrimary(conf *cfg.Config, stale bool) *cli.Command {
if stale {
name = "allocate-stale-primary"
usage = "allocate a stale primary shard to a node"
}
return &cli.Command{
@@ -185,7 +186,7 @@ func ClusterRerouteAllocatePrimary(conf *cfg.Config, stale bool) *cli.Command {
UsageText: name + " [options] <index>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cindex)
complete(cmd, Cindex)
},
Flags: []cli.Flag{

4
cmd/cluster_settings.go
View File

@@ -104,10 +104,6 @@ func ClusterSettingsSet(conf *cfg.Config) *cli.Command {
},
},
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cclustersettings)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
args := cmd.Args()

18
cmd/completion.go
View File

@@ -33,18 +33,22 @@ const (
Capi
Cilm
Cnode
Cclustersettings
)
func complete(conf *cfg.Config, cmd *cli.Command, what int) {
func complete(cmd *cli.Command, what int) {
if cmd.NArg() > 0 {
return
}
var (
list []string
err error
)
// FIXME: config should load from root.Before(), see https://github.com/urfave/cli/issues/2348
// workaround: load it directly here
conf := cfg.NewConfig()
if err := conf.Init(); err != nil {
return
}
var list []string
var err error
switch what {
case Cindex:
@@ -63,8 +67,6 @@ func complete(conf *cfg.Config, cmd *cli.Command, what int) {
list, err = es.IlmNames(conf)
case Cnode:
list, err = es.NodeNames(conf)
case Cclustersettings:
list, err = es.ClusterSettingsNames(conf)
}
if err != nil {

10
cmd/datastream.go
View File

@@ -91,7 +91,7 @@ func DatastreamShow(conf *cfg.Config) *cli.Command {
},
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cdatastream)
complete(cmd, Cdatastream)
},
}
}
@@ -113,7 +113,7 @@ func DatastreamCreate(conf *cfg.Config) *cli.Command {
},
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cdatastream)
complete(cmd, Cdatastream)
},
}
}
@@ -135,7 +135,7 @@ func DatastreamDelete(conf *cfg.Config) *cli.Command {
},
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cdatastream)
complete(cmd, Cdatastream)
},
}
}
@@ -148,7 +148,7 @@ func DatastreamRollover(conf *cfg.Config) *cli.Command {
UsageText: "rollover <data stream>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cindex)
complete(cmd, Cindex)
},
Flags: []cli.Flag{
@@ -205,7 +205,7 @@ func DatastreamIlm(conf *cfg.Config) *cli.Command {
UsageText: "ds ilm <name>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cdatastream)
complete(cmd, Cdatastream)
},
Action: func(ctx context.Context, cmd *cli.Command) error {

2
cmd/ilm.go
View File

@@ -123,7 +123,7 @@ func IlmShow(conf *cfg.Config) *cli.Command {
},
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cilm)
complete(cmd, Cilm)
},
}
}

4
cmd/ilm_forecast.go
View File

@@ -71,7 +71,7 @@ func IlmForecastList(conf *cfg.Config) *cli.Command {
},
&cli.StringFlag{
Name: "within",
Usage: "<duration> within which to forecast",
Usage: "<duration> withing which to forecast",
Destination: &conf.Ilm.Within,
Aliases: []string{"w"},
},
@@ -121,7 +121,7 @@ func IlmForecastShow(conf *cfg.Config) *cli.Command {
},
&cli.StringFlag{
Name: "within",
Usage: "duration within which to forecast",
Usage: "duration withing which to forecast",
Destination: &conf.Ilm.Within,
Aliases: []string{"w"},
},

72
cmd/index.go
View File

@@ -19,6 +19,7 @@ package cmd
import (
"context"
"errors"
"fmt"
"codeberg.org/scip/esctl/pkg/cfg"
"codeberg.org/scip/esctl/pkg/es"
@@ -39,9 +40,9 @@ func Index(conf *cfg.Config) *cli.Command {
IndexCreate(conf, true),
IndexDelete(conf),
IndexClose(conf),
IndexAllocation(conf),
IndexFields(conf),
IndexIlm(conf),
IndexDu(conf),
// sub commands
IndexAlias(conf),
@@ -76,10 +77,10 @@ func IndexList(conf *cfg.Config) *cli.Command {
Aliases: []string{"H"},
},
&cli.BoolFlag{
Name: "failed",
Name: "reds",
Usage: "include only red failed indicies",
Destination: &conf.Failed,
Aliases: []string{"f"},
Aliases: []string{"r"},
},
&cli.StringSliceFlag{
Name: "filter",
@@ -95,6 +96,38 @@ func IndexList(conf *cfg.Config) *cli.Command {
}
}
func IndexAllocation(conf *cfg.Config) *cli.Command {
return &cli.Command{
Name: "allocation",
Aliases: []string{"a"},
Usage: "explain index allocation",
Flags: []cli.Flag{
&cli.IntFlag{
Name: "shard",
Usage: "shard number to explain for",
Destination: &conf.Shards,
Aliases: []string{"s"},
},
&cli.BoolFlag{
Name: "primary",
Usage: "explain primary allocation (default true)",
Destination: &conf.Primary,
Aliases: []string{"p"},
},
},
Action: func(ctx context.Context, cmd *cli.Command) error {
index := cmd.Args().Get(0)
if index == "" {
return errors.New("no index specified")
}
return es.IndexAllocation(conf, cmd.Args().Get(0))
},
}
}
func IndexShow(conf *cfg.Config) *cli.Command {
return &cli.Command{
Name: "show",
@@ -111,7 +144,7 @@ func IndexShow(conf *cfg.Config) *cli.Command {
},
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cindex)
complete(cmd, Cindex)
},
}
}
@@ -163,7 +196,7 @@ Valid field mapping types: integer, text, date, keyword`,
Action: func(ctx context.Context, cmd *cli.Command) error {
args := cmd.Args()
if args.Len() == 0 {
return errors.New("no index specified")
return fmt.Errorf("no index specified")
}
mappings := args.Slice()[1:]
@@ -181,7 +214,7 @@ func IndexDelete(conf *cfg.Config) *cli.Command {
UsageText: "delete <index>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cindex)
complete(cmd, Cindex)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
@@ -202,7 +235,7 @@ func IndexClose(conf *cfg.Config) *cli.Command {
UsageText: "close <index>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cindex)
complete(cmd, Cindex)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
@@ -244,7 +277,7 @@ func IndexFields(conf *cfg.Config) *cli.Command {
},
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cindex)
complete(cmd, Cindex)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
@@ -265,7 +298,7 @@ func IndexIlm(conf *cfg.Config) *cli.Command {
UsageText: "index ilm <index>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cindex)
complete(cmd, Cindex)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
@@ -278,24 +311,3 @@ func IndexIlm(conf *cfg.Config) *cli.Command {
},
}
}
func IndexDu(conf *cfg.Config) *cli.Command {
return &cli.Command{
Name: "du",
Usage: "show index disk usage",
UsageText: "index du <index>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cindex)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
index := cmd.Args().Get(0)
if index == "" {
return errors.New("no index specified")
}
return es.IndexDiskusage(conf, index)
},
}
}

9
cmd/index_alias.go
View File

@@ -38,8 +38,7 @@ func IndexAlias(conf *cfg.Config) *cli.Command {
IndexAliasDelete(conf),
IndexAliasRollover(conf),
// FIXME: implement IndexAliasAdd
// see https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-update-aliases
// IndexAliasAdd(conf),
//IndexAliasAdd(conf), // see https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-update-aliases
},
}
}
@@ -52,7 +51,7 @@ func IndexAliasCreate(conf *cfg.Config) *cli.Command {
UsageText: "create <index> <alias>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cindex)
complete(cmd, Cindex)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
@@ -76,7 +75,7 @@ func IndexAliasDelete(conf *cfg.Config) *cli.Command {
UsageText: "delete <index> <alias>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cindex)
complete(cmd, Cindex)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
@@ -100,7 +99,7 @@ func IndexAliasRollover(conf *cfg.Config) *cli.Command {
UsageText: "rollover <alias>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cindex)
complete(cmd, Cindex)
},
Flags: []cli.Flag{

7
cmd/index_template.go
View File

@@ -19,6 +19,7 @@ package cmd
import (
"context"
"errors"
"fmt"
"slices"
"codeberg.org/scip/esctl/pkg/cfg"
@@ -71,7 +72,7 @@ func IndexTemplateShow(conf *cfg.Config) *cli.Command {
},
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cindex)
complete(cmd, Cindex)
},
}
}
@@ -179,7 +180,7 @@ https://www.elastic.co/docs/reference/elasticsearch/index-settings
Action: func(ctx context.Context, cmd *cli.Command) error {
args := cmd.Args()
if args.Len() == 0 {
return errors.New("no name specified")
return fmt.Errorf("no name specified")
}
if conf.Mode != "" {
@@ -215,7 +216,7 @@ func IndexTemplateDelete(conf *cfg.Config) *cli.Command {
},
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cindex)
complete(cmd, Cindex)
},
}
}

49
cmd/license.go
View File

@@ -1,49 +0,0 @@
/*
Copyright © 2026 Thomas von Dein
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package cmd
import (
"context"
"codeberg.org/scip/esctl/pkg/cfg"
"codeberg.org/scip/esctl/pkg/es"
"github.com/urfave/cli/v3"
)
func License(conf *cfg.Config) *cli.Command {
return &cli.Command{
Name: "license",
Usage: "manage cluster license",
Commands: []*cli.Command{
LicenseShow(conf),
},
}
}
func LicenseShow(conf *cfg.Config) *cli.Command {
return &cli.Command{
Name: "show",
Aliases: []string{"sh"},
Usage: "show details about the cluster license",
Action: func(ctx context.Context, cmd *cli.Command) error {
return es.LicenseShow(conf)
},
}
}

21
cmd/node.go
View File

@@ -36,7 +36,6 @@ func Node(conf *cfg.Config) *cli.Command {
NodeList(conf),
NodeShow(conf),
NodeClients(conf),
NodeUsage(conf),
},
}
}
@@ -61,7 +60,7 @@ func NodeShow(conf *cfg.Config) *cli.Command {
UsageText: "show [options] <node>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cnode)
complete(cmd, Cnode)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
@@ -91,7 +90,7 @@ func NodeClients(conf *cfg.Config) *cli.Command {
},
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cnode)
complete(cmd, Cnode)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
@@ -104,19 +103,3 @@ func NodeClients(conf *cfg.Config) *cli.Command {
},
}
}
func NodeUsage(conf *cfg.Config) *cli.Command {
return &cli.Command{
Name: "usage",
Usage: "show node usage stats",
UsageText: "usage [options] [<node>]",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cnode)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
return es.NodeUsage(conf, cmd.Args().Get(0))
},
}
}

2
cmd/roles.go
View File

@@ -68,7 +68,7 @@ func RoleShow(conf *cfg.Config) *cli.Command {
UsageText: "show [options] <role>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Crole)
complete(cmd, Crole)
},
Action: func(ctx context.Context, cmd *cli.Command) error {

157
cmd/root.go
View File

@@ -24,71 +24,16 @@ import (
"runtime/pprof"
"strings"
"codeberg.org/scip/esctl/assets"
"codeberg.org/scip/esctl/pkg/cfg"
"codeberg.org/scip/esctl/pkg/es"
"codeberg.org/scip/esctl/pkg/log"
"codeberg.org/scip/esctl/pkg/printer"
markdown "github.com/MichaelMure/go-term-markdown"
"github.com/urfave/cli/v3"
)
const (
versionFmt = `esctl version: %s
build: %s
branch: %s
commit: %s
go version: %s
API Version: %s
`
jsonPathUsage = `jsonPath usage:
name.last >> "Anderson"
age >> 37
children >> ["Sara","Alex","Jack"]
children.# >> 3
children.1 >> "Alex"
child*.2 >> "Jack"
c?ildren.0 >> "Sara"
fav\.movie >> "Deer Hunter"
friends.#.first >> ["Dale","Roger","Jane"]
friends.1.last >> "Craig"
You can also query an array for the first match by using #(...), or
find all matches with #(...)#. Queries support the ==, !=, <, <=, >,
>= comparison operators and the simple pattern matching % (like) and
!% (not like) operators. Eg:
friends.#(last=="Murphy").first >> "Dale"
friends.#(last=="Murphy")#.first >> ["Dale","Jane"]
friends.#(age>45)#.last >> ["Craig","Murphy"]
friends.#(first%"D*").last >> "Murphy"
friends.#(first!%"D*").last >> "Craig"
friends.#(nets.#(=="fb"))#.first >> ["Dale","Roger"]
To extract more than one field, use:
{"ns":source.namespace_name,"time":source.@timestamp} >>
{
"ns": "3f80316965c64405-275a75782e984bcc8d82",
"time": "2026-06-10T04:23:38.914080361+00:00"
}
{
"ns": "3f80316965c64405-f0e1dcef668e45bb9afd",
"time": "2026-06-10T09:39:00.554892542+00:00"
}
Documentation: https://github.com/tidwall/gjson/blob/master/SYNTAX.md`
)
func Finish(err error) int {
if err != nil {
fmt.Fprintln(os.Stderr, "Error: ", err.Error())
return 1
}
@@ -144,7 +89,7 @@ func Main() int {
Name: "output",
Aliases: []string{"o"},
Value: "",
Usage: "output mode (tsv, csv, json, yaml) default: tsv",
Usage: "output mode (tsv, markdown, json, yaml) default: tsv",
Destination: &conf.Output,
},
&cli.StringFlag{
@@ -163,11 +108,9 @@ func Main() int {
Doc(conf),
Ilm(conf),
Index(conf),
License(conf),
Node(conf),
Roles(conf),
Search(conf),
SearchQL(conf),
Shard(conf),
Snapshot(conf),
Task(conf),
@@ -175,7 +118,6 @@ func Main() int {
Debug(conf),
HelpJsonPath(conf),
HelpUsage(conf),
HelpEsQl(conf),
},
Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) {
@@ -184,7 +126,6 @@ func Main() int {
return nil, err
} else {
fmt.Println(cmd.UsageText)
return nil, nil
}
}
@@ -219,43 +160,66 @@ func Main() int {
}
func HelpJsonPath(conf *cfg.Config) *cli.Command {
msg := `jsonPath usage:
name.last >> "Anderson"
age >> 37
children >> ["Sara","Alex","Jack"]
children.# >> 3
children.1 >> "Alex"
child*.2 >> "Jack"
c?ildren.0 >> "Sara"
fav\.movie >> "Deer Hunter"
friends.#.first >> ["Dale","Roger","Jane"]
friends.1.last >> "Craig"
You can also query an array for the first match by using #(...), or
find all matches with #(...)#. Queries support the ==, !=, <, <=, >,
>= comparison operators and the simple pattern matching % (like) and
!% (not like) operators. Eg:
friends.#(last=="Murphy").first >> "Dale"
friends.#(last=="Murphy")#.first >> ["Dale","Jane"]
friends.#(age>45)#.last >> ["Craig","Murphy"]
friends.#(first%"D*").last >> "Murphy"
friends.#(first!%"D*").last >> "Craig"
friends.#(nets.#(=="fb"))#.first >> ["Dale","Roger"]
To extract more than one field, use:
{"ns":source.namespace_name,"time":source.@timestamp} >>
{
"ns": "3f80316965c64405-275a75782e984bcc8d82",
"time": "2026-06-10T04:23:38.914080361+00:00"
}
{
"ns": "3f80316965c64405-f0e1dcef668e45bb9afd",
"time": "2026-06-10T09:39:00.554892542+00:00"
}
Documentation: https://github.com/tidwall/gjson/blob/master/SYNTAX.md`
return &cli.Command{
Name: "help-jsonpath",
Usage: "show jsonpath help",
Action: func(ctx context.Context, cmd *cli.Command) error {
_, err := fmt.Println(jsonPathUsage)
_, err := fmt.Println(msg)
return err
},
}
}
func HelpEsQl(conf *cfg.Config) *cli.Command {
return &cli.Command{
Name: "help-esql",
Usage: "show esql help",
Action: func(ctx context.Context, cmd *cli.Command) error {
assets.LoadEsql()
width := cfg.GetTermWidth()
printer.Pager("esql cheat sheet", string(markdown.Render(assets.EsQlCheatSheet, width, cfg.DefaultMargin)))
return nil
},
}
}
func Version(conf *cfg.Config) *cli.Command {
return &cli.Command{
Name: "version",
Usage: "show esctl version information",
Action: func(ctx context.Context, cmd *cli.Command) error {
_, err := fmt.Printf(versionFmt,
_, err := fmt.Printf("esctl version: %s\n build: %s\n branch: %s\n commit: %s\n go version: %s\n API Version: %s\n",
cfg.Version, cfg.BUILD, cfg.BRANCH, cfg.COMMIT, cfg.GOVERSION, cfg.APIVERSION)
return err
},
}
@@ -284,11 +248,9 @@ func Debug(conf *cfg.Config) *cli.Command {
func HelpUsage(conf *cfg.Config) *cli.Command {
return &cli.Command{
Name: "help-usage",
Name: "help-command-overview",
Usage: "show overview of all available commands",
UsageText: "help-usage [<filter>]",
Aliases: []string{"usage"},
CustomHelpTemplate: addReference(`<filter> implies -f`),
Flags: []cli.Flag{
&cli.BoolFlag{
@@ -297,32 +259,16 @@ func HelpUsage(conf *cfg.Config) *cli.Command {
Destination: &conf.Hidden,
Aliases: []string{"H"},
},
&cli.BoolFlag{
Name: "full-commands",
Usage: "show full commands",
Destination: &conf.Force,
Aliases: []string{"f"},
},
},
Action: func(ctx context.Context, cmd *cli.Command) error {
maxCommandWidth := 0
filter := cmd.Args().Get(0)
if filter != "" {
conf.Force = true
}
// first pass, determine max command width
if err := walkVisible(conf, cmd.Root(), func(cmd *cli.Command) error {
path := cmd.Path()
size := len(path[len(path)-1])
if conf.Force {
path := strings.Join(cmd.Path(), " ")
size = len(path)
}
if size > maxCommandWidth {
maxCommandWidth = size
}
@@ -337,19 +283,9 @@ func HelpUsage(conf *cfg.Config) *cli.Command {
// second pass, build tree
return walkVisible(conf, cmd.Root(), func(cmd *cli.Command) error {
path := cmd.Path()
if filter != "" {
if !strings.Contains(strings.Join(path, " "), filter) {
return nil
}
}
command := path[len(path)-1]
if conf.Force {
command = strings.Join(cmd.Path(), " ")
}
if len(path) == 1 || strings.HasSuffix(command, "help") {
if len(path) == 1 || command == "help" {
return nil
}
@@ -362,6 +298,7 @@ func HelpUsage(conf *cfg.Config) *cli.Command {
})
},
}
}
// copy of cmd.Walk() with the exception to skip hidden commands and its siblings

22
cmd/search.go
View File

@@ -18,7 +18,6 @@ package cmd
import (
"context"
"strings"
"codeberg.org/scip/esctl/pkg/cfg"
"codeberg.org/scip/esctl/pkg/es"
@@ -46,11 +45,6 @@ https://www.elastic.co/docs/reference/elasticsearch/rest-apis/common-options#dat
For timestamp formats refer to:
https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/mapping-date-format`
const QlUsage = `ES|QL documentation:
https://www.elastic.co/docs/reference/query-languages/esql/esql-getting-started
See also: esctl help-esql`
func Search(conf *cfg.Config) *cli.Command {
return &cli.Command{
Name: "search",
@@ -155,19 +149,3 @@ func Search(conf *cfg.Config) *cli.Command {
},
}
}
func SearchQL(conf *cfg.Config) *cli.Command {
return &cli.Command{
Name: "searchql",
Aliases: []string{"/"},
Usage: "search using ES/QL language",
UsageText: "search <ES/QL term>",
CustomHelpTemplate: addReference(QlUsage),
Action: func(ctx context.Context, cmd *cli.Command) error {
args := cmd.Args()
return es.SearchQL(conf, strings.Join(args.Slice(), " "))
},
}
}

39
cmd/shards.go
View File

@@ -35,7 +35,6 @@ func Shard(conf *cfg.Config) *cli.Command {
Commands: []*cli.Command{
ShardList(conf),
ShardShow(conf),
ShardAllocation(conf),
},
}
}
@@ -95,41 +94,3 @@ func ShardShow(conf *cfg.Config) *cli.Command {
},
}
}
func ShardAllocation(conf *cfg.Config) *cli.Command {
return &cli.Command{
Name: "allocation",
Aliases: []string{"a"},
Usage: "explain shard allocation",
Flags: []cli.Flag{
&cli.IntFlag{
Name: "shard",
Usage: "shard number to explain for",
Destination: &conf.Shards,
Aliases: []string{"s"},
},
&cli.BoolFlag{
Name: "primary",
Usage: "explain primary allocation (default true)",
Destination: &conf.Primary,
Aliases: []string{"p"},
},
&cli.StringFlag{
Name: "node",
Usage: "explain a shard only if it is currently located on the specified node name or node ID",
Destination: &conf.FromNode,
Aliases: []string{"n"},
},
},
Action: func(ctx context.Context, cmd *cli.Command) error {
index := cmd.Args().Get(0)
if index == "" {
return errors.New("no index specified")
}
return es.ShardAllocation(conf, cmd.Args().Get(0))
},
}
}

3
cmd/utilities.go
View File

@@ -26,10 +26,9 @@ import (
func addReference(ref string) string {
indentedRef := []string{}
for line := range strings.SplitSeq(ref, "\n") {
for _, line := range strings.Split(ref, "\n") {
indentedRef = append(indentedRef, " "+line)
}
return fmt.Sprintf("%s\nREFERENCE:\n%s\n",
cli.SubcommandHelpTemplate,
strings.Join(indentedRef, "\n"))

17
go.mod
View File

@@ -14,13 +14,11 @@
// along with this program. If not, see <http://www.gnu.org/licenses/>.
module codeberg.org/scip/esctl
go 1.26
go 1.25.8
require (
github.com/MichaelMure/go-term-markdown v0.1.4
github.com/alecthomas/repr v0.5.2
github.com/charmbracelet/bubbles v1.0.0
github.com/charmbracelet/bubbletea v1.3.10
github.com/charmbracelet/lipgloss v1.1.0
github.com/chzyer/readline v1.5.1
github.com/dustin/go-humanize v1.0.1
@@ -31,11 +29,10 @@ require (
github.com/go-openapi/spec v0.22.5
github.com/go-openapi/swag/loading v0.26.1
github.com/mattn/go-isatty v0.0.22
github.com/seeruk/go-wordwrap v0.0.0-20191208221741-14ec4aac9550
github.com/olekukonko/tablewriter v1.1.4
github.com/tidwall/gjson v1.19.0
github.com/tlinden/yadu v0.1.3
github.com/urfave/cli/v3 v3.10.1-0.20260623012112-f980ca84bf65
golang.org/x/term v0.44.0
gopkg.in/yaml.v3 v3.0.1
)
@@ -43,6 +40,9 @@ require (
github.com/MichaelMure/go-term-text v0.3.1 // indirect
github.com/alecthomas/chroma v0.7.1 // indirect
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/charmbracelet/bubbles v1.0.0 // indirect
github.com/charmbracelet/bubbletea v1.3.10 // indirect
github.com/charmbracelet/colorprofile v0.4.1 // indirect
github.com/charmbracelet/x/ansi v0.11.6 // indirect
github.com/charmbracelet/x/cellbuf v0.0.15 // indirect
@@ -69,6 +69,7 @@ require (
github.com/go-openapi/swag/typeutils v0.26.1 // indirect
github.com/go-openapi/swag/yamlutils v0.26.1 // indirect
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
github.com/goccy/go-json v0.10.6 // indirect
github.com/gomarkdown/markdown v0.0.0-20191123064959-2c17d62f5098 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/kyokomi/emoji/v2 v2.2.8 // indirect
@@ -80,7 +81,11 @@ require (
github.com/muesli/cancelreader v0.2.2 // indirect
github.com/muesli/termenv v0.16.0 // indirect
github.com/oklog/ulid/v2 v2.1.1 // indirect
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect
github.com/olekukonko/errors v1.2.0 // indirect
github.com/olekukonko/ll v0.1.8 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/seeruk/go-wordwrap v0.0.0-20191208221741-14ec4aac9550 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
@@ -89,8 +94,10 @@ require (
go.opentelemetry.io/otel/metric v1.35.0 // indirect
go.opentelemetry.io/otel/trace v1.35.0 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
golang.org/x/image v0.0.0-20191206065243-da761ea9ff43 // indirect
golang.org/x/net v0.55.0 // indirect
golang.org/x/sys v0.46.0 // indirect
golang.org/x/term v0.44.0 // indirect
golang.org/x/text v0.37.0 // indirect
)

24
go.sum
View File

@@ -14,18 +14,28 @@ github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs
github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/charmbracelet/bubbles v1.0.0 h1:12J8/ak/uCZEMQ6KU7pcfwceyjLlWsDLAxB5fXonfvc=
github.com/charmbracelet/bubbles v1.0.0/go.mod h1:9d/Zd5GdnauMI5ivUIVisuEm3ave1XwXtD1ckyV6r3E=
github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw=
github.com/charmbracelet/bubbletea v1.3.10/go.mod h1:ORQfo0fk8U+po9VaNvnV95UPWA1BitP1E0N6xJPlHr4=
github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc h1:4pZI35227imm7yK2bGPcfpFEmuY1gc2YSTShr4iJBfs=
github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc/go.mod h1:X4/0JoqgTIPSFcRA/P6INZzIuyqdFY5rm8tb41s9okk=
github.com/charmbracelet/colorprofile v0.4.1 h1:a1lO03qTrSIRaK8c3JRxJDZOvhvIeSco3ej+ngLk1kk=
github.com/charmbracelet/colorprofile v0.4.1/go.mod h1:U1d9Dljmdf9DLegaJ0nGZNJvoXAhayhmidOdcBwAvKk=
github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY=
github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30=
github.com/charmbracelet/x/ansi v0.8.0 h1:9GTq3xq9caJW8ZrBTe0LIe2fvfLR/bYXKTx2llXn7xE=
github.com/charmbracelet/x/ansi v0.8.0/go.mod h1:wdYl/ONOLHLIVmQaxbIYEC/cRKOQyjTkowiI4blgS9Q=
github.com/charmbracelet/x/ansi v0.11.6 h1:GhV21SiDz/45W9AnV2R61xZMRri5NlLnl6CVF7ihZW8=
github.com/charmbracelet/x/ansi v0.11.6/go.mod h1:2JNYLgQUsyqaiLovhU2Rv/pb8r6ydXKS3NIttu3VGZQ=
github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd h1:vy0GVL4jeHEwG5YOXDmi86oYw2yuYUGqz6a8sLwg0X8=
github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd/go.mod h1:xe0nKWGd3eJgtqZRaN9RjMtK7xUYchjzPr7q6kcvCCs=
github.com/charmbracelet/x/cellbuf v0.0.15 h1:ur3pZy0o6z/R7EylET877CBxaiE1Sp1GMxoFPAIztPI=
github.com/charmbracelet/x/cellbuf v0.0.15/go.mod h1:J1YVbR7MUuEGIFPCaaZ96KDl5NoS0DAWkskup+mOY+Q=
github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ=
github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg=
github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk=
github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI=
github.com/chzyer/logex v1.2.1 h1:XHDu3E6q+gdHgsdTPH6ImJMIp436vR6MPtH8gP05QzM=
@@ -104,6 +114,8 @@ github.com/go-openapi/testify/v2 v2.5.1 h1:TMdhCaw8fUNraVSf3Omoob1dO/AzBfhtFAPW0
github.com/go-openapi/testify/v2 v2.5.1/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw=
github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU=
github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
github.com/gomarkdown/markdown v0.0.0-20191123064959-2c17d62f5098 h1:Qxs3bNRWe8GTcKMxYOSXm0jx6j0de8XUtb/fsP3GZ0I=
github.com/gomarkdown/markdown v0.0.0-20191123064959-2c17d62f5098/go.mod h1:aii0r/K0ZnHv7G0KF7xy1v0A7s2Ljrb5byB7MO5p6TU=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
@@ -142,6 +154,14 @@ github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc
github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
github.com/oklog/ulid/v2 v2.1.1 h1:suPZ4ARWLOJLegGFiZZ1dFAkqzhMjL3J1TzI+5wHz8s=
github.com/oklog/ulid/v2 v2.1.1/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ=
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 h1:zrbMGy9YXpIeTnGj4EljqMiZsIcE09mmF8XsD5AYOJc=
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6/go.mod h1:rEKTHC9roVVicUIfZK7DYrdIoM0EOr8mK1Hj5s3JjH0=
github.com/olekukonko/errors v1.2.0 h1:10Zcn4GeV59t/EGqJc8fUjtFT/FuUh5bTMzZ1XwmCRo=
github.com/olekukonko/errors v1.2.0/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y=
github.com/olekukonko/ll v0.1.8 h1:ysHCJRGHYKzmBSdz9w5AySztx7lG8SQY+naTGYUbsz8=
github.com/olekukonko/ll v0.1.8/go.mod h1:RPRC6UcscfFZgjo1nulkfMH5IM0QAYim0LfnMvUuozw=
github.com/olekukonko/tablewriter v1.1.4 h1:ORUMI3dXbMnRlRggJX3+q7OzQFDdvgbN9nVWj1drm6I=
github.com/olekukonko/tablewriter v1.1.4/go.mod h1:+kedxuyTtgoZLwif3P1Em4hARJs+mVnzKxmsCL/C5RY=
github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -169,6 +189,8 @@ github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/tlinden/yadu v0.1.3 h1:5cRCUmj+l5yvlM2irtpFBIJwVV2DPEgYSaWvF19FtcY=
github.com/tlinden/yadu v0.1.3/go.mod h1:l3bRmHKL9zGAR6pnBHY2HRPxBecf7L74BoBgOOpTcUA=
github.com/urfave/cli/v3 v3.9.1-0.20260524212652-be8b79d0c8de h1:ESKPiS7inVoBnv4FmgGNZdjWBI/wmvaragoyD3D9nM4=
github.com/urfave/cli/v3 v3.9.1-0.20260524212652-be8b79d0c8de/go.mod h1:ysVLtOEmg2tOy6PknnYVhDoouyC/6N42TMeoMzskhso=
github.com/urfave/cli/v3 v3.10.1-0.20260623012112-f980ca84bf65 h1:NXitXXO9DupDLEFj8hIyYtsubRxSFTp5JXqohElT0nU=
github.com/urfave/cli/v3 v3.10.1-0.20260623012112-f980ca84bf65/go.mod h1:ysVLtOEmg2tOy6PknnYVhDoouyC/6N42TMeoMzskhso=
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
@@ -201,6 +223,8 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=

174
pkg/cfg/automate.go Normal file
View File

@@ -0,0 +1,174 @@
/*
Copyright © 2026 Thomas von Dein
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package cfg
import (
"bytes"
"context"
"errors"
"fmt"
"log/slog"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"gopkg.in/yaml.v3"
)
type Automator struct {
IsReachable bool
Error error
Env []string
Cluster *Cluster
// all bash commands, .name must succeed first
Name string `yaml:"name"`
User string `yaml:"user"`
Pass string `yaml:"pass"`
Uri string `yaml:"uri"`
}
type AutomatorRunner struct {
Output string
Error error
}
func NewAutoEnv() []string {
return []string{
"PATH=" + os.Getenv("PATH"),
"HOME=" + os.Getenv("HOME"),
"SHELL=" + os.Getenv("SHELL"),
"KUBECONFIG=" + os.Getenv("KUBECONFIG"),
}
}
func NewAutomator() Automator {
autocfg := filepath.Join([]string{os.Getenv("HOME"), ".config", "esctl", "automate.yaml"}...)
auto := Automator{Env: NewAutoEnv()}
if !fileExists(autocfg) {
return auto
}
data, err := os.ReadFile(autocfg)
if err != nil {
auto.Error = fmt.Errorf("failed to read config file: %w", err)
return auto
}
err = yaml.Unmarshal(data, &auto)
if err != nil {
auto.Error = fmt.Errorf("failed to unmarshal config file: %w", err)
return auto
}
hasname := execute(auto.Name, auto.Env)
if hasname.Error != nil {
auto.Error = hasname.Error
return auto
}
auto.Name = hasname.Output
hasuser := execute(auto.User, auto.Env)
if hasuser.Error != nil {
auto.Error = hasuser.Error
return auto
}
auto.User = hasuser.Output
haspass := execute(auto.Pass, auto.Env)
if haspass.Error != nil {
auto.Error = haspass.Error
return auto
}
auto.Pass = haspass.Output
hasuri := execute(auto.Uri, auto.Env)
if hasuri.Error != nil {
auto.Error = hasuri.Error
return auto
}
auto.Uri = hasuri.Output
auto.IsReachable = true
auto.Cluster = &Cluster{
Name: auto.Name,
Uri: auto.Uri,
User: auto.User,
Pass: auto.Pass,
}
return auto
}
// FIXME: execute auto.Exec, if defined, in a go routine and let it run forever, might be a tunnel
func (auto *Automator) Exec() {}
// FIXME: cache automator results, only check auto.Name and if it matches the cache use those vars, but run auto.Exec anyway
func (auto *Automator) Cache() {}
func execute(code string, env []string) *AutomatorRunner {
timeoutCtx, cancel := context.WithTimeout(context.Background(),
time.Duration(10)*time.Second)
defer cancel()
var cmd *exec.Cmd
// pipe code into bash
cmd = exec.CommandContext(timeoutCtx, "bash")
cmd.Stdin = strings.NewReader(code)
cmd.Env = env
errbuf := &bytes.Buffer{}
cmd.Stderr = errbuf
done := make(chan bool)
out := AutomatorRunner{}
go func() {
output, err := cmd.Output()
out.Output = strings.TrimSpace(string(output))
switch {
case err != nil:
out.Error = err
case errbuf.Len() > 0:
out.Error = fmt.Errorf(errbuf.String())
case timeoutCtx.Err() == context.DeadlineExceeded:
out.Error = errors.New("timed out")
}
slog.Debug("executed automator",
"code", code,
"output", out.Output,
"error", out.Error)
done <- true
}()
for {
select {
case <-done:
return &out
}
}
}

117
pkg/cfg/cluster.go
View File

@@ -17,10 +17,10 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package cfg
import (
"context"
"crypto/tls"
"errors"
"fmt"
"net"
"net/http"
"os"
"strings"
@@ -55,6 +55,37 @@ func (cluster *Cluster) SetClient(client *elasticsearch.TypedClient) {
cluster.client = client
}
func (cluster *Cluster) getTransport() elastictransport.Option {
transport := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
if cluster.DebugHTTP {
return elastictransport.WithTransport(
&DebugTransport{Transport: transport},
)
}
return elastictransport.WithTransport(transport)
}
func (cluster *Cluster) getDefaultOptions() []elasticsearch.Option {
// These headers are not needed with ES 9, but with ES 8, we set
// them here so every API call uses it. The only exception being
// the api repl, which does it on its own.
headers := http.Header{}
headers.Add("content-type", "application/json")
headers.Add("Accept", "application/json")
return []elasticsearch.Option{
elasticsearch.WithAddresses(cluster.Uri),
elasticsearch.WithTransportOptions(
cluster.getTransport(),
elastictransport.WithHeader(headers),
),
}
}
// return the go-elasticsearch client object but before doing that,
// check if we need to tune auth
func (cluster *Cluster) ES() *elasticsearch.TypedClient {
@@ -82,9 +113,8 @@ func (cluster *Cluster) CheckAuth() error {
cluster.Pass = pass
} else {
// k, try interactively
fmt.Fprintf(os.Stderr, "Enter password for elasticsearch user %s@%s: ", cluster.User, cluster.Name)
pass, err := term.ReadPassword(syscall.Stdin)
fmt.Printf("Enter password for elasticsearch user %s@%s: ", cluster.User, cluster.Name)
pass, err := term.ReadPassword(int(syscall.Stdin))
if err != nil {
return err
}
@@ -95,7 +125,6 @@ func (cluster *Cluster) CheckAuth() error {
}
cluster.Pass = string(pass)
fmt.Println()
}
@@ -103,6 +132,7 @@ func (cluster *Cluster) CheckAuth() error {
opts = append(opts, elasticsearch.WithBasicAuth(cluster.User, cluster.Pass))
es, err := elasticsearch.NewTyped(opts...)
if err != nil {
return fmt.Errorf("failed to setup elasticsearch connection: %w", err)
}
@@ -155,62 +185,45 @@ func (conf *Config) SwitchCluster(name string) error {
// We do NOT use go-elasticsearch to check for cluster reachability,
// because at this stage, auth may not have been configured. So
// instead we just connect to the cluster using plan net/tcp
// instead we just connect to the cluster using plan net/http, ignore
// HTTP response status and return true if we could just reach ith
func (cluster *Cluster) IsReachable() (bool, error) {
timeout := 500 * time.Millisecond
ctx, cancel := context.WithTimeout(
context.Background(),
time.Duration(500)*time.Millisecond)
defer cancel()
url := strings.TrimPrefix(strings.TrimPrefix(cluster.Uri, "https://"), "http://")
req, err := http.NewRequestWithContext(
ctx,
"GET",
cluster.Uri,
nil,
)
host := strings.Split(url, "/")
if !strings.Contains(host[0], ":") {
host[0] += ":443"
}
conn, err := net.DialTimeout("tcp", host[0], timeout)
if err != nil {
return false, err
}
return true, conn.Close()
}
func (cluster *Cluster) getDefaultOptions() []elasticsearch.Option {
// These headers are not needed with ES 9, but with ES 8, we set
// them here so every API call uses it. The only exception being
// the api repl, which does it on its own.
headers := http.Header{}
headers.Add("Content-Type", "application/json")
headers.Add("Accept", "application/json")
return []elasticsearch.Option{
elasticsearch.WithAddresses(cluster.Uri),
elasticsearch.WithTransportOptions(
cluster.getTransport(),
elastictransport.WithHeader(headers),
),
elasticsearch.WithCompatibilityMode(),
}
}
func (cluster *Cluster) getTransport() elastictransport.Option {
transport := &http.Transport{
client := &http.Client{Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}}
resp, err := client.Do(req)
if err != nil {
return false, err
}
if cluster.DebugHTTP {
return elastictransport.WithTransport(
&DebugTransport{Transport: transport},
)
if resp != nil {
// at this stage we do not care if the elasticsearch cluster
// accepts our request or if it's misconfigured in some way
return true, nil
}
return elastictransport.WithTransport(
&CompatibilityTransport{Transport: transport},
)
return false, nil
}
func (conf *Config) SetupES() error {
for name, cluster := range conf.Clusters {
func (conf *Config) SetupElasticClient(name string, cluster *Cluster) error {
cluster.Name = name
cluster.DebugHTTP = conf.DebugHTTP
@@ -224,11 +237,21 @@ func (conf *Config) SetupES() error {
}
es, err := elasticsearch.NewTyped(opts...)
if err != nil {
return fmt.Errorf("failed to setup elasticsearch connection: %w", err)
}
cluster.SetClient(es)
return nil
}
func (conf *Config) SetupElasticClients() error {
for name, cluster := range conf.Clusters {
if err := conf.SetupElasticClient(name, cluster); err != nil {
return err
}
}
return nil

110
pkg/cfg/config.go
View File

@@ -28,7 +28,7 @@ import (
)
const (
Version string = `v0.0.26`
Version string = `v0.0.25`
)
var (
@@ -82,9 +82,8 @@ type Config struct {
SortBy string // sort: -k
Ascending bool // sort: -a
All bool // doc + node ls: -a
Exclude string // cluster compare: -e (regexp)
Verbose bool // cluster status: -v
All, Verbose bool // cluster status: -a -v
Persistent, Transient, Default bool // cluster settings set: -p -t -D
Force bool // ccr follower renew: -f
@@ -102,14 +101,11 @@ type Config struct {
DryRun bool // rollover: -n
Tag string // api ls: -t
HumanCat bool // api repl: -H
Ilm Ilm // ilm create
FromNode, ToNode string // cluster reroute move: -f + -t
AllowPrimary, AcceptDataLoss bool // cluster reroute cancel: -p,-a
Pager string // api repl: -p || PAGER
}
func NewConfig() *Config {
@@ -123,11 +119,10 @@ func getDefaultPath() string {
func (conf *Config) Init() error {
DefaultConfig := getDefaultPath()
if conf.ConfigFile == "" && fileExists(DefaultConfig) {
conf.ConfigFile = DefaultConfig
}
switch {
case fileExists(DefaultConfig):
conf.ConfigFile = DefaultConfig
fallthrough
case conf.ConfigFile != "":
if err := conf.LoadConfig(); err != nil {
return err
@@ -138,14 +133,56 @@ func (conf *Config) Init() error {
}
}
if err := conf.SetupES(); err != nil {
if err := conf.SetupElasticClients(); err != nil {
return err
}
if err := conf.determineDefaultCluster(); err != nil {
if conf.CurrentCluster != "" {
// -C specified, set current cluster explicitly, no matter what the config says
current, exists := conf.Clusters[conf.CurrentCluster]
if !exists {
return fmt.Errorf("no cluster with alias %s configured", conf.CurrentCluster)
} else {
conf.DefaultCluster = current
// disable all others
for _, cluster := range conf.Clusters {
cluster.Default = false
}
conf.DefaultCluster.Default = true
}
} else {
// load auto config, if any
auto := NewAutomator()
// we need to determine ourselfes
if len(conf.Clusters) == 1 {
// ok, just one cluster configured, use this, of course
for name, cluster := range conf.Clusters {
conf.DefaultCluster = cluster
conf.CurrentCluster = name
conf.DefaultCluster.Default = true
}
} else if auto.Error == nil && auto.IsReachable {
// use auto conf
if err := conf.SetupElasticClient(auto.Name, auto.Cluster); err != nil {
return err
}
conf.DefaultCluster = auto.Cluster
conf.CurrentCluster = auto.Name
} else {
// multiple ones exists, look if one is set as default
for name, cluster := range conf.Clusters {
if cluster.Default {
conf.DefaultCluster = cluster
conf.CurrentCluster = name
}
}
}
}
conf.HaveJQ = isJQinstalled()
conf.PrintDebug()
@@ -161,14 +198,14 @@ func (conf *Config) Clone() Config {
ref := reflect.ValueOf(*conf)
typeOfS := ref.Type()
for idx := range ref.NumField() {
field := typeOfS.Field(idx).Name
for i := 0; i < ref.NumField(); i++ {
field := typeOfS.Field(i).Name
if field == "Clusters" || field == "DefaultCluster" || !ref.Field(idx).CanInterface() {
if field == "Clusters" || field == "DefaultCluster" || !ref.Field(i).CanInterface() {
continue
}
reflect.ValueOf(&clone).Elem().FieldByName(field).Set(reflect.ValueOf(ref.Field(idx).Interface()))
reflect.ValueOf(&clone).Elem().FieldByName(field).Set(reflect.ValueOf(ref.Field(i).Interface()))
}
return clone
@@ -229,7 +266,6 @@ func (conf *Config) LoadConfig() error {
for _, cluster := range conf.Clusters {
if cluster.Default {
conf.DefaultCluster = cluster
break
}
}
@@ -242,47 +278,9 @@ func (conf *Config) LoadConfig() error {
return nil
}
func (conf *Config) determineDefaultCluster() error {
if conf.CurrentCluster != "" {
// -C specified, set current cluster explicitly, no matter what the config says
current, exists := conf.Clusters[conf.CurrentCluster]
if !exists {
return fmt.Errorf("no cluster with alias %s configured", conf.CurrentCluster)
} else {
conf.DefaultCluster = current
// disable all others
for _, cluster := range conf.Clusters {
cluster.Default = false
}
conf.DefaultCluster.Default = true
}
} else {
// we need to determine ourselfes
if len(conf.Clusters) == 1 {
// ok, just one cluster configured, use this, of course
for name, cluster := range conf.Clusters {
conf.DefaultCluster = cluster
conf.CurrentCluster = name
conf.DefaultCluster.Default = true
}
} else {
// multiple ones exists, look if one is set as default
for name, cluster := range conf.Clusters {
if cluster.Default {
conf.DefaultCluster = cluster
conf.CurrentCluster = name
}
}
}
}
return nil
}
func fileExists(filename string) bool {
info, err := os.Stat(filename)
if err != nil {
// return false on any error
return false

1
pkg/cfg/jq.go
View File

@@ -25,7 +25,6 @@ import (
func isJQinstalled() bool {
cmd := exec.CommandContext(context.Background(), "jq", "-h")
out, err := cmd.Output()
if err != nil {
return false

11
pkg/cfg/term.go
View File

@@ -36,14 +36,3 @@ func GetTermWidth() int {
return 80
}
func GetTermHeight() int {
if term.IsTerminal(int(os.Stdout.Fd())) {
_, height, err := term.GetSize(int(os.Stdout.Fd()))
if err == nil {
return height
}
}
return 25
}

34
pkg/cfg/transport.go
View File

@@ -43,18 +43,15 @@ func (t *DebugTransport) RoundTrip(req *http.Request) (*http.Response, error) {
}
var pretty bytes.Buffer
err = json.Indent(&pretty, buf.Bytes(), "", "\t")
if err != nil {
return nil, fmt.Errorf("json parse error: %w", err)
return nil, fmt.Errorf("json parse error: %s", err)
}
content = pretty.String()
contentline = buf.String()
}
req = fixRequestHeaders(req)
slog.Info("req", "host", req.URL.Host, "uri", req.URL.Path,
"body", content, "bodyline", contentline,
"headers", req.Header,
@@ -62,32 +59,3 @@ func (t *DebugTransport) RoundTrip(req *http.Request) (*http.Response, error) {
return t.Transport.RoundTrip(req)
}
type CompatibilityTransport struct {
Transport http.RoundTripper
}
func (t *CompatibilityTransport) RoundTrip(req *http.Request) (*http.Response, error) {
req = fixRequestHeaders(req)
return t.Transport.RoundTrip(req)
}
// Fixes https://codeberg.org/scip/esctl/issues/79:
// the API client has this Accept header hardcoded everywhere:
// req.Header.Set("Accept", "application/vnd.elasticsearch+json;compatible-with=9")
// While this works pretty well with ES9, it doesn't with ES8. So, we replace
// this header with a new one without the compatibility part.
func fixRequestHeaders(req *http.Request) *http.Request {
if req.Header.Get("Accept") != "" {
req.Header.Del("Accept")
req.Header.Add("Accept", "application/json")
}
if req.Header.Get("Content-Type") != "" {
req.Header.Del("Content-Type")
req.Header.Add("Content-Type", "application/json")
}
return req
}

174
pkg/es/api.go
View File

@@ -26,7 +26,6 @@ import (
"errors"
"fmt"
"io"
"log"
"log/slog"
"net/http"
"os"
@@ -45,25 +44,22 @@ import (
)
const (
intro = `# Input format: verb path [data]"
#
# Example:
#
# post /yourindex/_ccr/pause_follow
# put /yourindex/_settings {"number_of_replicas": 1}
#
# You can also put multiline JSON after the path like:
#
# put /yourindex/_settings
# {
# "number_of_replicas": 1
# }
#
# If you do NOT supply a JSON in the first line, you need to hit ENTER
# twice to complete.
#
# Supply the flag --human-readable-cat, -H to view /_cat API calls in
# human readable form.`
intro = `Input format: verb path [data]"
Example:
post /yourindex/_ccr/pause_follow
put /yourindex/_settings {"number_of_replicas": 1}
You can also put multiline JSON after the path like:
put /yourindex/_settings
{
"number_of_replicas": 1
}
If you do NOT supply a JSON in the first line, you need to hit ENTER
twice to complete.`
)
// holds an API operation via go-openapi/spec
@@ -96,8 +92,9 @@ func ApiRepl(conf *cfg.Config) error {
EOFPrompt: "exit",
HistorySearchFold: true,
})
if err != nil {
return fmt.Errorf("failed to initialize readline lib: %w", err)
return fmt.Errorf("failed to initialize readline lib: %s", err)
}
for {
@@ -115,13 +112,11 @@ func ApiRepl(conf *cfg.Config) error {
parts := strings.SplitN(strings.TrimSpace(text), " ", 3)
if len(parts) < 2 {
fmt.Println("error: you need to input a verb, uri [and post data]")
continue
}
if !slices.Contains(verbs, strings.ToLower(parts[0])) {
fmt.Println("error: verb must be one of " + strings.Join(verbs, ","))
continue
}
@@ -146,56 +141,12 @@ func ApiRepl(conf *cfg.Config) error {
fmt.Printf("failed to call API: %s\n", esErrorString(err))
}
if conf.HumanCat && strings.HasPrefix(parts[1], "/_cat") {
fmt.Println(string(raw))
} else {
pageJsonOutput(conf, raw)
}
}
//nolint:nilerr
return nil
}
func pageJsonOutput(conf *cfg.Config, raw []byte) {
tmpconf := &cfg.Config{HaveJQ: conf.HaveJQ}
if conf.Pager != "" {
tmpconf.HaveJQ = false
}
output, err := prettyfiJson(tmpconf, raw)
if err != nil {
if err := prettyfiJson(conf, raw); err != nil {
fmt.Println(err)
}
lines := len(strings.Split(output, "\n"))
height := cfg.GetTermHeight()
if lines > height {
if conf.Pager != "" {
cmd := strings.Split(conf.Pager, " ")
pager := exec.Command(cmd[0], cmd[1:]...)
var buf bytes.Buffer
buf.WriteString(output)
pager.Stdout = os.Stdout
pager.Stdin = &buf
pager.Stderr = os.Stderr
err := pager.Run()
if err != nil {
fmt.Printf("failed to execute pager '%s': %s", conf.Pager, err)
}
} else {
printer.Pager("json output", output)
}
return
}
fmt.Println(output)
return nil
}
func encodeAuth(username, password string) string {
@@ -206,27 +157,19 @@ func CallAPI(conf *cfg.Config, verb, path, data string) ([]byte, error) {
verb = strings.ToUpper(verb)
// we're using port-forwards anyway
noVerifyTransport := &http.Transport{
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
client := &http.Client{Transport: noVerifyTransport}
if conf.DebugHTTP {
client = &http.Client{
Transport: &cfg.DebugTransport{
Transport: noVerifyTransport}}
}
client := &http.Client{Transport: tr}
req, err := http.NewRequest(verb, conf.DefaultCluster.Uri+path, bytes.NewBuffer([]byte(data)))
if err != nil {
return nil, err
}
if !conf.HumanCat || (conf.HumanCat && !strings.HasPrefix(path, "/_cat")) {
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Accept", "application/json")
}
req.Header.Add("accept", "application/json")
// make sure we have got all we need
if err := conf.DefaultCluster.CheckAuth(); err != nil {
@@ -244,56 +187,52 @@ func CallAPI(conf *cfg.Config, verb, path, data string) ([]byte, error) {
if err != nil {
return nil, err
}
defer func() {
if err := resp.Body.Close(); err != nil {
log.Fatal(err)
}
}()
// Read and print response
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read response body: %w", err)
return nil, fmt.Errorf("failed to read response body: %s", err)
}
return body, nil
}
func prettyfiJson(conf *cfg.Config, raw []byte) (string, error) {
func prettyfiJson(conf *cfg.Config, raw []byte) error {
if conf.HaveJQ {
cmd := exec.CommandContext(context.Background(), "jq", "-C")
cmd.Stdin = bytes.NewReader(raw)
var out bytes.Buffer
cmd.Stdout = &out
err := cmd.Run()
if err != nil {
return "", err
return err
}
return out.String(), nil
fmt.Println(out.String())
return nil
}
var pretty bytes.Buffer
err := json.Indent(&pretty, raw, "", "\t")
if err != nil {
//nolint:nilerr
return string(raw), nil
return fmt.Errorf("json parse error: %s", err)
}
return pretty.String(), nil
fmt.Println(pretty.String())
return nil
}
// interactively read arbitrary JSON data from STDIN, which is
// virtually a repl inside the primary repl
func readJSON(input string) (string, error) {
data := strings.Builder{}
data := ""
if input != "" {
data.WriteString(input)
data = input
} else {
scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() {
@@ -303,23 +242,23 @@ func readJSON(input string) (string, error) {
break
}
data.WriteString(line)
data += line
}
}
if data.Len() == 0 {
return "", nil
if data == "" {
return data, nil
}
// validate
check := map[string]any{}
err := json.Unmarshal([]byte(data.String()), &check)
err := json.Unmarshal([]byte(data), &check)
if err != nil {
return "", fmt.Errorf("error: input data is not proper JSON: %w", err)
return "", fmt.Errorf("error: input data is not proper JSON: %s", err)
}
return data.String(), nil
return data, nil
}
func ApiList(conf *cfg.Config, pattern string) error {
@@ -358,7 +297,6 @@ func ApiList(conf *cfg.Config, pattern string) error {
}
table.Sort()
return table.Print()
}
@@ -366,13 +304,10 @@ func ApiList(conf *cfg.Config, pattern string) error {
func ApiPathNames() []string {
assets.LoadAssetOpenApi()
paths := make([]string, len(assets.OpenAPI.Spec().Paths.Paths))
idx := 0
paths := []string{}
for path := range assets.OpenAPI.Spec().Paths.Paths {
paths[idx] = path
idx++
paths = append(paths, path)
}
return paths
@@ -380,7 +315,6 @@ func ApiPathNames() []string {
func ApiShow(conf *cfg.Config, showpath, verb string) error {
assets.LoadAssetOpenApi()
out := printer.Builder{}
op, err := matchOperation(showpath, verb)
@@ -397,20 +331,18 @@ func ApiShow(conf *cfg.Config, showpath, verb string) error {
sample := getApiExample(op)
description := markdown.Render(cleanMarkup.ReplaceAllString(op.Op.Description, ""), width, cfg.DefaultMargin)
var (
bold = lipgloss.NewStyle().
var bold = lipgloss.NewStyle().
Bold(true)
paragraph = lipgloss.NewStyle().
var paragraph = lipgloss.NewStyle().
MarginBottom(1).
MarginLeft(cfg.DefaultMargin)
boldparagraph = lipgloss.NewStyle().
var boldparagraph = lipgloss.NewStyle().
MarginBottom(1).
MarginLeft(cfg.DefaultMargin).
Bold(true)
indentparagraph = lipgloss.NewStyle().
var indentparagraph = lipgloss.NewStyle().
MarginBottom(1).
MarginLeft(2)
)
out.WriteStringLine(bold.Render("ID: " + op.Op.ID))
out.WriteStringLine(paragraph.Render(fmt.Sprintf("%s %s", op.Verb, showpath)))
@@ -426,11 +358,10 @@ func ApiShow(conf *cfg.Config, showpath, verb string) error {
if len(params.Path) > 0 {
out.WriteStringLine(bold.Render("Path Parameters"))
for _, param := range params.Path {
out.WriteStringLine(boldparagraph.Render(param.Param))
if param.Description != "" {
//out.WriteStringLine(indentparagraph.Render(param.Description))
out.WriteStringLine(param.Description)
}
}
@@ -438,10 +369,8 @@ func ApiShow(conf *cfg.Config, showpath, verb string) error {
if len(params.Query) > 0 {
out.WriteStringLine(bold.Render("Query Parameters"))
for _, param := range params.Query {
out.WriteStringLine(boldparagraph.Render(param.Param))
if param.Description != "" {
out.WriteStringLine(indentparagraph.Render(param.Description))
}
@@ -523,8 +452,10 @@ func getApiExample(op *Op) string {
if hassource {
sample = strings.TrimSpace(source.(string))
}
}
}
}
}
@@ -537,13 +468,11 @@ func getApiExample(op *Op) string {
func matchOperation(showpath, verb string) (*Op, error) {
ops := []*Op{}
op := &Op{}
var found bool
for path, item := range assets.OpenAPI.Spec().Paths.Paths {
if path == showpath {
ops = findOperation(item.PathItemProps)
break
}
}
@@ -560,7 +489,6 @@ func matchOperation(showpath, verb string) (*Op, error) {
if strings.ToLower(item.Verb) == verb {
op = item
found = true
break
}
}

19
pkg/es/ccr.go
View File

@@ -46,8 +46,9 @@ func CcrStatus(conf *cfg.Config, leader, follower string) error {
}
// reverse attribution
follower, leader = leader, follower
f := follower
follower = leader
leader = f
slog.Debug("leader/follower attribution is invalid, reversing", "leader", leader, "follower", follower)
}
@@ -57,7 +58,7 @@ func CcrStatus(conf *cfg.Config, leader, follower string) error {
res, err := conf.Clusters[alias].ES().Cat.Indices().
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get indicies on %s: %w", alias, esErrorString(err))
return fmt.Errorf("failed to get indicies on %s: %s", alias, esErrorString(err))
}
indices[alias] = make(map[string]*types.IndicesRecord, len(res))
@@ -86,24 +87,22 @@ func CcrRemoteInfo(conf *cfg.Config, index string) error {
res, err := conf.DefaultCluster.ES().Cluster.RemoteInfo().
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to retrieve follower info: %w", esErrorString(err))
return fmt.Errorf("failed to retrieve follower info: %s", esErrorString(err))
}
slog.Debug("ccr remote info", "info", res)
remote := ""
var info *types.ClusterRemoteProxyInfo
for name, data := range res {
remote = name
info = data.(*types.ClusterRemoteProxyInfo)
break
}
if remote == "" {
return errors.New("cluster doesn't follow any other")
return fmt.Errorf("cluster doesn't follow any other: %s", err)
}
mode := "follower"
@@ -114,11 +113,11 @@ func CcrRemoteInfo(conf *cfg.Config, index string) error {
table := printer.NewTable(conf, 2, 5)
table.Addheaders("ccr remote property", "value")
table.Entries = [][]any{
table.Entries = [][]string{
{"Remote Cluster", remote},
{"CCR Mode", mode},
{"Connected", info.Connected},
{"Num Proxy Sockets Connected", info.NumProxySocketsConnected},
{"Connected", fmt.Sprintf("%t", info.Connected)},
{"Num Proxy Sockets Connected", fmt.Sprintf("%d", info.NumProxySocketsConnected)},
{"Proxy Address", info.ProxyAddress},
}

41
pkg/es/ccr_follower.go
View File

@@ -18,7 +18,6 @@ package es
import (
"context"
"errors"
"fmt"
"log/slog"
@@ -30,18 +29,17 @@ func getRemoteName(conf *cfg.Config) (string, error) {
res, err := conf.DefaultCluster.ES().Cluster.RemoteInfo().
Do(context.Background())
if err != nil {
return "", fmt.Errorf("failed to retrieve follower info: %w", esErrorString(err))
return "", fmt.Errorf("failed to retrieve follower info: %s", esErrorString(err))
}
remote := ""
for name := range res {
remote = name
break
}
if remote == "" {
return "", errors.New("cluster doesn't have a follower")
return "", fmt.Errorf("cluster doesn't have a follower")
}
return remote, nil
@@ -63,31 +61,26 @@ func CcrFollowerRenew(conf *cfg.Config, index string) error {
if err := wrapError(IndexClose, conf, index); err != nil {
return err
}
fmt.Printf("closed %s", index)
if err := wrapError(CcrFollowerPause, conf, index); err != nil {
return err
}
fmt.Printf("paused %s", index)
if err := wrapError(CcrFollowerUnfollow, conf, index); err != nil {
return err
}
fmt.Printf("unfollowed %s", index)
if err := wrapError(IndexDelete, conf, index); err != nil {
return err
}
fmt.Printf("deleted %s", index)
if err := CcrFollowerAdd(conf, index); err != nil {
return err
}
fmt.Printf("added follower %s", index)
return nil
@@ -96,8 +89,9 @@ func CcrFollowerRenew(conf *cfg.Config, index string) error {
func CcrFollowerResume(conf *cfg.Config, index string) error {
_, err := conf.DefaultCluster.ES().Ccr.ResumeFollow(index).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to resume ccr following: %w", esErrorString(err))
return fmt.Errorf("failed to resume ccr following: %s", esErrorString(err))
}
return nil
@@ -106,8 +100,9 @@ func CcrFollowerResume(conf *cfg.Config, index string) error {
func CcrFollowerPause(conf *cfg.Config, index string) error {
_, err := conf.DefaultCluster.ES().Ccr.PauseFollow(index).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to pause ccr following: %w", esErrorString(err))
return fmt.Errorf("failed to pause ccr following: %s", esErrorString(err))
}
return nil
@@ -116,8 +111,9 @@ func CcrFollowerPause(conf *cfg.Config, index string) error {
func CcrFollowerUnfollow(conf *cfg.Config, index string) error {
_, err := conf.DefaultCluster.ES().Ccr.ForgetFollower(index).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to unfollow index: %w", esErrorString(err))
return fmt.Errorf("failed to unfollow index: %s", esErrorString(err))
}
return nil
@@ -138,8 +134,9 @@ func CcrFollowerAdd(conf *cfg.Config, index string) error {
}
_, err = create.Do(context.Background())
if err != nil {
return fmt.Errorf("failed to create follower index: %w", esErrorString(err))
return fmt.Errorf("failed to create follower index: %s", esErrorString(err))
}
return nil
@@ -149,7 +146,7 @@ func CcrFollowerShow(conf *cfg.Config, index string) error {
res, err := conf.DefaultCluster.ES().Ccr.FollowStats(index).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to retrieve follower index info: %w", esErrorString(err))
return fmt.Errorf("failed to retrieve follower index info: %s", esErrorString(err))
}
slog.Debug("ES result", "follower stats", res.Indices)
@@ -167,16 +164,16 @@ func CcrFollowerShow(conf *cfg.Config, index string) error {
table := printer.NewTable(conf, 2, 9)
table.Addheaders("ccr follower property", "value")
table.Entries = [][]any{
table.Entries = [][]string{
{"name", index},
{"remote_cluster", follower.RemoteCluster},
{"leader_checkpoint", follower.LeaderGlobalCheckpoint},
{"follower_checkpoint", follower.FollowerGlobalCheckpoint},
{"bytes_read", follower.BytesRead},
{"failed_read_requests", follower.FailedReadRequests},
{"failed_write_requests", follower.FailedWriteRequests},
{"successful_read_requests", follower.SuccessfulReadRequests},
{"successful_write_requests", follower.SuccessfulWriteRequests},
{"leader_checkpoint", fmt.Sprintf("%d", follower.LeaderGlobalCheckpoint)},
{"follower_checkpoint", fmt.Sprintf("%d", follower.FollowerGlobalCheckpoint)},
{"bytes_read", fmt.Sprintf("%d", follower.BytesRead)},
{"failed_read_requests", fmt.Sprintf("%d", follower.FailedReadRequests)},
{"failed_write_requests", fmt.Sprintf("%d", follower.FailedWriteRequests)},
{"successful_read_requests", fmt.Sprintf("%d", follower.SuccessfulReadRequests)},
{"successful_write_requests", fmt.Sprintf("%d", follower.SuccessfulWriteRequests)},
}
if err := table.Print(); err != nil {

169
pkg/es/cluster.go
View File

@@ -17,7 +17,6 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package es
import (
"errors"
"fmt"
"log/slog"
"strings"
@@ -32,62 +31,44 @@ import (
type ClusterIndices map[string]map[string]*types.IndicesRecord
type clusterReachable struct {
reachable bool
err error
}
func ClusterList(conf *cfg.Config) error {
var (
mu sync.Mutex
wg sync.WaitGroup
)
reachable := make(map[string]clusterReachable, len(conf.Clusters))
// check endpoints in parallel to speed things up
for name, cluster := range conf.Clusters {
wg.Go(func() {
online, err := cluster.IsReachable()
mu.Lock()
reachable[name] = clusterReachable{reachable: online, err: err}
mu.Unlock()
})
}
wg.Wait()
table := printer.NewTable(conf, 5, len(conf.Clusters))
table.Addheaders("cluster", "uri", "reachable", "current", "error")
idx := 0
for name, cluster := range conf.Clusters {
reachableStr := "no"
reachable := "no"
current := "no"
errmsg := ""
if reachable[name].reachable {
reachableStr = printer.Colorize(conf, "green", "reachable")
online, err := cluster.IsReachable()
if online {
reachable = printer.Colorize(conf, "green", "reachable")
}
if cluster.Default {
current = printer.Colorize(conf, "green", "yes")
if !reachable[name].reachable {
reachableStr = printer.Colorize(conf, "red", "no")
errmsg = reachable[name].err.Error()
if !online {
reachable = printer.Colorize(conf, "red", "no")
errmsg = err.Error()
}
}
table.Entries[idx] = []any{name, cluster.Uri, reachableStr, current, errmsg}
table.Entries[idx] = []string{name, cluster.Uri, reachable, current, errmsg}
idx++
}
table.Sort()
return table.Print()
if err := table.Print(); err != nil {
return err
}
return nil
}
// We're using goroutines here to parallelize API requests, since we
@@ -119,50 +100,45 @@ func getClusterStatus(conf *cfg.Config) (*apiResponse, error) {
all := apiResponse{}
var err error
for i := 0; i < gocount; i++ {
r := <-responses
for range gocount {
res := <-responses
if r.error != nil {
return nil, r.error
}
err = errors.Join(err, res.error)
switch res.which {
switch r.which {
case ResponseHealth:
all.health = res.health
all.health = r.health
case ResponseCcr:
all.ccr = res.ccr
all.ccr = r.ccr
case ResponseInfo:
all.info = res.info
all.info = r.info
case ResponseStats:
all.stats = res.stats
all.stats = r.stats
case ResponseIndices:
all.indices = res.indices
all.indices = r.indices
case ResponseTasks:
all.tasks = res.tasks
all.tasks = r.tasks
case ResponseHealthReport:
all.healthreport = res.healthreport
all.healthreport = r.healthreport
}
}
return &all, err
return &all, nil
}
func ClusterStatus(conf *cfg.Config) error {
res, err := getClusterStatus(conf)
if err != nil && !strings.Contains(err.Error(), "current license is non-compliant") {
if err != nil {
return err
}
slog.Debug("ES result", "cluster health", res.health)
var (
isleader bool
ccrfollowing string
)
if res.ccr != nil {
isleader = len(res.ccr.AutoFollowStats.AutoFollowedClusters) == 0
isleader := len(res.ccr.AutoFollowStats.AutoFollowedClusters) == 0
ccrfollowing := ""
if len(res.ccr.AutoFollowStats.AutoFollowedClusters) > 0 {
// is following another cluster
ccrfollowing = fmt.Sprintf("%s (%d/%d)",
@@ -171,25 +147,17 @@ func ClusterStatus(conf *cfg.Config) error {
res.ccr.AutoFollowStats.NumberOfFailedFollowIndices,
)
}
}
// look for red indices, if any
failedIndices := 0
greenIndices := 0
redindices := 0
for _, index := range *res.indices {
switch {
case *index.Health != "green":
failedIndices++
case !strings.HasPrefix(*index.Index, "."):
// don't count internal indices
greenIndices++
if *index.Health == "red" {
redindices++
}
}
// look for long running tasks
longtasks := 0
for _, task := range *res.tasks {
if strings.Contains(*task.RunningTime, "d") {
longtasks++
@@ -199,49 +167,44 @@ func ClusterStatus(conf *cfg.Config) error {
table := printer.NewTable(conf, 2, 7)
table.Addheaders(conf.DefaultCluster.Name, "status")
table.Entries = [][]any{
table.Entries = [][]string{
{"Cluster Name", res.health.ClusterName},
{"ES Status", printer.Colorize(conf, res.health.Status.Name, res.health.Status.Name)},
{"ES Version", res.info.Version.Int},
{"Is Leader", isleader},
{"Active Shards", res.health.ActiveShards},
{"Active Primary Shards", res.health.ActivePrimaryShards},
{"Unassigned Shards", res.health.UnassignedShards},
{"Unassigned Primary Shards", res.health.UnassignedPrimaryShards},
{"Pending Tasks", res.health.NumberOfPendingTasks},
{"Nodes", res.health.NumberOfNodes},
{"Green Indices", greenIndices},
{"Failed Indices", failedIndices},
{"Long Running Tasks", longtasks},
{"Is Leader", fmt.Sprintf("%t", isleader)},
{"Active Shards", fmt.Sprintf("%d", res.health.ActiveShards)},
{"Active Primary Shards", fmt.Sprintf("%d", res.health.ActivePrimaryShards)},
{"Unassigned Shards", fmt.Sprintf("%d", res.health.UnassignedShards)},
{"Unassigned Primary Shards", fmt.Sprintf("%d", res.health.UnassignedPrimaryShards)},
{"Pending Tasks", fmt.Sprintf("%d", res.health.NumberOfPendingTasks)},
{"Nodes", fmt.Sprintf("%d", res.health.NumberOfNodes)},
{"Red Indices", fmt.Sprintf("%d", redindices)},
{"Long Running Tasks", fmt.Sprintf("%d", longtasks)},
}
if !isleader && res.ccr != nil {
table.Entries = append(table.Entries, [][]any{
if !isleader {
table.Entries = append(table.Entries, [][]string{
{"AutoFollow (success/failed indices)", ccrfollowing},
{"Followed Indices", len(res.ccr.FollowStats.Indices)},
{"Followed Indices", fmt.Sprintf("%d", len(res.ccr.FollowStats.Indices))},
}...)
}
if conf.Verbose {
table = gatherClusterStats(res.stats, table)
table = gatherClusterStats(conf, res.stats, table)
}
if res.health.Status.Name != "green" {
for name, indicator := range res.healthreport.Indicators {
if indicator.Status != "green" {
table.Entries = append(table.Entries, []any{
table.Entries = append(table.Entries, []string{
printer.Colorize(conf, indicator.Status, "Bad health "+name), indicator.Symptom,
})
for _, diag := range indicator.Diagnosis {
table.Entries = append(table.Entries, []any{" -> cause", diag.Cause})
table.Entries = append(table.Entries, []string{" -> cause", diag.Cause})
for resource, items := range diag.AffectedResources {
table.Entries = append(table.Entries, []any{" -> affected " + resource, strings.Join(items, ",")})
}
if diag.Action != "" {
table.AddRow(" -> suggested action to fix", diag.Action)
table.Entries = append(table.Entries, []string{" -> affected " + resource, strings.Join(items, ",")})
}
}
}
@@ -255,11 +218,9 @@ func ClusterStatus(conf *cfg.Config) error {
return nil
}
func gatherClusterStats(clusterstats *clusterstats.Response, table *printer.Table) *printer.Table {
var (
querycount int64
vmversion string
)
func gatherClusterStats(conf *cfg.Config, clusterstats *clusterstats.Response, table *printer.Table) *printer.Table {
var querycount int64
var vmversion string
for _, count := range clusterstats.Indices.Search.Queries {
querycount += count
@@ -271,13 +232,13 @@ func gatherClusterStats(clusterstats *clusterstats.Response, table *printer.Tabl
clusterstats.Nodes.Jvm.Versions[0].VmVersion}, " ")
}
table.Entries = append(table.Entries, [][]any{
{"Indicies", clusterstats.Indices.Count},
{"Docs", clusterstats.Indices.Docs.Count},
{"Total Size", printer.Bytes(clusterstats.Indices.Docs.TotalSizeInBytes)},
{"Total Queries", "%d", querycount},
{"Shards Primaries", clusterstats.Indices.Shards.Primaries},
{"Shards Total", clusterstats.Indices.Shards.Total},
table.Entries = append(table.Entries, [][]string{
{"Indicies", fmt.Sprintf("%d", clusterstats.Indices.Count)},
{"Docs", fmt.Sprintf("%d", clusterstats.Indices.Docs.Count)},
{"Total Size", humanize.Bytes(uint64(clusterstats.Indices.Docs.TotalSizeInBytes))},
{"Total Queries", fmt.Sprintf("%d", querycount)},
{"Shards Primaries", fmt.Sprintf("%d", clusterstats.Indices.Shards.Primaries)},
{"Shards Total", fmt.Sprintf("%d", clusterstats.Indices.Shards.Total)},
{"Storage", fmt.Sprintf(
"%s/%s",
humanize.Bytes(uint64(clusterstats.Indices.Store.SizeInBytes)),
@@ -288,11 +249,11 @@ func gatherClusterStats(clusterstats *clusterstats.Response, table *printer.Tabl
humanize.Bytes(uint64(clusterstats.Nodes.Jvm.Mem.HeapUsedInBytes)),
humanize.Bytes(uint64(clusterstats.Nodes.Jvm.Mem.HeapMaxInBytes)),
)},
{"JVM Threads", clusterstats.Nodes.Jvm.Threads},
{"JVM Threads", fmt.Sprintf("%d", clusterstats.Nodes.Jvm.Threads)},
{"JVM Version", vmversion},
{"CPUs", clusterstats.Nodes.Os.AllocatedProcessors},
{"CPUs", fmt.Sprintf("%d", clusterstats.Nodes.Os.AllocatedProcessors)},
{"CPU Usage", fmt.Sprintf("%d%%", clusterstats.Nodes.Process.Cpu.Percent)},
{"Open FDs", clusterstats.Nodes.Process.OpenFileDescriptors.Avg},
{"Open FDs", fmt.Sprintf("%d", clusterstats.Nodes.Process.OpenFileDescriptors.Avg)},
}...)
return table

1
pkg/es/cluster_reroute.go
View File

@@ -118,7 +118,6 @@ func ClusterRerouteAllocatePrimary(conf *cfg.Config, index string, stale bool) e
if stale {
which = "stale"
}
return fmt.Errorf("failed to allocate %s primary shard: %w", which, err)
}

38
pkg/es/cluster_settings.go
View File

@@ -20,28 +20,23 @@ import (
"context"
"encoding/json"
"fmt"
"log/slog"
"codeberg.org/scip/esctl/pkg/cfg"
"codeberg.org/scip/esctl/pkg/printer"
"github.com/urfave/cli/v3"
)
func ClusterSettingsNames(conf *cfg.Config) ([]string, error) {
return validClusterSettings, nil
}
func ClusterSettingsList(conf *cfg.Config) error {
res, err := conf.DefaultCluster.ES().Cluster.GetSettings().
FlatSettings(true).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get cluster settings: %w", esErrorString(err))
return fmt.Errorf("failed to get cluster settings: %s", esErrorString(err))
}
table := printer.NewTable(conf, 2, 0)
table.Addheaders("setting", "value")
entries := [][]any{}
entries := [][]string{}
settingshash := res.Persistent // == map[string]json.RawMessage
@@ -53,7 +48,19 @@ func ClusterSettingsList(conf *cfg.Config) error {
}
for topic, val := range settingshash {
entries = append(entries, []any{topic, string(val)})
data := map[string]any{}
err := json.Unmarshal(val, &data)
if err != nil {
return fmt.Errorf("failed to unmarshall setting for topic %s: %s", topic, err)
}
paths := getJsonPath(map[string]string{}, data, topic)
slog.Debug("settings", topic, paths)
for setting, value := range paths {
entries = append(entries, []string{setting, fmt.Sprintf("%v", value)})
}
}
table.Entries = entries
@@ -76,25 +83,23 @@ func ClusterSettingsSet(conf *cfg.Config, args cli.Args) error {
case conf.Transient:
message, err := json.Marshal(value)
if err != nil {
return fmt.Errorf("failed to marshall transient value <%v> to valid JSON: %w", value, err)
return fmt.Errorf("failed to marshall transient value <%v> to valid JSON: %s", value, err)
}
put.AddTransient(setting, message)
case conf.Persistent:
fallthrough
default:
message, err := json.Marshal(value)
if err != nil {
return fmt.Errorf("failed to marshall persistent value <%v> to valid JSON: %w", value, err)
return fmt.Errorf("failed to marshall persistent value <%v> to valid JSON: %s", value, err)
}
put.AddPersistent(setting, message)
}
}
_, err := put.Do(context.Background())
if err != nil {
return fmt.Errorf("failed to set settings: %w", esErrorString(err))
return fmt.Errorf("failed to set settings: %s", esErrorString(err))
}
return nil
@@ -103,14 +108,15 @@ func ClusterSettingsSet(conf *cfg.Config, args cli.Args) error {
func ClusterSettingsSetSingle(conf *cfg.Config, setting, value string) error {
message, err := json.Marshal(value)
if err != nil {
return fmt.Errorf("failed to marshall persistent value <%v> to valid JSON: %w", value, err)
return fmt.Errorf("failed to marshall persistent value <%v> to valid JSON: %s", value, err)
}
_, err = conf.DefaultCluster.ES().Cluster.PutSettings().
AddPersistent(setting, message).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to set %s: %w", setting, esErrorString(err))
return fmt.Errorf("failed to set %s: %s", setting, esErrorString(err))
}
return nil

730
pkg/es/cluster_settings_const.go
View File

@@ -1,730 +0,0 @@
/*
Copyright © 2026 Thomas von Dein
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package es
// manually extracted from https://www.elastic.co/docs/reference/elasticsearch/configuration-reference
var validClusterSettings []string = []string{
"action.auto_create_index",
"action.destructive_requires_name",
"attribute_delimiters.groups",
"attribute_patterns.dn",
"attribute_patterns.groups",
"attribute_patterns.mail",
"attribute_patterns.name",
"attribute_patterns.principal",
"attributes.dn",
"attributes.groups",
"attributes.mail",
"attributes.name",
"attributes.principal",
"authentication.enabled",
"auth.secure_password",
"auth.username",
"breaker.eql_sequence.limit",
"breaker.eql_sequence.overhead",
"breaker.eql_sequence.type",
"breaker.model_inference.limit",
"breaker.model_inference.overhead",
"breaker.model_inference.type",
"cache.hash_algo",
"cache.max_users",
"cache.ttl",
"ccr.indices.recovery.chunk_size",
"ccr.indices.recovery.internal_action_timeout",
"ccr.indices.recovery.max_bytes_per_sec",
"ccr.indices.recovery.max_concurrent_file_chunks",
"ccr.indices.recovery.recovery_activity_timeout",
"claim_patterns.dn",
"claim_patterns.group",
"claim_patterns.groups",
"claim_patterns.mail",
"claim_patterns.name",
"claim_patterns.principal",
"claims.dn",
"claims.groups",
"claims.mail",
"claims.name",
"claims.principal",
"client_authentication.rotation_grace_period",
"client_authentication.shared_secret",
"client_authentication.type",
"cluster_alerts.management.blacklist",
"cluster_alerts.management.enabled",
"cluster.auto_shrink_voting_configuration",
"cluster.discovery_configuration_check.interval",
"cluster.election.duration",
"cluster.election.initial_timeout",
"cluster.election.max_timeout",
"cluster.fault_detection.follower_check.retry_count",
"cluster.fault_detection.follower_check.timeout",
"cluster.fault_detection.leader_check.interval",
"cluster.fault_detection.leader_check.retry_count",
"cluster.fault_detection.leader_check.timeout",
"cluster.follower_lag.timeout",
"cluster.indices.close.enable",
"cluster.info.update.interval",
"cluster.initial_master_nodes",
"cluster.join_validation.cache_timeout",
"cluster.lifecycle.default.rollover",
"cluster.max_voting_config_exclusions",
"cluster.no_master_block",
"cluster.publish.info_timeout",
"cluster.publish.timeout",
"cluster.reindex.pit.keep_alive",
"cluster.remote.initial_connect_timeout",
"cluster.remote.node.attr",
"cluster.remote.signing.certificate_authorities",
"cluster.remote.signing.diagnose.trust",
"cluster.remote.signing.truststore.algorithm",
"cluster.remote.signing.truststore.path",
"cluster.remote.signing.truststore.secure_password",
"cluster.remote.signing.truststore.type",
"cluster.routing.allocation.allow_rebalance",
"cluster.routing.allocation.awareness.attributes",
"cluster.routing.allocation.awareness.force.*",
"cluster.routing.allocation.balance.disk_usage",
"cluster.routing.allocation.balance.index",
"cluster.routing.allocation.balance.shard",
"cluster.routing.allocation.balance.threshold",
"cluster.routing.allocation.balance.write_load",
"cluster.routing.allocation.cluster_concurrent_rebalance",
"cluster.routing.allocation.disk.threshold_enabled",
"cluster.routing.allocation.disk.watermark.enable_for_single_data_node",
"cluster.routing.allocation.disk.watermark.flood_stage",
"cluster.routing.allocation.disk.watermark.flood_stage.frozen",
"cluster.routing.allocation.disk.watermark.flood_stage.frozen.max_headroom",
"cluster.routing.allocation.disk.watermark.flood_stage.max_headroom",
"cluster.routing.allocation.disk.watermark.high",
"cluster.routing.allocation.disk.watermark.high.max_headroom",
"cluster.routing.allocation.disk.watermark.low",
"cluster.routing.allocation.disk.watermark.low.max_headroom",
"cluster.routing.allocation.enable",
"cluster.routing.allocation.exclude.{{attribute}}",
"cluster.routing.allocation.include.{{attribute}}",
"cluster.routing.allocation.node_concurrent_incoming_recoveries",
"cluster.routing.allocation.node_concurrent_outgoing_recoveries",
"cluster.routing.allocation.node_concurrent_recoveries",
"cluster.routing.allocation.node_initial_primaries_recoveries",
"cluster.routing.allocation.require.{{attribute}}",
"cluster.routing.allocation.same_shard.host",
"cluster.routing.allocation.stats.cache.ttl",
"cluster.routing.allocation.total_shards_per_node",
"cluster.routing.allocation.type",
"cluster.routing.rebalance.enable",
"connection.read_timeout",
"connection.timeout",
"data_streams.lifecycle.downsampling.max_indices_in_progress",
"data_streams.lifecycle.poll_interval",
"data_streams.lifecycle.retention.default",
"data_streams.lifecycle.retention.max",
"data_streams.lifecycle.signalling.error_retry_interval",
"data_streams.lifecycle.target.merge.policy.floor_segment",
"data_streams.lifecycle.target.merge.policy.merge_factor",
"delegation.enabled",
"discovery.cluster_formation_warning_timeout",
"discovery.find_peers_interval",
"discovery.find_peers_warning_timeout",
"discovery.probe.connect_timeout",
"discovery.probe.handshake_timeout",
"discovery.request_peers_timeout",
"discovery.seed_hosts",
"discovery.seed_providers",
"discovery.seed_resolver.max_concurrent_resolvers",
"discovery.seed_resolver.timeout",
"discovery.type",
"dlm.frozen.cleanup.poll_interval",
"dlm.frozen.transition.poll_interval",
"dlm.frozen.transition.thread_pool.queue_size",
"dlm.frozen.transition.thread_pool.size",
"email_defaults.*",
"encryption.certificate",
"encryption.key",
"encryption.keystore.alias",
"encryption.keystore.path",
"encryption.keystore.secure_key_password",
"encryption.keystore.secure_password",
"encryption.keystore.type",
"encryption.secure_key_passphrase",
"enrich.cache_size",
"enrich.cleanup_period",
"enrich.coordinator_proxy.max_concurrent_requests",
"enrich.coordinator_proxy.max_lookups_per_request",
"enrich.coordinator_proxy.queue_capacity",
"enrich.fetch_size",
"enrich.max_concurrent_policy_executions",
"enrich.max_force_merge_attempts",
"fallback_claims.aud",
"fallback_claims.sub",
"files.role_mapping",
"gateway.expected_data_nodes",
"gateway.recover_after_data_nodes",
"gateway.recover_after_time",
"group_search.base_dn",
"group_search.filter",
"group_search.scope",
"group_search.user_attribute",
"health.ilm.max_retries_per_step",
"health.ilm.max_time_on_action",
"health.ilm.max_time_on_step",
"health.master_history.has_master_lookup_timeframe",
"health.master_history.identity_changes_threshold",
"health.master_history.no_master_transitions_threshold",
"health.node.enabled",
"health.periodic_logger.enabled",
"health.periodic_logger.poll_interval",
"health.reporting.local.monitor.interval",
"health.shard_capacity.unhealthy_threshold.red",
"health.shard_capacity.unhealthy_threshold.yellow",
"http.bind_host",
"http.client_stats.closed_channels.max_age",
"http.client_stats.closed_channels.max_count",
"http.client_stats.enabled",
"http.compression",
"http.compression_level",
"http.connection_pool_ttl",
"http.connection_read_timeout",
"http.connect_timeout",
"http.cors.enabled",
"http.detailed_errors.enabled",
"http.host",
"http.max_connections",
"http.max_content_length",
"http.max_endpoint_connections",
"http.max_header_size",
"http.max_initial_line_length",
"http.max_warning_header_count",
"http.max_warning_header_size",
"http.pipelining.max_events",
"http.port",
"http.proxy.host",
"http.proxy.port",
"http.proxy.scheme",
"http.publish_host",
"http.publish_port",
"http.socket_timeout",
"http.tcp.keep_alive",
"http.tcp.keep_count",
"http.tcp.keep_idle",
"http.tcp.keep_interval",
"http.tcp.no_delay",
"http.tcp.receive_buffer_size",
"http.tcp.reuse_address",
"http.tcp.send_buffer_size",
"idp.entity_id",
"idp.metadata.http.connect_timeout",
"idp.metadata.http.fail_on_error",
"idp.metadata.http.minimum_refresh",
"idp.metadata.http.read_timeout",
"idp.metadata.http.refresh",
"idp.metadata.path",
"idp.use_single_logout",
"index.dlm.frozen.created",
"indexing_pressure.memory.limit",
"index.lifecycle.indexing_complete",
"index.lifecycle.name",
"index.lifecycle.origination_date",
"index.lifecycle.parse_origination_date",
"index.lifecycle.prefer_ilm",
"index.lifecycle.rollover_alias",
"index.lifecycle.step.wait_time_threshold",
"index.name.time_format",
"index.queries.cache.enabled",
"indices.breaker.fielddata.limit",
"indices.breaker.fielddata.overhead",
"indices.breaker.request.limit",
"indices.breaker.request.overhead",
"indices.breaker.total.limit",
"indices.breaker.total.use_real_memory",
"indices.fielddata.cache.size",
"indices.lifecycle.history_index_enabled",
"indices.lifecycle.poll_interval",
"indices.lifecycle.rollover.only_if_has_documents",
"indices.memory.index_buffer_size",
"indices.memory.max_index_buffer_size",
"indices.memory.min_index_buffer_size",
"indices.queries.cache.size",
"indices.query.bool.max_clause_count",
"indices.query.bool.max_nested_depth",
"indices.recovery.max_bytes_per_sec",
"indices.recovery.max_concurrent_file_chunks",
"indices.recovery.max_concurrent_operations",
"indices.recovery.max_concurrent_snapshot_file_downloads",
"indices.recovery.max_concurrent_snapshot_file_downloads_per_node",
"indices.recovery.use_snapshots",
"indices.requests.cache.expire",
"indices.requests.cache.size",
"jwt.cache.size",
"jwt.cache.ttl",
"keytab.path",
"krb.debug",
"load_balance.cache_ttl",
"load_balance.type",
"master_history.max_age",
"monitor.fs.health.enabled",
"monitor.fs.health.refresh_interval",
"monitor.fs.health.slow_path_logging_threshold",
"nameid.allow_create",
"nameid.sp_qualifier",
"network.bind_host",
"network.breaker.inflight_requests.limit",
"network.breaker.inflight_requests.overhead",
"network.host",
"network.publish_host",
"network.tcp.keep_alive",
"network.tcp.keep_count",
"network.tcp.keep_idle",
"network.tcp.keep_interval",
"network.tcp.no_delay",
"network.tcp.receive_buffer_size",
"network.tcp.reuse_address",
"network.tcp.send_buffer_size",
"network.thread.watchdog.interval",
"network.thread.watchdog.quiet_time",
"node.bandwidth.recovery.disk.read",
"node.bandwidth.recovery.disk.write",
"node.bandwidth.recovery.factor.read",
"node.bandwidth.recovery.factor.write",
"node.bandwidth.recovery.network",
"node.bandwidth.recovery.operator.factor",
"node.bandwidth.recovery.operator.factor.max_overcommit",
"node.bandwidth.recovery.operator.factor.read",
"node.bandwidth.recovery.operator.factor.write",
"node.roles: [ ml ]",
"node.roles: [ transform ]",
"op.authorization_endpoint",
"op.endsession_endpoint",
"op.issuer",
"op.jwkset_path",
"op.token_endpoint",
"op.userinfo_endpoint",
"org.elasticsearch.common.network.ThreadWatchdog",
"org.elasticsearch.transport.InboundHandler",
"org.elasticsearch.transport.OutboundHandler",
"pkc_jwkset_reload.enabled",
"pkc_jwkset_reload.file_interval",
"pkc_jwkset_reload.url_interval_max",
"pkc_jwkset_reload.url_interval_min",
"proxy.base_path",
"reindex.remote.blocklist",
"reindex.remote.whitelist",
"reindex.ssl.certificate",
"reindex.ssl.certificate_authorities",
"reindex.ssl.key",
"reindex.ssl.key_passphrase",
"reindex.ssl.keystore.key_password",
"reindex.ssl.keystore.password",
"reindex.ssl.keystore.path",
"reindex.ssl.keystore.secure_key_password",
"reindex.ssl.keystore.secure_password",
"reindex.ssl.keystore.type",
"reindex.ssl.secure_key_passphrase",
"reindex.ssl.truststore.password",
"reindex.ssl.truststore.path",
"reindex.ssl.truststore.secure_password",
"reindex.ssl.truststore.type",
"reindex.ssl.verification_mode",
"remote_cluster.bind_host",
"remote_cluster.host",
"remote_cluster.port",
"remote_cluster.publish_host",
"remote_cluster.publish_port",
"remote_cluster_server.enabled",
"remote_cluster.tcp.keep_alive",
"remote_cluster.tcp.keep_count",
"remote_cluster.tcp.keep_idle",
"remote_cluster.tcp.keep_interval",
"remote_cluster.tcp.no_delay",
"remote_cluster.tcp.receive_buffer_size",
"remote_cluster.tcp.reuse_address",
"remote_cluster.tcp.send_buffer_size",
"repositories.default_repository",
"rp.client_auth_jwt_signature_algorithm",
"rp.client_auth_method",
"rp.client_id",
"rp.client_secret",
"rp.post_logout_redirect_uri",
"rp.redirect_uri",
"rp.requested_scopes",
"rp.response_type",
"rp.signature_algorithm",
"script.max_compilations_rate",
"script.painless.regex.enabled",
"search.aggs.allowed_inline_metric_scripts",
"search.aggs.allowed_stored_metric_scripts",
"search.aggs.only_allowed_metric_scripts",
"search.max_buckets",
"search.task_watchdog.cooldown_period",
"search.task_watchdog.coordinator_threshold",
"search.task_watchdog.data_node_threshold",
"search.task_watchdog.enabled",
"search.task_watchdog.interval",
"signing.certificate",
"signing.key",
"signing.keystore.alias",
"signing.keystore.path",
"signing.keystore.secure_key_password",
"signing.keystore.secure_password",
"signing.keystore.type",
"signing.saml_messages",
"signing.secure_key_passphrase",
"slm.health.failed_snapshot_warn_threshold",
"slm.history_index_enabled",
"slm.retention_duration",
"slm.retention_schedule",
"smtp.auth",
"smtp.connection_timeout",
"smtp.host",
"smtp.local_address",
"smtp.local_port",
"smtp.port",
"smtp.secure_password",
"smtp.send_partial",
"smtp.ssl.trust",
"smtp.starttls.enable",
"smtp.starttls.required",
"smtp.timeout",
"smtp.user",
"smtp.wait_on_quit",
"smtp.write_timeout",
"snapshot.max_concurrent_operations",
"sp.acs",
"sp.entity_id",
"sp.logout",
"ssl.certificate",
"ssl.certificate_authorities",
"ssl.cipher_suites",
"ssl.key",
"ssl.key_passphrase",
"ssl.keystore.key_password",
"ssl.keystore.password",
"ssl.keystore.path",
"ssl.keystore.secure_key_password",
"ssl.keystore.secure_password",
"ssl.keystore.type",
"ssl.secure_key_passphrase",
"ssl.secure_keystore.password",
"ssl.supported_protocols",
"ssl.truststore.password",
"ssl.truststore.path",
"ssl.truststore.secure_password",
"ssl.truststore.type",
"ssl.verification_mode",
"stack.templates.enabled",
"timeout.ldap_search",
"timeout.response",
"timeout.tcp_connect",
"timeout.tcp_read",
"transport.bind_host",
"transport.compress",
"transport.compression_scheme",
"transport.connect_timeout",
"transport.host",
"transport.ping_schedule",
"transport.port",
"transport.profiles.$PROFILE.xpack.security.filter.allow",
"transport.profiles.$PROFILE.xpack.security.filter.deny",
"transport.publish_host",
"transport.publish_port",
"transport.tcp.keep_alive",
"transport.tcp.keep_count",
"transport.tcp.keep_idle",
"transport.tcp.keep_interval",
"transport.tcp.no_delay",
"transport.tcp.receive_buffer_size",
"transport.tcp.reuse_address",
"transport.tcp.send_buffer_size",
"truststore.algorithm",
"truststore.password",
"truststore.path",
"truststore.secure_password",
"user_search.attribute",
"user_search.base_dn",
"user_search.down_level_filter",
"user_search.filter",
"user_search.pool.enabled",
"user_search.pool.health_check.dn",
"user_search.pool.health_check.enabled",
"user_search.pool.health_check.interval",
"user_search.pool.initial_size",
"user_search.pool.size",
"user_search.scope",
"user_search.upn_filter",
"vectors.indexing.use_gpu",
"wait_master.timeout",
"xpack.http.connection_pool_ttl",
"xpack.http.default_connection_timeout",
"xpack.http.default_read_timeout",
"xpack.http.max_response_size",
"xpack.http.proxy.host",
"xpack.http.proxy.port",
"xpack.http.proxy.scheme",
"xpack.http.ssl.certificate",
"xpack.http.ssl.certificate_authorities",
"xpack.http.ssl.cipher_suites",
"xpack.http.ssl.key",
"xpack.http.ssl.keystore.path",
"xpack.http.ssl.keystore.secure_key_password",
"xpack.http.ssl.keystore.secure_password",
"xpack.http.ssl.keystore.type",
"xpack.http.ssl.secure_key_passphrase",
"xpack.http.ssl.supported_protocols",
"xpack.http.ssl.truststore.path",
"xpack.http.ssl.truststore.secure_password",
"xpack.http.ssl.truststore.type",
"xpack.http.ssl.verification_mode",
"xpack.http.tcp.keep_alive",
"xpack.http.whitelist",
"xpack.ilm.enabled",
"xpack.inference.ccm.cache.expiry_time",
"xpack.inference.ccm.cache.weight",
"xpack.inference.endpoint.cache.enabled",
"xpack.inference.endpoint.cache.expiry_time",
"xpack.inference.endpoint.cache.weight",
"xpack.inference.http.connection_eviction_interval",
"xpack.inference.http.connection_eviction_max_idle_time",
"xpack.inference.http.max_response_size",
"xpack.inference.http.max_route_connections",
"xpack.inference.http.max_total_connections",
"xpack.inference.http.request_executor.queue_capacity",
"xpack.inference.http.retry.initial_delay",
"xpack.inference.http.retry.max_delay_bound",
"xpack.inference.http.retry.timeout",
"xpack.inference.logging.reset_interval",
"xpack.inference.logging.wait_duration",
"xpack.inference.oauth2.token_cache.enabled",
"xpack.inference.oauth2.token_cache.expiry_time",
"xpack.inference.oauth2.token_cache.weight",
"xpack.inference.query_timeout",
"xpack.inference.truncator.reduction_percentage",
"xpack.license.self_generated.type",
"xpack.ml.anomalies.heal_reindexed_v7.enabled",
"xpack.ml.enable_config_migration",
"xpack.ml.enabled",
"xpack.ml.idle_job_auto_close_timeout",
"xpack.ml.inference_model.cache_size",
"xpack.ml.inference_model.time_to_live",
"xpack.ml.max_anomaly_records",
"xpack.ml.max_inference_processors",
"xpack.ml.max_lazy_ml_nodes",
"xpack.ml.max_machine_memory_percent",
"xpack.ml.max_ml_node_size",
"xpack.ml.max_model_memory_limit",
"xpack.ml.max_open_jobs",
"xpack.ml.model_repository",
"xpack.ml.nightly_maintenance_requests_per_second",
"xpack.ml.node_concurrent_job_allocations",
"xpack.ml.persist_results_max_retries",
"xpack.ml.process_connect_timeout",
"xpack.ml.results_index_rollover_max_size",
"xpack.ml.trained_models.graph_validation_enabled",
"xpack.ml.use_auto_machine_memory_percent",
"xpack.monitoring.collection.cluster.stats.timeout",
"xpack.monitoring.collection.enabled",
"xpack.monitoring.collection.index.recovery.active_only",
"xpack.monitoring.collection.index.recovery.timeout",
"xpack.monitoring.collection.index.stats.timeout",
"xpack.monitoring.collection.indices",
"xpack.monitoring.collection.interval",
"xpack.monitoring.collection.node.stats.timeout",
"xpack.monitoring.elasticsearch.collection.enabled",
"xpack.monitoring.enabled",
"xpack.monitoring.exporters",
"xpack.monitoring.exporters.$NAME.ssl.certificate",
"xpack.monitoring.exporters.$NAME.ssl.certificate_authorities",
"xpack.monitoring.exporters.$NAME.ssl.cipher_suites",
"xpack.monitoring.exporters.$NAME.ssl.key",
"xpack.monitoring.exporters.$NAME.ssl.key_passphrase",
"xpack.monitoring.exporters.$NAME.ssl.keystore.key_password",
"xpack.monitoring.exporters.$NAME.ssl.keystore.password",
"xpack.monitoring.exporters.$NAME.ssl.keystore.path",
"xpack.monitoring.exporters.$NAME.ssl.keystore.secure_key_password",
"xpack.monitoring.exporters.$NAME.ssl.keystore.secure_password",
"xpack.monitoring.exporters.$NAME.ssl.keystore.type",
"xpack.monitoring.exporters.$NAME.ssl.secure_key_passphrase",
"xpack.monitoring.exporters.$NAME.ssl.supported_protocols",
"xpack.monitoring.exporters.$NAME.ssl.truststore.password",
"xpack.monitoring.exporters.$NAME.ssl.truststore.path",
"xpack.monitoring.exporters.$NAME.ssl.truststore.secure_password",
"xpack.monitoring.exporters.$NAME.ssl.truststore.type",
"xpack.monitoring.exporters.$NAME.ssl.verification_mode",
"xpack.monitoring.history.duration",
"xpack.notification.email.account",
"xpack.notification.email.account.domain_allowlist",
"xpack.notification.email.default_account",
"xpack.notification.email.html.sanitization.allow",
"xpack.notification.email.html.sanitization.disallow",
"xpack.notification.email.html.sanitization.enabled",
"xpack.notification.email.recipient_allowlist",
"xpack.notification.email.ssl.certificate",
"xpack.notification.email.ssl.certificate_authorities",
"xpack.notification.email.ssl.cipher_suites",
"xpack.notification.email.ssl.key",
"xpack.notification.email.ssl.keystore.path",
"xpack.notification.email.ssl.keystore.secure_key_password",
"xpack.notification.email.ssl.keystore.secure_password",
"xpack.notification.email.ssl.keystore.type",
"xpack.notification.email.ssl.secure_key_passphrase",
"xpack.notification.email.ssl.supported_protocols",
"xpack.notification.email.ssl.truststore.path",
"xpack.notification.email.ssl.truststore.secure_password",
"xpack.notification.email.ssl.truststore.type",
"xpack.notification.email.ssl.verification_mode",
"xpack.notification.jira.account",
"xpack.notification.jira.default_account",
"xpack.notification.pagerduty",
"xpack.notification.pagerduty.account",
"xpack.notification.pagerduty.default_account",
"xpack.notification.slack",
"xpack.notification.slack.account",
"xpack.notification.slack.default_account",
"xpack.notification.webhook.additional_token_enabled",
"xpack.otel_data.histogram_field_type",
"xpack.otel_data.registry.enabled",
"xpack.profiling.enabled",
"xpack.profiling.templates.enabled",
"xpack.security.audit.enabled",
"xpack.security.audit.logfile.emit_node_host_address",
"xpack.security.audit.logfile.emit_node_host_name",
"xpack.security.audit.logfile.emit_node_id",
"xpack.security.audit.logfile.emit_node_name",
"xpack.security.audit.logfile.events.emit_request_body",
"xpack.security.audit.logfile.events.exclude",
"xpack.security.audit.logfile.events.include",
"xpack.security.authc.anonymous.authz_exception",
"xpack.security.authc.anonymous.roles",
"xpack.security.authc.anonymous.username",
"xpack.security.authc.api_key.cache.hash_algo",
"xpack.security.authc.api_key.cache.max_keys",
"xpack.security.authc.api_key.cache.ttl",
"xpack.security.authc.api_key.delete.interval",
"xpack.security.authc.api_key.delete.retention_period",
"xpack.security.authc.api_key.delete.timeout",
"xpack.security.authc.api_key.enabled",
"xpack.security.authc.api_key.hashing.algorithm",
"xpack.security.authc.password_hashing.algorithm",
"xpack.security.authc.realms.jwt.*",
"xpack.security.authc.realms.kerberos.*",
"xpack.security.authc.realms.oidc.*",
"xpack.security.authc.realms.saml.*",
"xpack.security.authc.token.enabled",
"xpack.security.authc.token.timeout",
"xpack.security.autoconfiguration.enabled",
"xpack.security.automata.cache.enabled",
"xpack.security.automata.cache.size",
"xpack.security.automata.cache.ttl",
"xpack.security.automata.max_determinized_states",
"xpack.security.dls.bitset.cache.size",
"xpack.security.dls.bitset.cache.ttl",
"xpack.security.dls_fls.enabled",
"xpack.security.enabled",
"xpack.security.enrollment.enabled",
"xpack.security.fips_mode.enabled",
"xpack.security.fips_mode.required_providers",
"xpack.security.hide_settings",
"xpack.security.http.filter.allow",
"xpack.security.http.filter.deny",
"xpack.security.http.ssl.certificate",
"xpack.security.http.ssl.certificate_authorities",
"xpack.security.http.ssl.cipher_suites",
"xpack.security.http.ssl.client_authentication",
"xpack.security.http.ssl.enabled",
"xpack.security.http.ssl.key",
"xpack.security.http.ssl.key_passphrase",
"xpack.security.http.ssl.keystore.key_password",
"xpack.security.http.ssl.keystore.password",
"xpack.security.http.ssl.keystore.path",
"xpack.security.http.ssl.keystore.secure_key_password",
"xpack.security.http.ssl.keystore.secure_password",
"xpack.security.http.ssl.keystore.type",
"xpack.security.http.ssl.secure_key_passphrase",
"xpack.security.http.ssl.supported_protocols",
"xpack.security.http.ssl.truststore.password",
"xpack.security.http.ssl.truststore.path",
"xpack.security.http.ssl.truststore.secure_password",
"xpack.security.http.ssl.truststore.type",
"xpack.security.http.ssl.verification_mode",
"xpack.security.loginAssistanceMessage",
"xpack.security.remote_cluster_client.ssl.certificate",
"xpack.security.remote_cluster_client.ssl.certificate_authorities",
"xpack.security.remote_cluster_client.ssl.cipher_suites",
"xpack.security.remote_cluster_client.ssl.enabled",
"xpack.security.remote_cluster_client.ssl.handshake_timeout",
"xpack.security.remote_cluster_client.ssl.key",
"xpack.security.remote_cluster_client.ssl.keystore.path",
"xpack.security.remote_cluster_client.ssl.keystore.secure_key_password",
"xpack.security.remote_cluster_client.ssl.keystore.secure_password",
"xpack.security.remote_cluster_client.ssl.keystore.type",
"xpack.security.remote_cluster_client.ssl.secure_key_passphrase",
"xpack.security.remote_cluster_client.ssl.supported_protocols",
"xpack.security.remote_cluster_client.ssl.truststore.path",
"xpack.security.remote_cluster_client.ssl.truststore.secure_password",
"xpack.security.remote_cluster_client.ssl.truststore.type",
"xpack.security.remote_cluster_client.ssl.verification_mode",
"xpack.security.remote_cluster.filter.allow",
"xpack.security.remote_cluster.filter.deny",
"xpack.security.remote_cluster_server.ssl.certificate",
"xpack.security.remote_cluster_server.ssl.certificate_authorities",
"xpack.security.remote_cluster_server.ssl.cipher_suites",
"xpack.security.remote_cluster_server.ssl.client_authentication",
"xpack.security.remote_cluster_server.ssl.enabled",
"xpack.security.remote_cluster_server.ssl.handshake_timeout",
"xpack.security.remote_cluster_server.ssl.key",
"xpack.security.remote_cluster_server.ssl.keystore.path",
"xpack.security.remote_cluster_server.ssl.keystore.secure_key_password",
"xpack.security.remote_cluster_server.ssl.keystore.secure_password",
"xpack.security.remote_cluster_server.ssl.keystore.type",
"xpack.security.remote_cluster_server.ssl.secure_key_passphrase",
"xpack.security.remote_cluster_server.ssl.supported_protocols",
"xpack.security.remote_cluster_server.ssl.truststore.path",
"xpack.security.remote_cluster_server.ssl.truststore.secure_password",
"xpack.security.remote_cluster_server.ssl.truststore.type",
"xpack.security.remote_cluster_server.ssl.verification_mode",
"xpack.security.ssl.diagnose.trust",
"xpack.security.transport.filter.allow",
"xpack.security.transport.filter.deny",
"xpack.security.transport.ssl.certificate",
"xpack.security.transport.ssl.certificate_authorities",
"xpack.security.transport.ssl.cipher_suites",
"xpack.security.transport.ssl.client_authentication",
"xpack.security.transport.ssl.enabled",
"xpack.security.transport.ssl.handshake_timeout",
"xpack.security.transport.ssl.key",
"xpack.security.transport.ssl.key_passphrase",
"xpack.security.transport.ssl.keystore.key_password",
"xpack.security.transport.ssl.keystore.password",
"xpack.security.transport.ssl.keystore.path",
"xpack.security.transport.ssl.keystore.secure_key_password",
"xpack.security.transport.ssl.keystore.secure_password",
"xpack.security.transport.ssl.keystore.type",
"xpack.security.transport.ssl.secure_key_passphrase",
"xpack.security.transport.ssl.supported_protocols",
"xpack.security.transport.ssl.trust_restrictions.x509_fields",
"xpack.security.transport.ssl.truststore.password",
"xpack.security.transport.ssl.truststore.path",
"xpack.security.transport.ssl.truststore.secure_password",
"xpack.security.transport.ssl.truststore.type",
"xpack.security.transport.ssl.verification_mode",
"xpack.transform.enabled",
"xpack.transform.num_transform_failure_retries",
"xpack.watcher.enabled",
"xpack.watcher.encryption_key",
"xpack.watcher.encrypt_sensitive_data",
"xpack.watcher.history.cleaner_service.enabled",
"xpack.watcher.max.history.record.size",
"xpack.watcher.trigger.schedule.engine",
}

93
pkg/es/cluster_util.go
View File

@@ -20,6 +20,7 @@ import (
"context"
"fmt"
"regexp"
"strconv"
"strings"
"codeberg.org/scip/esctl/pkg/cfg"
@@ -37,7 +38,6 @@ func checkClusterIsLeader(conf *cfg.Config, leader string) bool {
Do(context.Background())
if err != nil {
fmt.Printf("failed to get ccr stats from %s: %s", leader, esErrorString(err))
return false
}
@@ -48,7 +48,6 @@ func checkClusterIsLeader(conf *cfg.Config, leader string) bool {
if stats.AutoFollowStats.AutoFollowedClusters[0].ClusterName != "" {
fmt.Println("leader/follower attribution is invalid, reverse cluster attribution and retry")
return false
}
@@ -60,48 +59,46 @@ func checkClusterStatus(conf *cfg.Config, leader, follower string) bool {
status := map[string]*health.Response{}
for _, cluster := range []string{leader, follower} {
clusterHealth, err := conf.Clusters[leader].ES().Cluster.Health().
st, err := conf.Clusters[leader].ES().Cluster.Health().
Do(context.Background())
if err != nil {
fmt.Printf("failed to get health from %s: %s", cluster, esErrorString(err))
return false
}
status[cluster] = clusterHealth
status[cluster] = st
}
table := printer.NewTable(conf, 3, 6)
table.Addheaders("setting", "leader:"+leader, "follower:"+follower)
table.Entries = [][]any{
table.Entries = [][]string{
{"Cluster Name", status[leader].ClusterName, status[follower].ClusterName},
{"Cluster Status",
printer.Colorize(conf, status[leader].Status.Name, status[leader].Status.Name),
printer.Colorize(conf, status[follower].Status.Name, status[follower].Status.Name),
},
{"Active Shards",
status[leader].ActiveShards,
status[follower].ActiveShards,
fmt.Sprintf("%d", status[leader].ActiveShards),
fmt.Sprintf("%d", status[follower].ActiveShards),
},
{"Active Primary Shards",
status[leader].ActivePrimaryShards,
status[follower].ActivePrimaryShards,
fmt.Sprintf("%d", status[leader].ActivePrimaryShards),
fmt.Sprintf("%d", status[follower].ActivePrimaryShards),
},
{"Indicies",
len(status[leader].Indices),
len(status[follower].Indices),
fmt.Sprintf("%d", len(status[leader].Indices)),
fmt.Sprintf("%d", len(status[follower].Indices)),
},
{"Nodes",
status[leader].NumberOfNodes,
status[follower].NumberOfNodes,
fmt.Sprintf("%d", status[leader].NumberOfNodes),
fmt.Sprintf("%d", status[follower].NumberOfNodes),
},
}
if err := table.Print(); err != nil {
fmt.Println(err)
return false
}
@@ -116,7 +113,7 @@ func checkClusterStatus(conf *cfg.Config, leader, follower string) bool {
// finds indices on both clusters which have ilm errors
func findIlmErrors(conf *cfg.Config, leader, follower string) bool {
failed := map[string]map[string]any{}
failed := map[string]map[string]string{}
for _, cluster := range []string{leader, follower} {
ilm, err := conf.Clusters[cluster].ES().Ilm.ExplainLifecycle("_all").
@@ -124,17 +121,16 @@ func findIlmErrors(conf *cfg.Config, leader, follower string) bool {
Do(context.Background())
if err != nil {
fmt.Printf("failed to get ilm status from %s: %s", cluster, esErrorString(err))
return false
}
failed[cluster] = map[string]any{}
failed[cluster] = map[string]string{}
for name, ilmstate := range ilm.Indices {
count := ilmstate.(*types.LifecycleExplainManaged).FailedStepRetryCount
if count != nil && *count > 0 {
failed[cluster][name] = *count
failed[cluster][name] = fmt.Sprintf("%d", *count)
}
}
}
@@ -155,7 +151,7 @@ func findIlmErrors(conf *cfg.Config, leader, follower string) bool {
table.Addheaders("ilm errors on "+which, "errors")
for name, count := range failed[cluster] {
table.Entries[idx] = []any{name, count}
table.Entries[idx] = []string{name, count}
idx++
}
@@ -163,7 +159,6 @@ func findIlmErrors(conf *cfg.Config, leader, follower string) bool {
if err := table.Print(); err != nil {
fmt.Println(err)
return false
}
}
@@ -180,7 +175,6 @@ func findIndicesOnlyOnLeader(conf *cfg.Config, indices ClusterIndices, leader, f
}
indexOnlyOnLeader := map[string]*types.IndicesRecord{}
for name, index := range indices[leader] {
if exclude.MatchString(name) {
continue
@@ -202,11 +196,9 @@ func findIndicesOnlyOnLeader(conf *cfg.Config, indices ClusterIndices, leader, f
}
isWritable := false
for _, alias := range res[name].Aliases {
if *alias.IsWriteIndex {
isWritable = true
break
}
}
@@ -231,15 +223,13 @@ func findIndicesOnlyOnLeader(conf *cfg.Config, indices ClusterIndices, leader, f
for name, index := range indexOnlyOnLeader {
name := printer.Colorize(conf, "red", name)
table.Entries[idx] = []any{name, *index.DatasetSize, *index.DocsCount}
table.Entries[idx] = []string{name, *index.DatasetSize, *index.DocsCount}
idx++
}
table.Sort()
if err := table.Print(); err != nil {
fmt.Println(err)
return false
}
@@ -281,15 +271,13 @@ func findOrphanedIndices(conf *cfg.Config, indices ClusterIndices, leader, follo
for name, index := range orphaned {
name := printer.Colorize(conf, "red", name)
table.Entries[idx] = []any{name, *index.DatasetSize, *index.DocsCount}
table.Entries[idx] = []string{name, *index.DatasetSize, *index.DocsCount}
idx++
}
table.Sort()
if err := table.Print(); err != nil {
fmt.Println(err)
return false
}
@@ -317,15 +305,13 @@ func findFailedFollowerIndices(conf *cfg.Config, indices ClusterIndices, followe
for name, index := range red {
name := printer.Colorize(conf, "red", name)
table.Entries[idx] = []any{name, *index.DatasetSize, *index.DocsCount}
table.Entries[idx] = []string{name, *index.DatasetSize, *index.DocsCount}
idx++
}
table.Sort()
if err := table.Print(); err != nil {
fmt.Println(err)
return false
}
@@ -344,3 +330,44 @@ func splitArg(arg string) (string, string) {
return parts[0], parts[1]
}
}
// recursively traverse the raw settings hash and build a flat map
// consisting of the translated path and its value.
//
// e.g.
// logger:
//
// org:
// elasticsearch:
// transport:
// OutboundHandler: "ERROR"
//
// gets:
//
// logger.org.elasticsearch.transport.OutboundHandler: "ERROR"
func getJsonPath(paths map[string]string, raw map[string]any, topic string) map[string]string {
for name, data := range raw {
path := topic + "." + name
switch value := data.(type) {
case string:
paths[path] = value
case *string:
paths[path] = *value
case int:
paths[path] = strconv.Itoa(value)
case *int:
paths[path] = strconv.Itoa(*value)
case map[string]any:
paths = getJsonPath(paths, value, path)
case []any:
val := []string{}
for _, item := range value {
val = append(val, fmt.Sprintf("%v", item))
}
paths[path] = strings.Join(val, ",")
}
}
return paths
}

92
pkg/es/datastream.go
View File

@@ -25,6 +25,7 @@ import (
"codeberg.org/scip/esctl/pkg/cfg"
"codeberg.org/scip/esctl/pkg/printer"
"github.com/dustin/go-humanize"
"github.com/elastic/go-elasticsearch/v9/typedapi/types"
)
@@ -32,7 +33,7 @@ func DatastreamNames(conf *cfg.Config) ([]string, error) {
res, err := conf.DefaultCluster.ES().Indices.GetDataStream().
Do(context.Background())
if err != nil {
return nil, fmt.Errorf("failed to get data streams: %w", esErrorString(err))
return nil, fmt.Errorf("failed to get data streams: %s", esErrorString(err))
}
dss := make([]string, len(res.DataStreams))
@@ -47,7 +48,7 @@ func DatastreamList(conf *cfg.Config) error {
res, err := conf.DefaultCluster.ES().Indices.GetDataStream().
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get data streams: %w", esErrorString(err))
return fmt.Errorf("failed to get data streams: %s", esErrorString(err))
}
slog.Debug("ES result", "data streams", res)
@@ -65,22 +66,21 @@ func DatastreamList(conf *cfg.Config) error {
table := printer.NewTable(conf, 7, size)
table.Addheaders("name", "ilm policy", "hidden", "system", "replicated", "generation", "timestamp field")
for idx, datastream := range list {
name := printer.Colorize(conf, datastream.Status.String(), datastream.Name)
for idx, ds := range list {
name := printer.Colorize(conf, ds.Status.String(), ds.Name)
policy := ""
if datastream.IlmPolicy != nil {
policy = *datastream.IlmPolicy
if ds.IlmPolicy != nil {
policy = *ds.IlmPolicy
}
table.Entries[idx] = []any{
table.Entries[idx] = []string{
name,
policy,
datastream.Hidden,
*datastream.System,
*datastream.Replicated,
datastream.Generation,
datastream.TimestampField.Name,
fmt.Sprintf("%t", ds.Hidden),
fmt.Sprintf("%t", *ds.System),
fmt.Sprintf("%t", *ds.Replicated),
fmt.Sprintf("%d", ds.Generation),
ds.TimestampField.Name,
}
if idx == size-1 {
@@ -89,7 +89,6 @@ func DatastreamList(conf *cfg.Config) error {
}
table.Sort()
if err := table.Print(); err != nil {
return err
}
@@ -130,7 +129,7 @@ func DatastreamShow(conf *cfg.Config, dsname string) error {
Name(dsname).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get data stream: %w", esErrorString(err))
return fmt.Errorf("failed to get data stream: %s", esErrorString(err))
}
slog.Debug("ES result", "data stream", res)
@@ -143,63 +142,61 @@ func DatastreamShow(conf *cfg.Config, dsname string) error {
Name(dsname).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get data stream stats: %w", esErrorString(err))
return fmt.Errorf("failed to get data stream stats: %s", esErrorString(err))
}
table := printer.NewTable(conf, 11, 2)
table.Addheaders("data stream property", "value")
datastream := res.DataStreams[0]
name := printer.Colorize(conf, datastream.Status.String(), datastream.Name)
ds := res.DataStreams[0]
name := printer.Colorize(conf, ds.Status.String(), ds.Name)
policy := ""
if datastream.IlmPolicy != nil {
policy = *datastream.IlmPolicy
if ds.IlmPolicy != nil {
policy = *ds.IlmPolicy
}
table.Entries = [][]any{
table.Entries = [][]string{
{"name", name},
{"ilm policy", policy},
{"hidden", datastream.Hidden},
{"system", *datastream.System},
{"replicated", *datastream.Replicated},
{"generation", datastream.Generation},
{"timestamp field", datastream.TimestampField.Name},
{"backing indices", stats.BackingIndices},
{"size", printer.Bytes(stats.TotalStoreSizeBytes)},
{"shards-failed", stats.Shards_.Failed},
{"shards-successful", stats.Shards_.Successful},
{"hidden", fmt.Sprintf("%t", ds.Hidden)},
{"system", fmt.Sprintf("%t", *ds.System)},
{"replicated", fmt.Sprintf("%t", *ds.Replicated)},
{"generation", fmt.Sprintf("%d", ds.Generation)},
{"timestamp field", ds.TimestampField.Name},
{"backing indices", fmt.Sprintf("%d", stats.BackingIndices)},
{"size", humanize.Bytes(uint64(stats.TotalStoreSizeBytes))},
{"shards-failed", fmt.Sprintf("%d", stats.Shards_.Failed)},
{"shards-successful", fmt.Sprintf("%d", stats.Shards_.Successful)},
}
if err := table.Print(); err != nil {
return err
}
table = printer.NewTable(conf, 5, len(datastream.Indices))
table = printer.NewTable(conf, 5, len(ds.Indices))
table.Addheaders("backing index name", "uuid", "prefer ilm", "ilm policy", "managed by")
for idx, index := range datastream.Indices {
for idx, index := range ds.Indices {
policy := ""
if datastream.IlmPolicy != nil {
policy = *datastream.IlmPolicy
if ds.IlmPolicy != nil {
policy = *ds.IlmPolicy
}
table.Entries[idx] = []any{
table.Entries[idx] = []string{
index.IndexName,
index.IndexUuid,
*index.PreferIlm,
fmt.Sprintf("%t", *index.PreferIlm),
policy,
index.ManagedBy.Name,
}
if idx < len(datastream.Indices) {
if idx < len(ds.Indices) {
fmt.Println()
}
}
table.Sort()
if err := table.Print(); err != nil {
return err
}
@@ -210,8 +207,9 @@ func DatastreamShow(conf *cfg.Config, dsname string) error {
func DatastreamCreate(conf *cfg.Config, dsname string) error {
_, err := conf.DefaultCluster.ES().Indices.CreateDataStream(dsname).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to create datastream: %w", esErrorString(err))
return fmt.Errorf("failed to create datastream: %s", esErrorString(err))
}
return nil
@@ -220,8 +218,9 @@ func DatastreamCreate(conf *cfg.Config, dsname string) error {
func DatastreamDelete(conf *cfg.Config, dsname string) error {
_, err := conf.DefaultCluster.ES().Indices.DeleteDataStream(dsname).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to delete datastream: %w", esErrorString(err))
return fmt.Errorf("failed to delete datastream: %s", esErrorString(err))
}
return nil
@@ -229,16 +228,17 @@ func DatastreamDelete(conf *cfg.Config, dsname string) error {
func DatastreamRollover(conf *cfg.Config, ds string) error {
res, err := RolloverAlias(conf, ds)
if err != nil {
return fmt.Errorf("failed to rollover data stream: %w", esErrorString(err))
return fmt.Errorf("failed to rollover data stream: %s", esErrorString(err))
}
table := printer.NewTable(conf, 2, 5)
table.Addheaders("rollover response", "value")
table.Entries = [][]any{
{"acknowledged", res.Acknowledged},
{"rolled over", res.RolledOver},
{"shards acknowledged", res.ShardsAcknowledged},
table.Entries = [][]string{
{"acknowledged", fmt.Sprintf("%t", res.Acknowledged)},
{"rolled over", fmt.Sprintf("%t", res.RolledOver)},
{"shards acknowledged", fmt.Sprintf("%t", res.ShardsAcknowledged)},
{"old index", res.OldIndex},
{"new index", res.NewIndex},
}

13
pkg/es/doc.go
View File

@@ -22,7 +22,6 @@ import (
"errors"
"fmt"
"math/rand/v2"
"strconv"
"strings"
"codeberg.org/scip/esctl/pkg/cfg"
@@ -43,16 +42,16 @@ func DocAdd(conf *cfg.Config, jsondoc string) error {
err := json.Unmarshal([]byte(jsondoc), &data)
if err != nil {
return fmt.Errorf("supplied document was not valid JSON: %w", err)
return fmt.Errorf("supplied document was not valid JSON: %s", err)
}
now := strconv.FormatInt(rand.Int64(), 10)
now := fmt.Sprintf("%d", rand.Int64())
res, err := conf.DefaultCluster.ES().Create(conf.Index, now).
Document(data).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to create new doc in index %s: %w", conf.Index, esErrorString(err))
return fmt.Errorf("failed to create new doc in index %s: %s", conf.Index, esErrorString(err))
}
fmt.Println(res.Id_)
@@ -64,7 +63,7 @@ func DocShow(conf *cfg.Config, id string) error {
res, err := conf.DefaultCluster.ES().Get(conf.Index, id).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to retrieve doc in index %s: %w", conf.Index, esErrorString(err))
return fmt.Errorf("failed to retrieve doc in index %s: %s", conf.Index, esErrorString(err))
}
if !res.Found {
@@ -95,7 +94,7 @@ func DocDelete(conf *cfg.Config, queries []string) error {
_, err := conf.DefaultCluster.ES().Delete(conf.Index, id).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to delete doc in index %s: %w", conf.Index, esErrorString(err))
return fmt.Errorf("failed to delete doc in index %s: %s", conf.Index, esErrorString(err))
}
return nil
@@ -118,7 +117,7 @@ func DocDelete(conf *cfg.Config, queries []string) error {
Request(req).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to delete docs in index %s: %w", conf.Index, esErrorString(err))
return fmt.Errorf("failed to delete docs in index %s: %s", conf.Index, esErrorString(err))
}
return nil

3
pkg/es/duration.go
View File

@@ -45,6 +45,7 @@ Convert a duration into an integer. Valid time units are "s", "m", "h" and
via https://codeberg.org/scip/tablizer/src/branch/main/lib/sort.go#L113
*/
func parseDuration(duration string) time.Duration {
seconds := 0
for _, match := range matchDuration.FindAllStringSubmatch(duration, -1) {
@@ -110,14 +111,12 @@ func formatDuration(val time.Duration) string {
if esd.minutes == 0 {
return fmt.Sprintf("%dh", esd.hours)
}
return fmt.Sprintf("%dh:%dm", esd.hours, int64(esd.minutes))
default:
if esd.minutes == 0 && hoursLeft == 0 {
return fmt.Sprintf("%dd", int64(days))
}
return fmt.Sprintf("%dd:%dh:%dm",
int64(days),
int64(hoursLeft),

24
pkg/es/errors.go
View File

@@ -17,31 +17,29 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package es
import (
"errors"
"fmt"
"strings"
"github.com/elastic/go-elasticsearch/v9/typedapi/types"
)
func esErrorString(err error) error {
func esErrorString(err error) string {
msg := err.Error()
//nolint:gocritic
switch errVal := err.(type) {
switch e := err.(type) {
case *types.ElasticsearchError:
var causes strings.Builder
causes := ""
for _, cause := range errVal.ErrorCause.RootCause {
fmt.Fprintf(&causes, "%s\n", *cause.Reason)
for _, cause := range e.ErrorCause.RootCause {
causes += fmt.Sprintf("%s\n", *cause.Reason)
}
if errVal.ErrorCause.Reason != nil {
msg = *errVal.ErrorCause.Reason + ": " + causes.String()
if e.ErrorCause.Reason != nil {
msg = *e.ErrorCause.Reason + ": " + causes
} else {
msg = fmt.Sprintf("http status %d: ", errVal.Status)
}
msg = fmt.Sprintf("http status %d: ", e.Status)
}
return errors.New(msg)
}
return msg
}

56
pkg/es/ilm.go
View File

@@ -37,7 +37,7 @@ func IlmRetry(conf *cfg.Config, index string) error {
_, err := conf.DefaultCluster.ES().Ilm.Retry(index).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to retry ilm: %w", esErrorString(err))
return fmt.Errorf("failed to retry ilm: %s", esErrorString(err))
}
return nil
@@ -47,7 +47,7 @@ func IlmStatus(conf *cfg.Config) error {
res, err := conf.DefaultCluster.ES().Ilm.GetStatus().
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get ilm status: %w", esErrorString(err))
return fmt.Errorf("failed to get ilm status: %s", esErrorString(err))
}
fmt.Println(res.OperationMode.Name)
@@ -59,7 +59,7 @@ func IlmNames(conf *cfg.Config) ([]string, error) {
res, err := conf.DefaultCluster.ES().Ilm.GetLifecycle().
Do(context.Background())
if err != nil {
return nil, fmt.Errorf("failed to get ilm policies: %w", esErrorString(err))
return nil, fmt.Errorf("failed to get ilm policies: %s", esErrorString(err))
}
names := make([]string, len(res))
@@ -67,7 +67,6 @@ func IlmNames(conf *cfg.Config) ([]string, error) {
for name := range res {
names[idx] = name
idx++
}
return names, nil
@@ -82,7 +81,7 @@ func IlmList(conf *cfg.Config, pattern string) error {
res, err := ilm.Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get ilm policies: %w", esErrorString(err))
return fmt.Errorf("failed to get ilm policies: %s", esErrorString(err))
}
if conf.Debug {
@@ -102,7 +101,6 @@ func IlmList(conf *cfg.Config, pattern string) error {
}
table.Sort()
return table.Print()
}
@@ -111,7 +109,7 @@ func IlmShow(conf *cfg.Config, policy string) error {
Policy(policy).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get ilm status: %w", esErrorString(err))
return fmt.Errorf("failed to get ilm status: %s", esErrorString(err))
}
if conf.Debug {
@@ -130,7 +128,7 @@ func IlmShow(conf *cfg.Config, policy string) error {
table := printer.NewTable(conf, 2, 5)
table.Addheaders("ilm policy setting", "value")
table.Entries = [][]any{
table.Entries = [][]string{
{"policy", policy},
{"hot phase", ilmPhaseString(ilm.Policy.Phases.Hot, false)},
{"warm phase", ilmPhaseString(ilm.Policy.Phases.Warm, false)},
@@ -167,7 +165,6 @@ func IlmShowTree(conf *cfg.Config, ilm types.IlmPolicy) error {
}
indent += " "
continue
}
@@ -262,7 +259,7 @@ func ilmPhaseString(phase *types.Phase, short bool) string {
}
if phase.Actions.SearchableSnapshot != nil {
out = append(out, "searchable-snapshot:"+phase.Actions.SearchableSnapshot.SnapshotRepository)
out = append(out, fmt.Sprintf("searchable-snapshot:%s", phase.Actions.SearchableSnapshot.SnapshotRepository))
}
return strings.Join(out, ",")
@@ -272,7 +269,7 @@ func IlmExplain(conf *cfg.Config, index string) error {
res, err := conf.DefaultCluster.ES().Ilm.ExplainLifecycle(index).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get ilm state: %w", esErrorString(err))
return fmt.Errorf("failed to get ilm state: %s", esErrorString(err))
}
slog.Debug("ilm status", "ilm", res)
@@ -288,23 +285,22 @@ func IlmExplain(conf *cfg.Config, index string) error {
table.Addheaders("ilm status field", "value")
info := ""
err = json.Unmarshal(ilm.StepInfo["reason"], &info)
if err != nil {
return fmt.Errorf("failed to unmarshal step info: %w", err)
}
table.Entries = [][]any{
table.Entries = [][]string{
{"index", ilm.Index},
{"ilm policy", *ilm.Policy},
{"action", *ilm.Action},
{"age", ilm.Age},
{"managed", ilm.Managed},
{"age", fmt.Sprintf("%s", ilm.Age)},
{"managed", fmt.Sprintf("%t", ilm.Managed)},
{"phase", *ilm.Phase},
{"phase execution", ilmPhaseString(ilm.PhaseExecution.PhaseDefinition, false)},
{"step", *ilm.Step},
{"failed step", *ilm.FailedStep},
{"failed step retry count", ilm.FailedStepRetryCount},
{"failed step retry count", fmt.Sprintf("%d", ilm.FailedStepRetryCount)},
}
if err := table.Print(); err != nil {
@@ -348,9 +344,7 @@ func IlmCreate(conf *cfg.Config, policyname string) error {
if cfg.HaveHot() {
hot := types.Phase{}
var actions types.IlmActionsVariant = esdsl.NewIlmActions()
rollover := &types.RolloverAction{}
haveroll := false
@@ -388,14 +382,15 @@ func IlmCreate(conf *cfg.Config, policyname string) error {
hot.Actions = actions.IlmActionsCaster()
phases.PhasesCaster().Hot = &hot
} else if policy != nil {
} else {
if policy != nil {
// update
phases.PhasesCaster().Hot = policy.Phases.Hot
}
}
if cfg.HaveWarm() {
warm := types.Phase{}
var actions types.IlmActionsVariant = esdsl.NewIlmActions()
if policy != nil {
@@ -421,14 +416,15 @@ func IlmCreate(conf *cfg.Config, policyname string) error {
warm.Actions = actions.IlmActionsCaster()
phases.PhasesCaster().Warm = &warm
} else if policy != nil && policy.Phases.Warm != nil {
} else {
if policy != nil && policy.Phases.Warm != nil {
// update
phases.PhasesCaster().Warm = policy.Phases.Warm
}
}
if cfg.HaveCold() {
cold := types.Phase{}
var actions types.IlmActionsVariant = esdsl.NewIlmActions()
if policy != nil {
@@ -455,14 +451,15 @@ func IlmCreate(conf *cfg.Config, policyname string) error {
cold.Actions = actions.IlmActionsCaster()
phases.PhasesCaster().Cold = &cold
} else if policy != nil && policy.Phases.Cold != nil {
} else {
if policy != nil && policy.Phases.Cold != nil {
// update
phases.PhasesCaster().Cold = policy.Phases.Cold
}
}
if cfg.HaveFrozen() {
froze := types.Phase{}
var actions types.IlmActionsVariant = esdsl.NewIlmActions()
if policy != nil {
@@ -481,15 +478,16 @@ func IlmCreate(conf *cfg.Config, policyname string) error {
froze.Actions = actions.IlmActionsCaster()
phases.PhasesCaster().Frozen = &froze
} else if policy != nil && policy.Phases.Frozen != nil {
} else {
if policy != nil && policy.Phases.Frozen != nil {
// update
phases.PhasesCaster().Frozen = policy.Phases.Frozen
}
}
if cfg.HaveDelete() {
del := types.Phase{}
delete := types.DeleteAction{}
var actions types.IlmActionsVariant = esdsl.NewIlmActions()
if policy != nil {
@@ -508,10 +506,12 @@ func IlmCreate(conf *cfg.Config, policyname string) error {
actions.IlmActionsCaster().Delete = &delete
del.Actions = actions.IlmActionsCaster()
phases.PhasesCaster().Delete = &del
} else if policy != nil && policy.Phases.Delete != nil {
} else {
if policy != nil && policy.Phases.Delete != nil {
// update
phases.PhasesCaster().Delete = policy.Phases.Delete
}
}
put := &putlifecycle.Request{}
newpolicy := &types.IlmPolicy{}
@@ -522,7 +522,7 @@ func IlmCreate(conf *cfg.Config, policyname string) error {
_, err = ilm.Do(context.Background())
if err != nil {
return fmt.Errorf("failed create ilm policy: %w", esErrorString(err))
return fmt.Errorf("failed create ilm policy: %s", esErrorString(err))
}
return nil

38
pkg/es/ilm_forecast.go
View File

@@ -119,13 +119,13 @@ func IlmForecastList(conf *cfg.Config, filter string) error {
virtualAge := virtualAge(&phase)
row := []any{
row := []string{
phase.index,
printer.Bytes(phase.size),
humanize.Bytes(uint64(phase.size)),
formatDuration(phase.age),
formatDuration(virtualAge),
formatDuration(phase.minage),
printer.Bytes(phase.minsize),
humanize.Bytes(uint64(phase.minsize)),
phase.currentPhase,
phase.nextPhase,
}
@@ -138,7 +138,6 @@ func IlmForecastList(conf *cfg.Config, filter string) error {
}
table.Sort()
return table.Print()
}
@@ -155,11 +154,14 @@ func IlmForecastShow(conf *cfg.Config) error {
}
within := parseDuration(conf.Ilm.Within)
var toBeFreed int64 = 0
for _, phase := range phaseData {
age := max(virtualAge(&phase), phase.age)
age := virtualAge(&phase)
if age < phase.age {
age = phase.age
}
if age+within >= phase.minage {
toBeFreed += phase.size
@@ -194,26 +196,24 @@ func getIlmPhaseData(conf *cfg.Config) ([]PhaseData, error) {
wg.Wait()
var (
ilmdetails *explainlifecycle.Response
indicesres *indices.Response
ilmpolicies getlifecycle.Response
)
var ilmdetails *explainlifecycle.Response
var indicesres *indices.Response
var ilmpolicies getlifecycle.Response
for range 3 {
res := <-responses
for i := 0; i < 3; i++ {
r := <-responses
if res.error != nil {
return nil, res.error
if r.error != nil {
return nil, r.error
}
switch res.which {
switch r.which {
case ResponseExplain:
ilmdetails = res.explainlifecycle
ilmdetails = r.explainlifecycle
case ResponseIndices:
indicesres = res.indicesbytes
indicesres = r.indicesbytes
case ResponseLifecycle:
ilmpolicies = *res.lifecycle
ilmpolicies = *r.lifecycle
}
}

177
pkg/es/index.go
View File

@@ -18,8 +18,6 @@ package es
import (
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
"regexp"
@@ -30,7 +28,6 @@ import (
"codeberg.org/scip/esctl/pkg/cfg"
"codeberg.org/scip/esctl/pkg/printer"
"github.com/charmbracelet/lipgloss"
"github.com/elastic/go-elasticsearch/v9/typedapi/cat/indices"
"github.com/elastic/go-elasticsearch/v9/typedapi/esdsl"
"github.com/elastic/go-elasticsearch/v9/typedapi/types/enums/healthstatus"
@@ -40,8 +37,9 @@ import (
func IndexNames(conf *cfg.Config) ([]string, error) {
res, err := conf.DefaultCluster.ES().Cat.Indices().
Do(context.Background())
if err != nil {
return nil, fmt.Errorf("failed to get indicies: %w", esErrorString(err))
return nil, fmt.Errorf("failed to get indicies: %s", esErrorString(err))
}
indices := make([]string, len(res))
@@ -99,7 +97,7 @@ func IndexList(conf *cfg.Config) error {
res, err := cat.Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get indicies: %w", esErrorString(err))
return fmt.Errorf("failed to get indicies: %s", esErrorString(err))
}
slog.Debug("ES result", "indicies", res)
@@ -120,7 +118,7 @@ func IndexList(conf *cfg.Config) error {
for idx, index := range list {
name := printer.Colorize(conf, *index.Health, *index.Index)
table.Entries[idx] = []any{name, *index.DatasetSize, *index.DocsCount}
table.Entries[idx] = []string{name, *index.DatasetSize, *index.DocsCount}
if idx == size-1 {
break
@@ -128,7 +126,6 @@ func IndexList(conf *cfg.Config) error {
}
table.Sort()
return table.Print()
}
@@ -136,14 +133,13 @@ func IndexShow(conf *cfg.Config, indexpattern string) error {
res, err := conf.DefaultCluster.ES().Indices.Get(indexpattern).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get index: %w", esErrorString(err))
return fmt.Errorf("failed to get index: %s", esErrorString(err))
}
slog.Debug("index show", "index", res)
for name, index := range res {
fields := make([]string, len(index.Mappings.Properties))
idx := 0
for field := range index.Mappings.Properties {
fields[idx] = field
@@ -160,18 +156,18 @@ func IndexShow(conf *cfg.Config, indexpattern string) error {
created := time.Unix(ts/1000, 0)
table.Entries = [][]any{
table.Entries = [][]string{
{"name", name},
{"replicas", *index.Settings.Index.NumberOfReplicas},
{"shards", *index.Settings.Index.NumberOfShards},
{"created", created},
{"created", created.Format("2006-01-02 15:04:05")},
{"uuid", *index.Settings.Index.Uuid},
{"version", *index.Settings.Index.Version.Created},
{"fields", fields},
{"fields", strings.Join(fields, ",")},
}
if index.Settings.Index.Lifecycle != nil {
table.Entries = append(table.Entries, [][]any{
table.Entries = append(table.Entries, [][]string{
{"ilm policy", *index.Settings.Index.Lifecycle.Name},
{"ilm rollover alias", *index.Settings.Index.Lifecycle.RolloverAlias},
}...)
@@ -199,7 +195,6 @@ func IndexCreate(conf *cfg.Config, index string, mappings []string) error {
if conf.Shards > 0 {
settings.NumberOfShards(strconv.Itoa(conf.Shards))
}
if conf.Replicas > 0 {
settings.NumberOfReplicas(strconv.Itoa(conf.Replicas))
}
@@ -237,8 +232,9 @@ func IndexCreate(conf *cfg.Config, index string, mappings []string) error {
_, err := create.Settings(settings).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to create index: %w", esErrorString(err))
return fmt.Errorf("failed to create index: %s", esErrorString(err))
}
return nil
@@ -248,7 +244,7 @@ func IndexDelete(conf *cfg.Config, index string) error {
_, err := conf.DefaultCluster.ES().Indices.Delete(index).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to delete index: %w", esErrorString(err))
return fmt.Errorf("failed to delete index: %s", esErrorString(err))
}
return nil
@@ -257,27 +253,85 @@ func IndexDelete(conf *cfg.Config, index string) error {
func IndexClose(conf *cfg.Config, index string) error {
_, err := conf.DefaultCluster.ES().Indices.Close(index).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to close index: %w", esErrorString(err))
return fmt.Errorf("failed to close index: %s", esErrorString(err))
}
return nil
}
func IndexAllocation(conf *cfg.Config, index string) error {
res, err := conf.DefaultCluster.ES().Cluster.AllocationExplain().
Index(index).
Primary(conf.Primary).
Shard(conf.Shards).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get index allocation explain: %s", esErrorString(err))
}
slog.Debug("ES result", "index", res)
currentNode := res.CurrentNode
table := printer.NewTable(conf, 2, 10)
table.Addheaders("index allocation setting", "value")
roles := make([]string, len(currentNode.Roles))
for idx, role := range currentNode.Roles {
roles[idx] = role.Name
}
table.Entries = [][]string{
{"Index", index},
{"Current node", currentNode.Name},
{"Current k8s node", currentNode.Attributes["k8s_node_name"]},
{"Current node address", currentNode.TransportAddress},
{"Current node id", currentNode.Id},
{"Current node weight", fmt.Sprintf("%d", currentNode.WeightRanking)},
{"Current node roles", strings.Join(roles, ",")},
{"Can rebalance cluster", res.CanRebalanceCluster.Name},
{"Can rebalance to another node", res.CanRebalanceToOtherNode.Name},
{"Can remain on current node", res.CanRemainOnCurrentNode.Name},
}
if err := table.Print(); err != nil {
return err
}
return nil
}
/*
func IndexModify(conf *cfg.Config, index string) error {
settings := esdsl.NewIndexSettings().NumberOfReplicas(strconv.Itoa(conf.Replicas))
_, err := conf.DefaultCluster.ES().Indices.PutSettings().
Indices(index).
Index(settings).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to modify index settings: %s", esErrorString(err))
}
return nil
}
*/
func IndexFields(conf *cfg.Config, index string) error {
res, err := conf.DefaultCluster.ES().FieldCaps().
Index(index).
Fields("*").
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to retrieve field capabilties: %w", esErrorString(err))
return fmt.Errorf("failed to retrieve field capabilties: %s", esErrorString(err))
}
table := printer.NewTable(conf, 5, 0)
table.Addheaders("field", "type", "searchable", "aggretable", "metadata")
idx := 0
for name, field := range res.Fields {
for fieldtype, caps := range field {
// fields only have 1 type, so this one is it
@@ -290,11 +344,10 @@ func IndexFields(conf *cfg.Config, index string) error {
continue
}
table.AddRow(name, fieldtype,
caps.Searchable,
caps.Aggregatable,
*caps.MetadataField)
table.Entries = append(table.Entries, []string{name, fieldtype,
fmt.Sprintf("%t", caps.Searchable),
fmt.Sprintf("%t", caps.Aggregatable),
fmt.Sprintf("%t", *caps.MetadataField)})
break
}
@@ -309,79 +362,3 @@ func IndexFields(conf *cfg.Config, index string) error {
return nil
}
type Diskusage struct {
Total int64 `json:"total_in_bytes"`
Points int64 `json:"points_in_bytes"`
Norms int64 `json:"norms_in_bytes"`
TermVectors int64 `json:"term_vectors_in_bytes"`
KnnVectors int64 `json:"knn_vectors_in_bytes"`
BloomFilter int64 `json:"bloom_filter_in_bytes"`
}
type IndexDiskUsage struct {
AllFields Diskusage `json:"all_fields"`
Fields map[string]Diskusage `json:"fields"`
}
type ResIndexDiskUsage map[string]IndexDiskUsage
func IndexDiskusage(conf *cfg.Config, index string) error {
var bold = lipgloss.NewStyle().Bold(true)
res, err := conf.DefaultCluster.ES().Indices.DiskUsage(index).
RunExpensiveTasks(true).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to retrieve index disk usage: %w", esErrorString(err))
}
duRes := ResIndexDiskUsage{}
if err := json.Unmarshal(res, &duRes); err != nil {
return fmt.Errorf("failed to unmarshal disk usage response: %w", err)
}
diskusage, exists := duRes[index]
if !exists {
return errors.New("no disk usage reported for index")
}
table := printer.NewTableEmpty(conf).
WithHeaders("field", "bloom filter", "norms", "points", "term vectors", "knn vectors", "total")
for name, field := range diskusage.Fields {
if strings.HasPrefix(name, "_") || strings.HasSuffix(name, ".keyword") {
continue
}
table.AddRow(
name,
printer.Bytes(field.BloomFilter),
printer.Bytes(field.Norms),
printer.Bytes(field.Points),
printer.Bytes(field.TermVectors),
printer.Bytes(field.KnnVectors),
printer.Bytes(field.Total),
)
}
all := diskusage.AllFields
table.Sort()
table.AddRowLate(
bold.Render("Summary"),
printer.Bytes(all.BloomFilter),
printer.Bytes(all.Norms),
printer.Bytes(all.Points),
printer.Bytes(all.TermVectors),
printer.Bytes(all.KnnVectors),
printer.Bytes(all.Total),
)
if err := table.Print(); err != nil {
return err
}
return nil
}

20
pkg/es/index_alias.go
View File

@@ -35,7 +35,7 @@ func IndexAliasCreate(conf *cfg.Config, index, alias string) error {
slog.Debug("create alias", "result", res)
if err != nil {
return fmt.Errorf("failed to create index alias: %w", esErrorString(err))
return fmt.Errorf("failed to create index alias: %s", esErrorString(err))
}
return nil
@@ -52,8 +52,9 @@ func IndexAliasList(conf *cfg.Config) error {
res, err := conf.DefaultCluster.ES().Indices.GetAlias().
Index("_all").
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to list index aliases: %w", esErrorString(err))
return fmt.Errorf("failed to list index aliases: %s", esErrorString(err))
}
slog.Debug("aliases list", "result", res)
@@ -77,7 +78,7 @@ func IndexAliasList(conf *cfg.Config) error {
idx := 0
for index, aliases := range aliaslist {
table.Entries[idx] = []any{
table.Entries[idx] = []string{
index,
strings.Join(aliases, ","),
}
@@ -101,7 +102,7 @@ func IndexAliasDelete(conf *cfg.Config, index, alias string) error {
slog.Debug("delete alias", "result", res)
if err != nil {
return fmt.Errorf("failed to delete index alias: %w", esErrorString(err))
return fmt.Errorf("failed to delete index alias: %s", esErrorString(err))
}
return nil
@@ -109,16 +110,17 @@ func IndexAliasDelete(conf *cfg.Config, index, alias string) error {
func IndexAliasRollover(conf *cfg.Config, alias string) error {
res, err := RolloverAlias(conf, alias)
if err != nil {
return fmt.Errorf("failed to rollover index alias: %w", esErrorString(err))
return fmt.Errorf("failed to rollover index alias: %s", esErrorString(err))
}
table := printer.NewTable(conf, 2, 5)
table.Addheaders("rollover response", "value")
table.Entries = [][]any{
{"acknowledged", res.Acknowledged},
{"rolled over", res.RolledOver},
{"shards acknowledged", res.ShardsAcknowledged},
table.Entries = [][]string{
{"acknowledged", fmt.Sprintf("%t", res.Acknowledged)},
{"rolled over", fmt.Sprintf("%t", res.RolledOver)},
{"shards acknowledged", fmt.Sprintf("%t", res.ShardsAcknowledged)},
{"old index", res.OldIndex},
{"new index", res.NewIndex},
}

62
pkg/es/index_template.go
View File

@@ -22,7 +22,6 @@ import (
"errors"
"fmt"
"log/slog"
"maps"
"strconv"
"strings"
@@ -36,8 +35,9 @@ import (
func IndexTemplateList(conf *cfg.Config) error {
res, err := conf.DefaultCluster.ES().Indices.GetIndexTemplate().
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get index templates: %w", esErrorString(err))
return fmt.Errorf("failed to get index templates: %s", esErrorString(err))
}
slog.Debug("res", "index templates", res)
@@ -51,11 +51,14 @@ func IndexTemplateList(conf *cfg.Config) error {
return fmt.Errorf("failed to unmarshal meta json data: %w", err)
}
table.Entries[idx] = []any{tpl.Name, desc, tpl.IndexTemplate.Priority}
table.Entries[idx] = []string{
tpl.Name,
string(desc),
fmt.Sprintf("%d", tpl.IndexTemplate.Priority),
}
}
table.Sort()
return table.Print()
}
@@ -63,8 +66,9 @@ func IndexTemplateShow(conf *cfg.Config, tplname string) error {
res, err := conf.DefaultCluster.ES().Indices.GetIndexTemplate().
Name(tplname).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get index template: %w", esErrorString(err))
return fmt.Errorf("failed to get index template: %s", esErrorString(err))
}
slog.Debug("res", "index template", res)
@@ -90,13 +94,13 @@ func IndexTemplateShow(conf *cfg.Config, tplname string) error {
aliases = append(aliases, alias)
}
table.Entries = [][]any{
table.Entries = [][]string{
{"name", tpl.Name},
{"description", desc},
{"index patterns", tpl.IndexTemplate.IndexPatterns},
{"composed of", tpl.IndexTemplate.ComposedOf},
{"data stream enabled", hasds},
{"aliases", aliases},
{"description", string(desc)},
{"index patterns", strings.Join(tpl.IndexTemplate.IndexPatterns, ",")},
{"composed of", strings.Join(tpl.IndexTemplate.ComposedOf, ",")},
{"data stream enabled", fmt.Sprintf("%t", hasds)},
{"aliases", strings.Join(aliases, ",")},
}
if err := table.Print(); err != nil {
@@ -108,11 +112,10 @@ func IndexTemplateShow(conf *cfg.Config, tplname string) error {
err = getIndexTemplateSettings(conf, tplname, table)
if err != nil {
return err
return nil
}
fmt.Println()
if err := table.Print(); err != nil {
return err
}
@@ -133,11 +136,12 @@ func IndexTemplateShow(conf *cfg.Config, tplname string) error {
typeval = val.Type
}
table.AddRow(name, typeval)
table.Entries = append(table.Entries, []string{
name, typeval,
})
}
fmt.Println()
if err := table.Print(); err != nil {
return err
}
@@ -225,8 +229,9 @@ func IndexTemplateCreate(conf *cfg.Config, name string, mappings []string) error
create.IndexPatterns(conf.Patterns...)
_, err := create.Do(context.Background())
if err != nil {
return fmt.Errorf("failed to create index template: %w", esErrorString(err))
return fmt.Errorf("failed to create index template: %s", esErrorString(err))
}
return nil
@@ -241,8 +246,9 @@ func IndexTemplateModify(conf *cfg.Config, name string, mappings []string) error
res, err := conf.DefaultCluster.ES().Indices.GetIndexTemplate().
Name(name).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get index template: %w", esErrorString(err))
return fmt.Errorf("failed to get index template: %s", esErrorString(err))
}
slog.Debug("res", "index template", res)
@@ -334,8 +340,9 @@ func IndexTemplateModify(conf *cfg.Config, name string, mappings []string) error
modify.IndexPatterns(patterns...)
_, err = modify.Do(context.Background())
if err != nil {
return fmt.Errorf("failed to modify index template: %w", esErrorString(err))
return fmt.Errorf("failed to modify index template: %s", esErrorString(err))
}
if conf.Rollover {
@@ -351,7 +358,7 @@ func IndexTemplateDelete(conf *cfg.Config, name string) error {
_, err := conf.DefaultCluster.ES().Indices.DeleteIndexTemplate(name).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to delete index template: %w", esErrorString(err))
return fmt.Errorf("failed to delete index template: %s", esErrorString(err))
}
return nil
@@ -364,8 +371,9 @@ func rolloverAliasIndexTemplate(conf *cfg.Config, name string) error {
res, err := conf.DefaultCluster.ES().Indices.GetIndexTemplate().
Name(name).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get index template: %w", esErrorString(err))
return fmt.Errorf("failed to get index template: %s", esErrorString(err))
}
slog.Debug("res", "index template", res)
@@ -382,7 +390,7 @@ func rolloverAliasIndexTemplate(conf *cfg.Config, name string) error {
res, err := conf.DefaultCluster.ES().Indices.ResolveIndex(pattern).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to resolve index pattern: %w", esErrorString(err))
return fmt.Errorf("failed to resolve index pattern: %s", esErrorString(err))
}
for _, index := range res.Indices {
@@ -406,7 +414,12 @@ func rolloverAliasIndexTemplate(conf *cfg.Config, name string) error {
return err
}
table.AddRow(alias, res.RolledOver, res.Acknowledged, res.NewIndex)
table.Entries = append(table.Entries, []string{
alias,
fmt.Sprintf("%t", res.RolledOver),
fmt.Sprintf("%t", res.Acknowledged),
res.NewIndex,
})
if err := table.Print(); err != nil {
return err
@@ -414,6 +427,7 @@ func rolloverAliasIndexTemplate(conf *cfg.Config, name string) error {
}
return nil
}
func modMappings(mappings []string) (types.TypeMappingVariant, error) {
@@ -445,7 +459,9 @@ func modMappings(mappings []string) (types.TypeMappingVariant, error) {
func modMeta(conf *cfg.Config, meta types.Metadata) (map[string]json.RawMessage, error) {
metadata := map[string]json.RawMessage{}
maps.Copy(metadata, meta)
for key, value := range meta {
metadata[key] = value
}
for _, meta := range conf.Meta {
parts := strings.Split(meta, ":")

13
pkg/es/index_template_settings.go
View File

@@ -28,7 +28,7 @@ type Tpl struct {
}
func getIndexTemplateSettings(conf *cfg.Config, tplname string, table *printer.Table) error {
raw, err := CallAPI(conf, "GET", "/_index_template/"+tplname+"?flat_settings", "")
raw, err := CallAPI(conf, "GET", "/_index_template/"+tplname, "")
if err != nil {
return err
}
@@ -39,12 +39,9 @@ func getIndexTemplateSettings(conf *cfg.Config, tplname string, table *printer.T
}
if conf.Debug {
output, err := prettyfiJson(conf, raw)
if err != nil {
if err := prettyfiJson(conf, raw); err != nil {
return err
}
fmt.Println(output)
}
if len(data.IndexTemplates) == 0 {
@@ -53,7 +50,11 @@ func getIndexTemplateSettings(conf *cfg.Config, tplname string, table *printer.T
tpl := data.IndexTemplates[0].IndexTemplate.Template.Settings
for topic, val := range tpl {
table.AddRow(topic, val)
paths := getJsonPath(map[string]string{}, val.(map[string]any), topic)
for setting, value := range paths {
table.Entries = append(table.Entries, []string{setting, fmt.Sprintf("%v", value)})
}
}
return nil

72
pkg/es/license.go
View File

@@ -1,72 +0,0 @@
/*
Copyright © 2026 Thomas von Dein
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package es
import (
"context"
"fmt"
"log/slog"
"strconv"
"codeberg.org/scip/esctl/pkg/cfg"
"codeberg.org/scip/esctl/pkg/printer"
)
func LicenseShow(conf *cfg.Config) error {
res, err := conf.DefaultCluster.ES().License.Get().
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get license: %w", esErrorString(err))
}
slog.Debug("license show", "license", res)
table := printer.NewTableEmpty(conf).WithHeaders("license setting", "value")
lic := res.License
var (
maxnodes = "infinite"
maxunits = "infinite"
expire = "never"
)
if lic.MaxNodes != nil {
maxnodes = strconv.FormatInt(*lic.MaxNodes, 10)
}
if lic.ExpiryDate != nil {
expire = lic.ExpiryDate.(string)
}
if lic.MaxResourceUnits != nil {
maxunits = strconv.Itoa(*lic.MaxResourceUnits)
}
table.Entries = [][]any{
{"UID", lic.Uid},
{"Issued to", lic.IssuedTo},
{"Expires", expire},
{"Issued", lic.IssueDate},
{"Max nodes", maxnodes},
{"Max resource units", maxunits},
{"Type", lic.Type.Name},
{"Status", lic.Status.Name},
}
return table.Print()
}

159
pkg/es/node.go
View File

@@ -20,40 +20,39 @@ import (
"context"
"fmt"
"log/slog"
"strconv"
"strings"
"time"
"codeberg.org/scip/esctl/pkg/cfg"
"codeberg.org/scip/esctl/pkg/printer"
"github.com/dustin/go-humanize"
)
func NodeList(conf *cfg.Config) error {
// get nodes
nodes, err := conf.DefaultCluster.ES().Cat.Nodes().Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get nodes: %w", esErrorString(err))
return fmt.Errorf("failed to get nodes: %s", esErrorString(err))
}
slog.Debug("ES result", "nodes", nodes)
table := printer.NewTableEmpty(conf).WithHeaders(
"name", "ip", "load1m", "load5m", "load15m", "ram %", "heap %")
table := printer.NewTable(conf, 7, len(nodes))
table.Addheaders("name", "ip", "load1m", "load5m", "load15m", "ram %", "heap %")
for _, node := range nodes {
table.AddRow(
for idx, node := range nodes {
table.Entries[idx] = []string{
*node.Name,
*node.Ip,
*node.Load1M,
*node.Load5M,
*node.Load15M,
node.RamPercent,
node.HeapPercent,
)
node.RamPercent.(string),
node.HeapPercent.(string),
}
}
table.Sort()
if err := table.Print(); err != nil {
return err
}
@@ -65,7 +64,7 @@ func NodeNames(conf *cfg.Config) ([]string, error) {
nodes, err := conf.DefaultCluster.ES().Cat.Nodes().
Do(context.Background())
if err != nil {
return nil, fmt.Errorf("failed to get nodes: %w", esErrorString(err))
return nil, fmt.Errorf("failed to get nodes: %s", esErrorString(err))
}
slog.Debug("ES result", "nodes", nodes)
@@ -87,7 +86,7 @@ func NodeShow(conf *cfg.Config, nodename string) error {
Metric("os, jvm, thread_pool, remote_cluster_server").
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get node info: %w", esErrorString(err))
return fmt.Errorf("failed to get node info: %s", esErrorString(err))
}
slog.Debug("ES result", "node", res)
@@ -96,7 +95,7 @@ func NodeShow(conf *cfg.Config, nodename string) error {
NodeId(nodename).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get node stats: %w", esErrorString(err))
return fmt.Errorf("failed to get node stats: %s", esErrorString(err))
}
slog.Debug("ES result", "stat", stats)
@@ -113,70 +112,38 @@ func NodeShow(conf *cfg.Config, nodename string) error {
}
k8snode := info.Attributes["k8s_node_name"]
rank := "none"
adsel, exists := stat.AdaptiveSelection[id]
if exists {
rank = *adsel.Rank
}
table.Entries = [][]any{
table.Entries = [][]string{
{"Id", id},
{"Name", nodename},
{"Kubernetes node", k8snode},
{"Ip address", info.Ip},
{"Node rank", rank},
{"Node rank", *stat.AdaptiveSelection[id].Rank},
{"JVM", info.Jvm.VmName + " " + info.Jvm.Version},
{"JVM Started", time.UnixMilli(info.Jvm.StartTimeInMillis)},
{"JVM Started", time.UnixMilli(info.Jvm.StartTimeInMillis).String()},
{"OS", info.Os.PrettyName + " " + info.Os.Version},
{"Node roles", roles},
{"Node roles", strings.Join(roles, ",")},
{"Node version", info.Version},
// FIXME: not implemented upstream
// see: https://github.com/elastic/go-elasticsearch/issues/1526
// {"Allocated shards", stat.Allocations.XXX},
{"HTTP clients", *stat.Http.CurrentOpen},
{"CPUs", *info.Os.AllocatedProcessors},
{"HTTP clients", fmt.Sprintf("%d", *stat.Http.CurrentOpen)},
{"CPUs", fmt.Sprintf("%d", *info.Os.AllocatedProcessors)},
{"Load 15m/5m/1m", fmt.Sprintf("%.2f/%.2f/%.2f",
stat.Os.Cpu.LoadAverage["15m"],
stat.Os.Cpu.LoadAverage["5m"],
stat.Os.Cpu.LoadAverage["1m"],
)},
{"Open FD's", *stat.Process.OpenFileDescriptors},
{"HTTP sesssions current/total", fmt.Sprintf("%d/%d",
*stat.Http.CurrentOpen,
*stat.Http.TotalOpened,
)},
{"Traffic rx/tx",
printer.ByteString(*stat.Transport.RxSizeInBytes) + " / " + printer.ByteString(*stat.Transport.TxSizeInBytes)},
{"Response time avg", time.Duration(*stat.AdaptiveSelection[id].AvgResponseTimeNs)},
{"Open FD's", fmt.Sprintf("%d", *stat.Process.OpenFileDescriptors)},
{"Response time avg", fmt.Sprintf("%dns", *stat.AdaptiveSelection[id].AvgResponseTimeNs)},
{"Memory usage (used/avail)",
printer.ByteString(*stat.Os.Mem.UsedInBytes) + " / " + printer.ByteString(*stat.Os.Mem.TotalInBytes)},
{"Search queries current/total", fmt.Sprintf("%d/%d",
stat.Indices.Search.QueryCurrent,
stat.Indices.Search.QueryTotal,
)},
{"Search efficiency", stat.Indices.Search.QueryTimeInMillis / stat.Indices.Search.QueryTotal},
{"Docs count", stat.Indices.Docs.Count},
{"Merges current/total", fmt.Sprintf("%d/%d",
stat.Indices.Merges.Current,
stat.Indices.Merges.Total,
)},
{"Merge docs count current/total", fmt.Sprintf("%d/%d",
stat.Indices.Merges.CurrentDocs,
stat.Indices.Merges.TotalDocs,
)},
{"Merge size current/total", fmt.Sprintf("%s/%s",
printer.ByteString(stat.Indices.Merges.CurrentSizeInBytes),
printer.ByteString(stat.Indices.Merges.TotalSizeInBytes),
)},
{"CircuitBreaker trip count", *stat.Breakers["fielddata"].Tripped},
humanize.Bytes(uint64(*stat.Os.Mem.UsedInBytes)) + " / " + humanize.Bytes(uint64(*stat.Os.Mem.TotalInBytes))},
}
if len(stat.Fs.Data) > 0 {
fs := stat.Fs.Data[0]
table.AddRow("Storage usage (used/avail)",
printer.ByteString(*fs.AvailableInBytes)+" / "+printer.ByteString(*fs.TotalInBytes))
table.AddRow("Storage mount", *fs.Mount)
table.Entries = append(table.Entries, [][]string{
{"Storage usage (used/avail)",
humanize.Bytes(uint64(*fs.AvailableInBytes)) + " / " + humanize.Bytes(uint64(*fs.TotalInBytes))},
{"Storage mount", *fs.Mount},
}...)
}
if err := table.Print(); err != nil {
@@ -192,7 +159,7 @@ func NodeClients(conf *cfg.Config, nodename string) error {
NodeId(nodename).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get node stats: %w", esErrorString(err))
return fmt.Errorf("failed to get node stats: %s", esErrorString(err))
}
slog.Debug("ES result", "stat", stats)
@@ -220,7 +187,7 @@ func NodeClients(conf *cfg.Config, nodename string) error {
table.AddRow(
agent,
strconv.FormatInt(*client.Id, 10),
fmt.Sprintf("%d", *client.Id),
time.UnixMilli(*client.LastRequestTimeMillis).String(),
*client.RemoteAddress,
uri,
@@ -232,75 +199,5 @@ func NodeClients(conf *cfg.Config, nodename string) error {
}
table.Sort()
return table.Print()
}
func NodeUsage(conf *cfg.Config, nodeid string) error {
usage := conf.DefaultCluster.ES().Nodes.Usage()
if nodeid != "" {
usage.NodeId(nodeid)
}
stats, err := usage.Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get node usage: %w", esErrorString(err))
}
slog.Debug("ES result", "usage", stats)
table := printer.NewTableEmpty(conf).WithHeaders(
"node",
"bulk",
"doc get",
"doc mget",
"doc update",
"index doc",
"index stats",
"search",
"msearch",
"open pit",
)
for id, actions := range stats.Nodes {
node, err := getNodeName(conf, id)
if err != nil {
return err
}
stat := actions.RestActions
table.AddRow(
node,
stat["bulk_action"],
stat["document_get_action"],
stat["document_mget_action"],
stat["document_update_action"],
stat["document_index_action"],
stat["indices_stats_action"],
stat["search_action"],
stat["msearch_action"],
stat["open_point_in_time"],
)
}
return table.Print()
}
func getNodeName(conf *cfg.Config, id string) (string, error) {
res, err := conf.DefaultCluster.ES().Nodes.Info().
Metric("os").
Do(context.Background())
if err != nil {
return "", fmt.Errorf("failed to get node info: %w", esErrorString(err))
}
for nodeid, node := range res.Nodes {
if id == nodeid {
return node.Name, nil
}
}
return "", nil
}

47
pkg/es/parallel.go
View File

@@ -69,8 +69,7 @@ func getApiData(
which string) {
defer wg.Done()
apiRes := apiResponse{}
ar := apiResponse{}
var arerr error
switch which {
@@ -78,47 +77,47 @@ func getApiData(
res, err := es.Cluster.Health().
Do(context.Background())
apiRes.health = res
apiRes.which = ResponseHealth
ar.health = res
ar.which = ResponseHealth
arerr = err
case "healthreport":
report, err := getHealthReport(conf)
apiRes.healthreport = report
apiRes.which = ResponseHealthReport
ar.healthreport = report
ar.which = ResponseHealthReport
arerr = err
case "info":
res, err := es.Info().
Do(context.Background())
apiRes.info = res
apiRes.which = ResponseInfo
ar.info = res
ar.which = ResponseInfo
arerr = err
case "ccr":
res, err := es.Ccr.Stats().
Do(context.Background())
apiRes.ccr = res
apiRes.which = ResponseCcr
ar.ccr = res
ar.which = ResponseCcr
arerr = err
case "stats":
res, err := es.Cluster.Stats().
Do(context.Background())
apiRes.stats = res
apiRes.which = ResponseStats
ar.stats = res
ar.which = ResponseStats
arerr = err
case "indices":
res, err := es.Cat.Indices().
Do(context.Background())
apiRes.indices = &res
apiRes.which = ResponseIndices
ar.indices = &res
ar.which = ResponseIndices
arerr = err
case "indicesbytes":
@@ -127,16 +126,16 @@ func getApiData(
Bytes(bytes.Bytes{Name: "b"}).
Do(context.Background())
apiRes.indicesbytes = &res
apiRes.which = ResponseIndices
ar.indicesbytes = &res
ar.which = ResponseIndices
arerr = err
case "tasks":
res, err := es.Cat.Tasks().
Do(context.Background())
apiRes.tasks = &res
apiRes.which = ResponseTasks
ar.tasks = &res
ar.which = ResponseTasks
arerr = err
case "explain":
@@ -144,8 +143,8 @@ func getApiData(
ExplainLifecycle("_all").
Do(context.Background())
apiRes.explainlifecycle = res
apiRes.which = ResponseExplain
ar.explainlifecycle = res
ar.which = ResponseExplain
arerr = err
case "policies":
@@ -153,14 +152,14 @@ func getApiData(
GetLifecycle().
Do(context.Background())
apiRes.lifecycle = &res
apiRes.which = ResponseLifecycle
ar.lifecycle = &res
ar.which = ResponseLifecycle
arerr = err
}
if arerr != nil {
apiRes.error = fmt.Errorf("failed to get data from API: %w", arerr)
ar.error = fmt.Errorf("failed to get data from API: %s", arerr)
}
reschan <- apiRes
reschan <- ar
}

54
pkg/es/role.go
View File

@@ -20,6 +20,7 @@ import (
"context"
"fmt"
"log/slog"
"strings"
"codeberg.org/scip/esctl/pkg/cfg"
"codeberg.org/scip/esctl/pkg/printer"
@@ -30,11 +31,10 @@ func RoleNames(conf *cfg.Config) ([]string, error) {
res, err := conf.DefaultCluster.ES().Security.GetRole().
Do(context.Background())
if err != nil {
return nil, fmt.Errorf("failed to get roles: %w", esErrorString(err))
return nil, fmt.Errorf("failed to get roles: %s", esErrorString(err))
}
roles := make([]string, len(res))
idx := 0
for name := range res {
roles[idx] = name
@@ -48,7 +48,7 @@ func RoleList(conf *cfg.Config) error {
res, err := conf.DefaultCluster.ES().Security.GetRole().
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get roles: %w", esErrorString(err))
return fmt.Errorf("failed to get roles: %s", esErrorString(err))
}
slog.Debug("ES result", "roles", res)
@@ -58,16 +58,15 @@ func RoleList(conf *cfg.Config) error {
idx := 0
for name, role := range res {
table.Entries[idx] = []any{
table.Entries[idx] = []string{
name,
role.Cluster,
role.Indices,
fmt.Sprintf("%d", len(role.Cluster)),
fmt.Sprintf("%d", len(role.Indices)),
}
idx++
}
table.Sort()
return table.Print()
}
@@ -76,7 +75,7 @@ func RoleShow(conf *cfg.Config, rolename string) error {
Name(rolename).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get role: %w", esErrorString(err))
return fmt.Errorf("failed to get role: %s", esErrorString(err))
}
slog.Debug("ES result", "role", res)
@@ -134,22 +133,21 @@ func roleRemoteClusters(conf *cfg.Config, role types.Role) error {
table := printer.NewTable(conf, 2, len(role.RemoteCluster))
table.Addheaders("remote cluster", "privilege")
idx := 0
for _, priv := range role.RemoteCluster {
perms := []string{}
for _, perm := range priv.Privileges {
perms = append(perms, perm.Name)
}
table.Entries[idx] = []any{priv.Clusters, perms}
table.Entries[idx] = []string{
strings.Join(priv.Clusters, ","),
strings.Join(perms, ","),
}
idx++
}
table.Sort()
return table.Print()
}
@@ -160,15 +158,13 @@ func roleClusters(conf *cfg.Config, role types.Role) error {
table := printer.NewTable(conf, 1, len(role.Cluster))
table.Addheaders("cluster rights")
idx := 0
for _, cluster := range role.Cluster {
table.Entries[idx] = []any{cluster.Name}
table.Entries[idx] = []string{cluster.Name}
idx++
}
table.Sort()
return table.Print()
}
@@ -179,22 +175,23 @@ func roleRemoteIndices(conf *cfg.Config, role types.Role) error {
table := printer.NewTable(conf, 3, len(role.RemoteIndices))
table.Addheaders("remote index names", "index permissions", "allow restricted")
idx := 0
for _, priv := range role.RemoteIndices {
perms := []string{}
for _, perm := range priv.Privileges {
perms = append(perms, perm.Name)
}
table.Entries[idx] = []any{priv.Names, perms, *priv.AllowRestrictedIndices}
table.Entries[idx] = []string{
strings.Join(priv.Names, ", "),
strings.Join(perms, ", "),
fmt.Sprintf("%t", *priv.AllowRestrictedIndices),
}
idx++
}
table.Sort()
return table.Print()
}
@@ -205,22 +202,23 @@ func roleIndices(conf *cfg.Config, role types.Role) error {
table := printer.NewTable(conf, 3, len(role.Indices))
table.Addheaders("index names", "index permissions", "allow restricted")
idx := 0
for _, priv := range role.Indices {
perms := []string{}
for _, perm := range priv.Privileges {
perms = append(perms, perm.Name)
}
table.Entries[idx] = []any{priv.Names, perms, *priv.AllowRestrictedIndices}
table.Entries[idx] = []string{
strings.Join(priv.Names, ", "),
strings.Join(perms, ", "),
fmt.Sprintf("%t", *priv.AllowRestrictedIndices),
}
idx++
}
table.Sort()
return table.Print()
}
@@ -231,19 +229,17 @@ func roleApplications(conf *cfg.Config, role types.Role) error {
table := printer.NewTable(conf, 3, len(role.Applications))
table.Addheaders("application", "privileges", "resources")
idx := 0
for _, priv := range role.Applications {
table.Entries[idx] = []any{
table.Entries[idx] = []string{
priv.Application,
priv.Privileges,
priv.Resources,
strings.Join(priv.Privileges, ", "),
strings.Join(priv.Resources, ", "),
}
idx++
}
table.Sort()
return table.Print()
}

32
pkg/es/role_diff.go
View File

@@ -75,7 +75,7 @@ type Register struct {
func getCsvRecords(conf *cfg.Config, csvfile string) (map[string]Record, error) {
data, err := os.ReadFile(csvfile)
if err != nil {
return nil, fmt.Errorf("failed to read CSV file: %w", err)
return nil, fmt.Errorf("failed to read CSV file: %s", err)
}
csvreader := csv.NewReader(bytes.NewReader(data))
@@ -85,7 +85,7 @@ func getCsvRecords(conf *cfg.Config, csvfile string) (map[string]Record, error)
rows, err := csvreader.ReadAll()
if err != nil {
return nil, fmt.Errorf("failed to parse CSV: %w", err)
return nil, fmt.Errorf("failed to parse CSV: %s", err)
}
records := make(map[string]Record, len(rows)-1)
@@ -118,7 +118,7 @@ func getCsvRecords(conf *cfg.Config, csvfile string) (map[string]Record, error)
func getCsvRecord(conf *cfg.Config, csvfile, rolename string) (*Record, error) {
fd, err := os.Open(csvfile)
if err != nil {
return nil, fmt.Errorf("failed to open CSV file: %w", err)
return nil, fmt.Errorf("failed to open CSV file: %s", err)
}
defer func() {
if err := fd.Close(); err != nil {
@@ -179,7 +179,6 @@ func diffRoles(conf *cfg.Config, records map[string]Record, res getrole.Response
// iterate over records from CSV and register only those which are not deployed
for name := range records {
reg := Register{name: name, defined: true}
_, deployed := deployed[name]
if !deployed {
rows = append(rows, reg)
@@ -211,7 +210,7 @@ func RoleDiff(conf *cfg.Config, csvfile, role string) error {
res, err := conf.DefaultCluster.ES().Security.GetRole().
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get roles: %w", esErrorString(err))
return fmt.Errorf("failed to get roles: %s", esErrorString(err))
}
rows := diffRoles(conf, records, res)
@@ -230,7 +229,7 @@ func RoleDiff(conf *cfg.Config, csvfile, role string) error {
defined = printer.Colorize(conf, "red", "undefined")
}
table.Entries[idx] = []any{
table.Entries[idx] = []string{
row.name,
deployed,
defined,
@@ -246,11 +245,10 @@ func getRoleMappingGroups(conf *cfg.Config, rolename string) ([]string, error) {
mappings, err := conf.DefaultCluster.ES().Security.GetRoleMapping().
Do(context.Background())
if err != nil {
return nil, fmt.Errorf("failed to get role mappings: %w", esErrorString(err))
return nil, fmt.Errorf("failed to get role mappings: %s", esErrorString(err))
}
groups := []string{}
for _, mapping := range mappings {
if slices.Contains(mapping.Roles, rolename) {
for _, rule := range mapping.Rules.Any {
@@ -264,13 +262,13 @@ func getRoleMappingGroups(conf *cfg.Config, rolename string) ([]string, error) {
return groups, nil
}
func compareSlices(name string, left, right []string) {
slices.Sort(left)
slices.Sort(right)
func compareSlices(name string, a, b []string) {
slices.Sort(a)
slices.Sort(b)
if slices.Compare(left, right) != 0 {
if slices.Compare(a, b) != 0 {
fmt.Printf("%s differs:\ndeployed: %s\n csv: %s\n",
name, strings.Join(left, ","), strings.Join(right, ","))
name, strings.Join(a, ","), strings.Join(b, ","))
} else {
fmt.Printf("deployed %s matches csv definition\n", name)
}
@@ -281,7 +279,7 @@ func RoleDiffSingle(conf *cfg.Config, csvfile, rolename string) error {
Name(rolename).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get role: %w", esErrorString(err))
return fmt.Errorf("failed to get role: %s", esErrorString(err))
}
record, err := getCsvRecord(conf, csvfile, rolename)
@@ -291,19 +289,16 @@ func RoleDiffSingle(conf *cfg.Config, csvfile, rolename string) error {
if !record.defined {
fmt.Printf("role %s is not defined\n", rolename)
return nil
}
role, exists := res[rolename]
if !exists {
fmt.Printf("role %s is not deployed\n", rolename)
return nil
} else {
fmt.Printf("role %s is deployed\n", rolename)
}
slog.Debug("found role", "role", role)
if conf.Debug {
@@ -315,7 +310,6 @@ func RoleDiffSingle(conf *cfg.Config, csvfile, rolename string) error {
if err != nil {
return err
}
slog.Debug("group mappings", "groups", groups)
// check cluster setting
@@ -327,7 +321,6 @@ func RoleDiffSingle(conf *cfg.Config, csvfile, rolename string) error {
// check index names+privs
indices := []string{}
privs := []string{}
for _, index := range role.Indices {
for _, name := range index.Names {
indices = append(indices, strings.ReplaceAll(name, "**", "*"))
@@ -340,7 +333,6 @@ func RoleDiffSingle(conf *cfg.Config, csvfile, rolename string) error {
// check kibana application space
spaces := []string{}
for _, app := range role.Applications {
for _, resource := range app.Resources {
if strings.Contains(resource, "space:") {

3
pkg/es/rollover.go
View File

@@ -31,17 +31,14 @@ func RolloverConditions(conf *cfg.Config) types.RolloverConditionsVariant {
if conf.MaxAge != "" {
cond.MaxAge(esdsl.NewDuration().String(conf.MaxAge))
}
if conf.MaxDocs > 0 {
cond.MaxDocs(int64(conf.MaxDocs))
}
if conf.MaxShardSize > 0 {
bs := esdsl.NewByteSize()
bs.Int64(int64(conf.MaxShardSize))
cond.MaxPrimaryShardSize(bs)
}
if conf.MaxShardDocs > 0 {
cond.MaxPrimaryShardDocs(int64(conf.MaxShardDocs))
}

16
pkg/es/search.go
View File

@@ -82,13 +82,13 @@ func explainSearch(conf *cfg.Config, search *search.Search) error {
Size(1). // one's enough for explain
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to call explain search (esdsl): %w", esErrorString(err))
return fmt.Errorf("failed to call explain search (esdsl): %s", esErrorString(err))
}
if conf.Debug {
raw, err := json.Marshal(res)
if err != nil {
return fmt.Errorf("failed to marshal explain result: %w", err)
return fmt.Errorf("failed to marshal explain result: %s", err)
}
value := gjson.Get(string(raw), "hits.hits.0._explanation")
@@ -111,7 +111,6 @@ func explainSearch(conf *cfg.Config, search *search.Search) error {
func explain(res *types.ExplanationDetail, indent string) {
fmt.Println(indent + "- " + res.Description)
for _, ex := range res.Details {
fmt.Println(indent + " - " + ex.Description)
fmt.Println(indent + fmt.Sprintf(" score: %f", res.Value))
@@ -135,7 +134,7 @@ func validateSearch(conf *cfg.Config, queries []string) error {
res, err := validate.
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to validate search (esdsl): %w", esErrorString(err))
return fmt.Errorf("failed to validate search (esdsl): %s", esErrorString(err))
}
slog.Debug("ES result", "search", res)
@@ -155,7 +154,7 @@ func searchOnce(conf *cfg.Config, search *search.Search) error {
Size(conf.To).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to run search (esdsl): %w", esErrorString(err))
return fmt.Errorf("failed to run search (esdsl): %s", esErrorString(err))
}
slog.Debug("ES result", "search", res)
@@ -168,10 +167,9 @@ func searchOnce(conf *cfg.Config, search *search.Search) error {
// https://www.elastic.co/docs/reference/elasticsearch/clients/go/using-the-api/searching#_pit_search_after
func searchPit(conf *cfg.Config, req *search.Request) error {
ctx := context.Background()
pit, err := conf.DefaultCluster.ES().OpenPointInTime(conf.Index).KeepAlive("1m").Do(ctx)
if err != nil {
return fmt.Errorf("failed to open point-in-time request for search: %w", err)
return fmt.Errorf("failed to open point-in-time request for search: %s", err)
}
defer func() {
_, err := conf.DefaultCluster.ES().ClosePointInTime().Id(pit.Id).Do(ctx)
@@ -194,7 +192,7 @@ func searchPit(conf *cfg.Config, req *search.Request) error {
for {
res, err := search.Do(ctx)
if err != nil {
return fmt.Errorf("failed to run search (esdsl pit): %w", esErrorString(err))
return fmt.Errorf("failed to run search (esdsl pit): %s", esErrorString(err))
}
if len(res.Hits.Hits) == 0 {
@@ -224,7 +222,7 @@ func searchTail(conf *cfg.Config, search *search.Search) error {
for {
res, err := search.Do(context.Background())
if err != nil {
return fmt.Errorf("failed to run search (esdsl): %w", esErrorString(err))
return fmt.Errorf("failed to run search (esdsl): %s", esErrorString(err))
}
slog.Debug("ES result", "search", res)

5
pkg/es/search_filter.go
View File

@@ -63,7 +63,6 @@ type filter struct {
// must match literally, and range filters using -r "@timestamp:2026-05-28T10:00:00 to now"
func NewFilter(query string) (*filter, error) {
var separator string
var criteria int // we use the constants on top for this
switch {
@@ -77,7 +76,7 @@ func NewFilter(query string) (*filter, error) {
part := strings.Split(query, separator)
if len(part) != 2 {
return nil, errors.New("search queries must be in the form field<sep>pattern where <sep> must be one of: = or !=")
return nil, fmt.Errorf("search queries must be in the form field<sep>pattern where <sep> must be one of: = or !=")
}
flt := &filter{term: part[0], filter: part[1], criteria: criteria}
@@ -235,6 +234,7 @@ func addFilters(conf *cfg.Config) ([]types.QueryVariant, error) {
}
filters[count-1] = rng
}
return filters, nil
@@ -247,6 +247,7 @@ func addSort(conf *cfg.Config, search *search.Search) *search.Search {
Index(conf.Index).
Fields(conf.SortBy).
Do(context.Background())
if err != nil {
// whatever it was, do not add Sort()
slog.Debug("get field capabilities", "field", conf.SortBy, "error", esErrorString(err))

79
pkg/es/searchql.go
View File

@@ -1,79 +0,0 @@
/*
Copyright © 2026 Thomas von Dein
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package es
import (
"context"
"encoding/json"
"fmt"
"log/slog"
"codeberg.org/scip/esctl/pkg/cfg"
"codeberg.org/scip/esctl/pkg/printer"
)
type searchQlColumn struct {
Name string `json:"name"`
Type string `json:"type"`
}
type searchQlResult struct {
Partial bool `json:"is_partial"`
Docs int `json:"documents_found"`
Columns []searchQlColumn `json:"columns"`
Values [][]any `json:"values"`
}
func SearchQL(conf *cfg.Config, querystring string) error {
query := conf.DefaultCluster.ES().Esql.Query().
Query(querystring).
DropNullColumns(true)
res, err := query.Do(context.Background())
if err != nil {
return fmt.Errorf("failed to run esql query: %w", esErrorString(err))
}
if conf.Debug {
fmt.Println(string(res))
}
qlResult := searchQlResult{}
if err = json.Unmarshal(res, &qlResult); err != nil {
return fmt.Errorf("failed to unmarshal esql result: %w", err)
}
slog.Debug("searchql result", "qlres", qlResult)
if qlResult.Docs == 0 {
return nil
}
table := printer.NewTable(conf, len(qlResult.Columns), len(qlResult.Values))
headers := make([]string, len(qlResult.Columns))
for idx, col := range qlResult.Columns {
headers[idx] = col.Name
}
table.Addheaders(headers...)
table.Entries = qlResult.Values
return table.Print()
}

77
pkg/es/shard.go
View File

@@ -40,7 +40,8 @@ func colorzizeShard(conf *cfg.Config, state, name string) string {
}
func resolvePrirep(state string) string {
if state == "p" {
switch state {
case "p":
return "primary"
}
@@ -85,7 +86,7 @@ func ShardList(conf *cfg.Config) error {
res, err := conf.DefaultCluster.ES().Cat.Shards().
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get shards: %w", esErrorString(err))
return fmt.Errorf("failed to get shards: %s", esErrorString(err))
}
shardlist := filterShards(conf, res)
@@ -106,7 +107,7 @@ func printShards(conf *cfg.Config, shardlist shards.Response) error {
for idx, shard := range shardlist {
name := colorzizeShard(conf, *shard.State, *shard.Index)
table.Entries[idx] = []any{
table.Entries[idx] = []string{
name,
*shard.Shard,
resolvePrirep(*shard.Prirep),
@@ -124,7 +125,6 @@ func printShards(conf *cfg.Config, shardlist shards.Response) error {
}
table.Sort()
if err := table.Print(); err != nil {
return err
}
@@ -136,7 +136,7 @@ func ShardShow(conf *cfg.Config, index string) error {
res, err := conf.DefaultCluster.ES().Cat.Shards().Index(index).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get shards: %w", esErrorString(err))
return fmt.Errorf("failed to get shards: %s", esErrorString(err))
}
slog.Debug("ES result", "shards", res)
@@ -145,70 +145,3 @@ func ShardShow(conf *cfg.Config, index string) error {
return printShards(conf, res)
}
func ShardAllocation(conf *cfg.Config, index string) error {
explain := conf.DefaultCluster.ES().Cluster.AllocationExplain().
Index(index).
Primary(conf.Primary).
Shard(conf.Shards)
if conf.FromNode != "" {
explain.CurrentNode(conf.FromNode)
}
res, err := explain.Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get shard allocation explain: %w", esErrorString(err))
}
slog.Debug("ES result", "explain", res)
table := printer.NewTable(conf, 2, 10)
table.Addheaders("shard allocation setting", "value")
table.Entries = [][]any{
{"Index", index},
{"Current state", res.CurrentState},
{"Can rebalance cluster", res.CanRebalanceCluster.Name},
{"Can rebalance to another node", res.CanRebalanceToOtherNode.Name},
{"Can remain on current node", res.CanRemainOnCurrentNode.Name},
}
if res.CurrentNode != nil {
currentNode := res.CurrentNode
roles := make([]string, len(currentNode.Roles))
for idx, role := range currentNode.Roles {
roles[idx] = role.Name
}
table.Entries = append(table.Entries, [][]any{
{"Current node", currentNode.Name},
{"Current k8s node", currentNode.Attributes["k8s_node_name"]},
{"Current node address", currentNode.TransportAddress},
{"Current node id", currentNode.Id},
{"Current node weight", currentNode.WeightRanking},
{"Current node roles", roles},
}...)
} else {
table.AddRow("Current node", "not currently assigned to any node")
}
if res.CurrentState == "unassigned" {
table.AddRow("Unassignment reason", res.UnassignedInfo.Reason.String()+" at "+res.UnassignedInfo.At.(string))
}
for _, nodeDecision := range res.NodeAllocationDecisions {
for _, decider := range nodeDecision.Deciders {
table.AddRow("Allocation decider", decider.Decider)
table.AddRow(" -> decision", decider.Decision.String())
table.AddRow(" -> explanation", decider.Explanation)
}
}
if err := table.Print(); err != nil {
return err
}
return nil
}

27
pkg/es/snapshot.go
View File

@@ -45,7 +45,7 @@ func SnapshotList(conf *cfg.Config) error {
// get partial indicies
ires, err := conf.DefaultCluster.ES().Cat.Indices().Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get indicies: %w", esErrorString(err))
return fmt.Errorf("failed to get indicies: %s", esErrorString(err))
}
indicies := map[string]int{}
@@ -58,7 +58,7 @@ func SnapshotList(conf *cfg.Config) error {
// get snapshots
sres, err := conf.DefaultCluster.ES().Cat.Snapshots().Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get snapshots: %w", esErrorString(err))
return fmt.Errorf("failed to get snapshots: %s", esErrorString(err))
}
slog.Debug("ES result", "indicies", sres)
@@ -88,7 +88,7 @@ func SnapshotList(conf *cfg.Config) error {
table.Addheaders("name", "index", "start", "orphaned", "status")
for idx, snap := range snapshots {
table.Entries[idx] = []any{
table.Entries[idx] = []string{
snap.Name,
snap.Forindex,
snap.Start,
@@ -98,7 +98,6 @@ func SnapshotList(conf *cfg.Config) error {
}
table.Sort()
if err := table.Print(); err != nil {
return err
}
@@ -109,7 +108,7 @@ func SnapshotList(conf *cfg.Config) error {
func SnapshotShow(conf *cfg.Config, snapshot string) error {
res, err := conf.DefaultCluster.ES().Snapshot.Get("*", snapshot).Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get snapshot: %w", esErrorString(err))
return fmt.Errorf("failed to get snapshot: %s", esErrorString(err))
}
slog.Debug("ES result", "snapshot", res)
@@ -123,20 +122,20 @@ func SnapshotShow(conf *cfg.Config, snapshot string) error {
snap := res.Snapshots[0]
table.Entries = [][]any{
table.Entries = [][]string{
{"snapshot", snapshot},
{"uuid", snap.Uuid},
{"repository", *snap.Repository},
{"version_id", snap.VersionId},
{"version_id", fmt.Sprintf("%d", snap.VersionId)},
{"version", *snap.Version},
{"include_global_state", *snap.IncludeGlobalState},
{"include_global_state", fmt.Sprintf("%t", *snap.IncludeGlobalState)},
{"state", *snap.State},
{"start_time", snap.StartTime},
{"end_time", snap.EndTime},
{"duration_in_millis", *snap.DurationInMillis},
{"shards-total", snap.Shards.Total},
{"shards-failed", snap.Shards.Failed},
{"shards-successful", snap.Shards.Successful},
{"start_time", fmt.Sprintf("%s", snap.StartTime)},
{"end_time", fmt.Sprintf("%s", snap.EndTime)},
{"duration_in_millis", fmt.Sprintf("%d", *snap.DurationInMillis)},
{"shards-total", fmt.Sprintf("%d", snap.Shards.Total)},
{"shards-failed", fmt.Sprintf("%d", snap.Shards.Failed)},
{"shards-successful", fmt.Sprintf("%d", snap.Shards.Successful)},
}
if err := table.Print(); err != nil {

11
pkg/es/task.go
View File

@@ -30,15 +30,15 @@ import (
func TaskList(conf *cfg.Config) error {
res, err := conf.DefaultCluster.ES().Cat.Tasks().
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get index templates: %w", esErrorString(err))
return fmt.Errorf("failed to get index templates: %s", esErrorString(err))
}
slog.Debug("res", "tasks", res)
table := printer.NewTable(conf, 8, len(res))
table.Addheaders("task id", "action", "start time", "run time", "node", "type")
for idx, task := range res {
var ts time.Time
@@ -49,10 +49,10 @@ func TaskList(conf *cfg.Config) error {
ts = time.Unix(int64(epc/1000), 0)
}
table.Entries[idx] = []any{
table.Entries[idx] = []string{
*task.TaskId,
*task.Action,
ts,
ts.Format("2006-01-02T15:04:05"),
*task.RunningTime,
*task.Node,
*task.Type,
@@ -66,8 +66,9 @@ func TaskCancel(conf *cfg.Config, taskid string) error {
_, err := conf.DefaultCluster.ES().Tasks.Cancel().
TaskId(taskid).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to cancel task: %w", esErrorString(err))
return fmt.Errorf("failed to cancel task: %s", esErrorString(err))
}
return nil

37
pkg/printer/bytes.go
View File

@@ -1,37 +0,0 @@
/*
Copyright © 2026 Thomas von Dein
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package printer
import "github.com/dustin/go-humanize"
type ByteSize struct {
size uint64
}
func (b *ByteSize) String() string {
return humanize.Bytes(b.size)
}
func Bytes(size int64) *ByteSize {
return &ByteSize{size: uint64(size)}
}
func ByteString(size int64) string {
b := ByteSize{size: uint64(size)}
return b.String()
}

105
pkg/printer/cast.go
View File

@@ -1,105 +0,0 @@
/*
Copyright © 2026 Thomas von Dein
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package printer
import (
"fmt"
"strconv"
"strings"
"time"
"github.com/elastic/go-elasticsearch/v9/typedapi/types"
)
func any2string(in any) string {
//nolint:gocritic
switch val := in.(type) {
case string:
return val
case bool:
return strconv.FormatBool(val)
case int64:
return strconv.FormatInt(val, 10)
case int:
return strconv.Itoa(val)
case float64:
return fmt.Sprintf("%.2f", val)
case float32:
return fmt.Sprintf("%.2f", val)
case []string:
return strings.Join(val, ",")
case ByteSize:
return val.String()
case *ByteSize:
return val.String()
case time.Time:
return val.Format("2006-01-02 15:04:05")
case time.Duration:
return val.String()
case []byte:
return string(val)
case nil:
return "null"
case types.Percentage, types.DateTime:
return val.(string)
}
return ""
}
func (data *Table) preprocessRows() {
if data.processed {
// only do it once
return
}
// convert entries to strings
data.rows = make([][]string, len(data.Entries))
for rowidx, entries := range data.Entries {
data.rows[rowidx] = make([]string, len(entries))
for colidx, entry := range data.Entries[rowidx] {
data.rows[rowidx][colidx] = any2string(entry)
}
}
// determine header lenght's
for idx, head := range data.Headers {
data.lenHeaders[idx] = visibleLen(head)
}
// determine max width per column
for _, entries := range data.rows {
currentWidth := 0
for idx, entry := range entries {
length := visibleLen(entry)
if data.lenHeaders[idx] < length {
if length > currentWidth+data.maxwidth {
data.lenHeaders[idx] = data.maxwidth - currentWidth
} else {
data.lenHeaders[idx] = length
}
}
currentWidth += data.lenHeaders[idx]
}
}
data.processed = true
}

3
pkg/printer/doc.go
View File

@@ -33,6 +33,7 @@ func PrintDocs(conf *cfg.Config, hits []types.Hit) {
for _, hit := range hits {
PrintDoc(conf, hit)
}
}
func PrintDoc(conf *cfg.Config, hit types.Hit) {
@@ -51,6 +52,6 @@ func PrintDoc(conf *cfg.Config, hit types.Hit) {
value := gjson.Get(docjson, conf.Path)
fmt.Println(value.String())
} else {
fmt.Println(docjson)
fmt.Print(docjson)
}
}

25
pkg/printer/pager.go
View File

@@ -34,14 +34,12 @@ var (
titleStyle = func() lipgloss.Style {
b := lipgloss.RoundedBorder()
b.Right = "├"
return lipgloss.NewStyle().BorderStyle(b).Padding(0, 1)
}()
infoStyle = func() lipgloss.Style {
b := lipgloss.RoundedBorder()
b.Left = "┤"
return titleStyle.BorderStyle(b)
}()
)
@@ -58,9 +56,10 @@ func (m model) Init() tea.Cmd {
}
func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
var cmd tea.Cmd
cmds := make([]tea.Cmd, 1)
var (
cmd tea.Cmd
cmds []tea.Cmd
)
switch msg := msg.(type) {
case tea.KeyMsg:
@@ -91,7 +90,7 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
// Handle keyboard and mouse events in the viewport
m.viewport, cmd = m.viewport.Update(msg)
cmds[0] = cmd
cmds = append(cmds, cmd)
return m, tea.Batch(cmds...)
}
@@ -100,7 +99,6 @@ func (m model) View() string {
if !m.ready {
return "\n Initializing..."
}
return fmt.Sprintf("%s\n%s\n%s", m.headerView(), m.viewport.View(), m.footerView())
}
@@ -108,25 +106,30 @@ func (m model) headerView() string {
// title := titleStyle.Render("RPN Help Overview")
title := titleStyle.Render(m.title)
line := strings.Repeat("─", max(0, m.viewport.Width-lipgloss.Width(title)))
return lipgloss.JoinHorizontal(lipgloss.Center, title, line)
}
func (m model) footerView() string {
info := infoStyle.Render(fmt.Sprintf("%3.f%%", m.viewport.ScrollPercent()*100))
line := strings.Repeat("─", max(0, m.viewport.Width-lipgloss.Width(info)))
return lipgloss.JoinHorizontal(lipgloss.Center, line, info)
}
func max(a, b int) int {
if a > b {
return a
}
return b
}
func Pager(title, message string) {
pager := tea.NewProgram(
p := tea.NewProgram(
model{content: message, title: title},
tea.WithAltScreen(), // use the full size of the terminal in its "alternate screen buffer"
tea.WithMouseCellMotion(), // turn on mouse support so we can track the mouse wheel
)
if _, err := pager.Run(); err != nil {
if _, err := p.Run(); err != nil {
fmt.Println("could not run pager:", err)
os.Exit(1)
}

304
pkg/printer/table.go
View File

@@ -25,18 +25,18 @@ import (
"strings"
"codeberg.org/scip/esctl/pkg/cfg"
"github.com/olekukonko/tablewriter"
"github.com/olekukonko/tablewriter/renderer"
"github.com/olekukonko/tablewriter/tw"
"github.com/seeruk/go-wordwrap"
"gopkg.in/yaml.v3"
)
type Table struct {
Mode string // tsv, json, yaml
Mode string // table, markdown, json
Headers []string
RawHeaders []string
Entries [][]any
Entries [][]string
rows [][]string // representation used for printing
processed bool
lenHeaders []int
alignInts bool
maxwidth int
@@ -46,44 +46,23 @@ func NewTable(conf *cfg.Config, columns, rows int) *Table {
table := Table{Mode: conf.Output, maxwidth: cfg.GetTermWidth()}
table.Headers = make([]string, columns)
table.RawHeaders = make([]string, columns)
table.Entries = make([][]any, rows)
table.Entries = make([][]string, rows)
table.lenHeaders = make([]int, columns)
table.alignInts = conf.AlignInts
return &table
}
func NewTableEmpty(conf *cfg.Config) *Table {
table := Table{Mode: conf.Output, maxwidth: cfg.GetTermWidth()}
table.alignInts = conf.AlignInts
return &table
}
func (table *Table) WithHeaders(headers ...string) *Table {
count := len(headers)
table.Entries = [][]any{}
table.lenHeaders = make([]int, count)
table.Headers = make([]string, count)
table.RawHeaders = make([]string, count)
table.Addheaders(headers...)
return table
}
func (table *Table) Print() error {
switch table.Mode {
func (data *Table) Print() error {
switch data.Mode {
case "markdown", "md":
return data.PrintMarkdown()
case "json":
return table.PrintJSON()
return data.PrintJSON()
case "yaml":
return table.PrintYAML()
case "csv":
return table.PrintCSV()
return data.PrintYAML()
default:
return table.PrintTSV()
return data.PrintTSV()
}
}
@@ -91,12 +70,26 @@ var (
ansiCtrlSeq = regexp.MustCompile(`\033.[0-9;]+m`)
)
func (table *Table) PrintYAML() error {
raw := table.toMap()
// needed for json and yaml output
func (data *Table) toMap() []map[string]string {
raw := make([]map[string]string, len(data.Entries))
for idx, entries := range data.Entries {
raw[idx] = make(map[string]string, len(data.Headers))
for eidx, entry := range entries {
raw[idx][data.Headers[eidx]] = entry
}
}
return raw
}
func (data *Table) PrintYAML() error {
raw := data.toMap()
body, err := yaml.Marshal(raw)
if err != nil {
return fmt.Errorf("failed to produce YAML output: %w", err)
return fmt.Errorf("failed to produce YAML output: %s", err)
}
fmt.Println(string(body))
@@ -104,12 +97,12 @@ func (table *Table) PrintYAML() error {
return nil
}
func (table *Table) PrintJSON() error {
raw := table.toMap()
func (data *Table) PrintJSON() error {
raw := data.toMap()
body, err := json.MarshalIndent(raw, "", " ")
if err != nil {
return fmt.Errorf("failed to produce JSON output: %w", err)
return fmt.Errorf("failed to produce JSON output: %s", err)
}
fmt.Println(string(body))
@@ -117,35 +110,56 @@ func (table *Table) PrintJSON() error {
return nil
}
func (table *Table) PrintTSV() error {
// length's, convert cell types
table.preprocessRows()
// Well. tablewriter should be able to print in TSV mode, but it
// doesn't work because of the various ansic color escapes we're
// using. Therefore we do it the hard way and use our own TSV printer.
func (data *Table) PrintTSV() error {
// determine lenght's
for idx, head := range data.Headers {
data.lenHeaders[idx] = visibleLen(head)
}
for _, entries := range data.Entries {
currentWidth := 0
for idx, entry := range entries {
length := visibleLen(entry)
if data.lenHeaders[idx] < length {
if length > currentWidth+data.maxwidth {
data.lenHeaders[idx] = data.maxwidth - currentWidth
} else {
data.lenHeaders[idx] = length
}
}
currentWidth += data.lenHeaders[idx]
}
}
// output headers
for idx, header := range table.Headers {
if idx+1 != len(table.Headers) {
fmt.Print(header, strings.Repeat(" ", table.lenHeaders[idx]-visibleLen(header)))
for idx, header := range data.Headers {
if idx+1 != len(data.Headers) {
fmt.Print(header, strings.Repeat(" ", data.lenHeaders[idx]-visibleLen(header)))
} else {
// no padding for last header
fmt.Print(header)
}
if idx < len(table.Headers)-1 {
if idx < len(data.Headers)-1 {
fmt.Print(" ")
}
}
fmt.Println()
for _, entries := range table.rows {
for _, entries := range data.Entries {
currentWidth := 0
for idx, entry := range entries {
length := visibleLen(entry)
if length+currentWidth > table.maxwidth && table.maxwidth-currentWidth > 1 {
if length+currentWidth > data.maxwidth {
// text is too wide to be put into one line, wrap it
wrapper := wordwrap.Wrapper(table.maxwidth-currentWidth, false)
wrapper := wordwrap.Wrapper(data.maxwidth-currentWidth, false)
wrapped := wrapper(entry)
// and indent it
@@ -158,111 +172,169 @@ func (table *Table) PrintTSV() error {
}
}
currentWidth += table.lenHeaders[idx]
currentWidth += data.lenHeaders[idx]
switch {
case isInt(entry) && table.alignInts:
if isInt(entry) && data.alignInts {
// align right
fmt.Print(strings.Repeat(" ", table.lenHeaders[idx]-length), entry)
case length < table.lenHeaders[idx] && idx+1 != len(entries):
fmt.Print(strings.Repeat(" ", data.lenHeaders[idx]-length), entry)
} else if length < data.lenHeaders[idx] && idx+1 != len(entries) {
// pad right, if required
fmt.Print(entry, strings.Repeat(" ", table.lenHeaders[idx]-length))
default:
fmt.Print(entry, strings.Repeat(" ", data.lenHeaders[idx]-length))
} else {
// no padding for last entry
fmt.Print(entry)
}
if idx < len(table.Headers)-1 {
if idx < len(data.Headers)-1 {
fmt.Print(" ")
}
}
fmt.Println()
}
return nil
}
func (table *Table) PrintCSV() error {
table.preprocessRows()
func (data *Table) PrintTabularFail() error {
OFS := " "
fmt.Println(strings.Join(table.RawHeaders, ","))
tableString := &strings.Builder{}
for _, entries := range table.rows {
row := make([]string, len(entries))
styleTSV := tw.NewSymbolCustom("space").WithColumn("\t")
for idx, entry := range entries {
if strings.Contains(entry, " ") || strings.Contains(entry, ",") {
row[idx] = `"` + entry + `"`
} else {
row[idx] = entry
}
table := tablewriter.NewTable(tableString,
tablewriter.WithRenderer(
renderer.NewBlueprint(tw.Rendition{
Borders: tw.BorderNone,
Symbols: styleTSV,
Settings: tw.Settings{
Separators: tw.SeparatorsNone,
Lines: tw.LinesNone,
},
})),
tablewriter.WithConfig(tablewriter.Config{
Header: tw.CellConfig{
Formatting: tw.CellFormatting{
AutoFormat: tw.Off,
},
Padding: tw.CellPadding{Global: tw.Padding{Left: "", Right: OFS}},
},
Row: tw.CellConfig{
Formatting: tw.CellFormatting{
AutoWrap: tw.WrapNone,
Alignment: tw.AlignLeft,
},
Padding: tw.CellPadding{Global: tw.Padding{Right: OFS}},
},
Debug: true,
}),
)
table.Header(data.Headers)
if err := table.Bulk(data.Entries); err != nil {
return fmt.Errorf("failed to add data to table renderer: %s", err)
}
fmt.Println(strings.Join(row, ","))
if err := table.Render(); err != nil {
return fmt.Errorf("failed to render table: %s", err)
}
// we need to trim our output here, because tablewriter appends
// excess whitespace to our rows.
cleanedString := &strings.Builder{}
for _, row := range strings.Split(strings.TrimSpace(tableString.String()), "\n") {
cleanedString.WriteString(strings.TrimSpace(row))
cleanedString.WriteString("\n")
}
fmt.Println(cleanedString.String())
return nil
}
func (table *Table) Sort() {
func (data *Table) PrintMarkdown() error {
tableString := &strings.Builder{}
table := tablewriter.NewTable(tableString,
tablewriter.WithRenderer(
renderer.NewBlueprint(
tw.Rendition{
Borders: tw.Border{
Left: tw.On,
Right: tw.On,
Top: tw.Off,
Bottom: tw.Off,
},
Settings: tw.Settings{
Separators: tw.Separators{
ShowHeader: tw.On,
ShowFooter: tw.Off,
BetweenRows: tw.Off,
BetweenColumns: 0,
},
},
Symbols: tw.NewSymbols(tw.StyleMarkdown),
})),
tablewriter.WithConfig(
tablewriter.Config{
Header: tw.CellConfig{
Formatting: tw.CellFormatting{
Alignment: tw.AlignLeft,
AutoFormat: tw.Off,
},
},
Row: tw.CellConfig{
Formatting: tw.CellFormatting{
Alignment: tw.AlignLeft,
},
},
},
),
)
table.Header(data.Headers)
if err := table.Bulk(data.Entries); err != nil {
return fmt.Errorf("failed to add data to table renderer: %s", err)
}
if err := table.Render(); err != nil {
return fmt.Errorf("failed to render table: %s", err)
}
fmt.Println(tableString.String())
return nil
}
func (data *Table) Sort() {
// sanity checks
if len(table.Entries) == 0 {
if len(data.Entries) == 0 {
return
}
table.preprocessRows()
sort.Slice(table.rows, func(i, j int) bool {
return table.rows[i][0] < table.rows[j][0]
sort.Slice(data.Entries, func(i, j int) bool {
return data.Entries[i][0] < data.Entries[j][0]
})
}
func (table *Table) Addheaders(headers ...string) {
func (data *Table) Addheaders(headers ...string) {
for idx, header := range headers {
switch table.Mode {
switch data.Mode {
case "json", "yaml":
table.Headers[idx] = strings.ReplaceAll(strings.ToLower(header), " ", "_")
data.Headers[idx] = strings.ReplaceAll(strings.ToLower(header), " ", "_")
default:
table.Headers[idx] = bold(strings.ReplaceAll(strings.ToUpper(header), " ", "-"))
data.Headers[idx] = bold(strings.ReplaceAll(strings.ToUpper(header), " ", "-"))
}
table.RawHeaders[idx] = header
}
}
func (table *Table) AddRow(fields ...any) {
table.Entries = append(table.Entries, fields)
}
func (table *Table) AddRowLate(fields ...any) {
table.AddRow(fields)
if !table.processed {
return
}
row := make([]string, len(fields))
for idx, field := range fields {
row[idx] = any2string(field)
}
table.rows = append(table.rows, row)
}
// needed for json and yaml output
func (table *Table) toMap() []map[string]any {
raw := make([]map[string]any, len(table.Entries))
for idx, entries := range table.Entries {
raw[idx] = make(map[string]any, len(table.Headers))
for eidx, entry := range entries {
raw[idx][table.Headers[eidx]] = entry
}
}
return raw
func (data *Table) AddRow(fields ...string) {
data.Entries = append(data.Entries, fields)
}
// return the length of a string but only visible chars, w/o ansi color escapes

6
t/.env
View File

@@ -1,6 +0,0 @@
ES_PASS=tuscador1
STACK_VERSION=9.4.2
CLUSTER_NAME=domdoc
LICENSE=basic
ES_PORT=9200
MEM_LIMIT=1073741824

46
t/Makefile
View File

@@ -1,46 +0,0 @@
.PHONY: test clean cluster docs search up down delete
# docker stuff
up:
@echo "booting up docker containers"
docker compose up -d --remove-orphans
waitup:
@echo "wait til es cluster is up and running"
mosscap msc-actions.yaml msc-docker.yaml -n -t 300
down:
@echo "shutting down docker containers"
docker compose down
delete:
@echo "delete docker volumes"
docker volume rm t_certs t_esdata01 t_esdata02 t_esdata03
clean-docker: down delete
# mosscap esctl stuff
test: up waitup cluster docs wait search
cluster:
@echo "setting up elastisearch cluster"
mosscap msc-actions.yaml msc-cluster.yaml -n
docs:
@echo "put some docs"
mosscap msc-actions.yaml msc-docs.yaml -n -c 1000 -p 100
wait:
sleep 10
search:
@echo "make some searches"
mosscap msc-actions.yaml msc-search.yaml -n -t 60
clean:
@echo "cleaning up cluster"
mosscap msc-actions.yaml msc-clean.yaml -n
rm -f *.state debug.log

77
t/README.md
View File

@@ -1,77 +0,0 @@
# Testing
Building regular unit tests would require to create a mock
elasticsearch cluster, which is too much a chore for a one man
show. Running a regular elasticsearch cluster on Codeberg CI is not
economically reasonable.
Therefore I run tests manually. Here's how.
# Dependencies
The following is required:
- linux, any distro will do
- docker in a decent version
- [mosscap](https://codeberg.org/scip/mosscap) version `0.0.17` or higher.
# Docker
There's a ready to use docker compose file, which fires up an
elasticsearch cluster with 3 nodes. To start it manually just execute
`make up`.
When it's up and running it should look like this:
```console
$ docker ps
NAMES STATUS PORTS
t-es03-1 Up 16 minutes (healthy) 9200/tcp, 9300/tcp
t-es02-1 Up 16 minutes (healthy) 9200/tcp, 9300/tcp
t-es01-1 Up 16 minutes (healthy) 0.0.0.0:9200->9200/tcp, [::]:9200->9200/tcp, 9300/tcp
```
There's an esctl config file `cluster.yaml`, which you can use to
access that elasticsearch cluster, e.g.:
```console
$ esctl -c cluster.yaml cluster status
DOCKER/DOMDOC STATUS
Cluster Name domdoc
ES Status green
ES Version 9.4.2
Is Leader false
Active Shards 8
Active Primary Shards 4
Unassigned Shards 0
Unassigned Primary Shards 0
Pending Tasks 0
Nodes 3
Red Indices 0
Long Running Tasks 0
```
# Automated Tests
To execute the automated tests, just execute `make test`. This boots
up the elasticsearch containers, waits until they are up and running,
creates data on it and queries it.
# Cleanup
To just clean up the elasticsearch cluster run `make clean`.
To remove everything, run `make clean-docker`.
# Reference
## Mosscap configs
| CONFIG | DESCRIPTION |
|------------------|----------------------------------------------------------------------|
| msc-actions.yaml | contains all actions, used by all item configs |
| msc-docker.yaml | single item: used to wait until elasticsearch containers are healthy |
| msc-cluster.yaml | single item: create indices etc |
| msc-docs.yaml | generator items: put docs into the index |
| msc-search.yaml | single item: do some searches |
| msc-clean.yaml | single item: clean up the elasticsearch cluster |

7
t/cluster.yaml
View File

@@ -1,7 +0,0 @@
# esctl config
clusters:
docker/domdoc:
uri: https://elastic:9200
user: elastic
token: ""
default: false

207
t/docker-compose.yaml
View File

@@ -1,207 +0,0 @@
services:
setup:
image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
volumes:
- certs:/usr/share/elasticsearch/config/certs
user: "0"
command: >
bash -c '
if [ x${ES_PASS} == x ]; then
echo "Set the ES_PASS environment variable in the .env file";
exit 1;
fi;
if [ ! -f config/certs/ca.zip ]; then
echo "Creating CA";
bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
unzip config/certs/ca.zip -d config/certs;
fi;
if [ ! -f config/certs/certs.zip ]; then
echo "Creating certs";
echo -ne \
"instances:\n"\
" - name: es01\n"\
" dns:\n"\
" - es01\n"\
" - localhost\n"\
" ip:\n"\
" - 127.0.0.1\n"\
" - name: es02\n"\
" dns:\n"\
" - es02\n"\
" - localhost\n"\
" ip:\n"\
" - 127.0.0.1\n"\
" - name: es03\n"\
" dns:\n"\
" - es03\n"\
" - localhost\n"\
" ip:\n"\
" - 127.0.0.1\n"\
> config/certs/instances.yml;
bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
unzip config/certs/certs.zip -d config/certs;
fi;
echo "Setting file permissions"
chown -R root:root config/certs;
find . -type d -exec chmod 750 \{\} \;;
find . -type f -exec chmod 640 \{\} \;;
echo "Waiting for Elasticsearch availability";
until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
echo "All done!";
'
healthcheck:
test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
interval: 1s
timeout: 5s
retries: 120
es01:
depends_on:
setup:
condition: service_healthy
image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
volumes:
- certs:/usr/share/elasticsearch/config/certs
- esdata01:/usr/share/elasticsearch/data
ports:
- ${ES_PORT}:9200
environment:
- node.name=es01
- cluster.name=${CLUSTER_NAME}
- cluster.initial_master_nodes=es01
- discovery.seed_hosts=es02
- ELASTIC_PASSWORD=${ES_PASS}
- bootstrap.memory_lock=true
- xpack.security.enabled=true
- xpack.security.http.ssl.enabled=true
- xpack.security.http.ssl.key=certs/es01/es01.key
- xpack.security.http.ssl.certificate=certs/es01/es01.crt
- xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
- xpack.security.transport.ssl.enabled=true
- xpack.security.transport.ssl.key=certs/es01/es01.key
- xpack.security.transport.ssl.certificate=certs/es01/es01.crt
- xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
- xpack.security.transport.ssl.verification_mode=certificate
- xpack.license.self_generated.type=${LICENSE}
- xpack.ml.use_auto_machine_memory_percent=true
- node_roles=data_content
- xpack.searchable.snapshot.shared_cache.size="1gb"
- cluster.routing.allocation.disk.threshold_enabled=false
- cluster.routing.allocation.disk.watermark.low="2gb"
- cluster.routing.allocation.disk.watermark.high="1gb"
- cluster.routing.allocation.disk.watermark.flood_stage="500mb"
mem_limit: ${MEM_LIMIT}
ulimits:
memlock:
soft: -1
hard: -1
healthcheck:
test:
[
"CMD-SHELL",
"curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
]
interval: 10s
timeout: 10s
retries: 120
es02:
depends_on:
- es01
image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
volumes:
- certs:/usr/share/elasticsearch/config/certs
- esdata02:/usr/share/elasticsearch/data
environment:
- node.name=es02
- cluster.name=${CLUSTER_NAME}
- cluster.initial_master_nodes=es01,es02
- discovery.seed_hosts=es01
- ELASTIC_PASSWORD=${ES_PASS}
- bootstrap.memory_lock=true
- xpack.security.enabled=true
- xpack.security.http.ssl.enabled=true
- xpack.security.http.ssl.key=certs/es02/es02.key
- xpack.security.http.ssl.certificate=certs/es02/es02.crt
- xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
- xpack.security.transport.ssl.enabled=true
- xpack.security.transport.ssl.key=certs/es02/es02.key
- xpack.security.transport.ssl.certificate=certs/es02/es02.crt
- xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
- xpack.security.transport.ssl.verification_mode=certificate
- xpack.license.self_generated.type=${LICENSE}
- xpack.ml.use_auto_machine_memory_percent=true
- cluster.routing.allocation.disk.threshold_enabled=false
- cluster.routing.allocation.disk.watermark.low="2gb"
- cluster.routing.allocation.disk.watermark.high="1gb"
- cluster.routing.allocation.disk.watermark.flood_stage="500mb"
mem_limit: ${MEM_LIMIT}
ulimits:
memlock:
soft: -1
hard: -1
healthcheck:
test:
[
"CMD-SHELL",
"curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
]
interval: 10s
timeout: 10s
retries: 120
es03:
depends_on:
- es01
image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
volumes:
- certs:/usr/share/elasticsearch/config/certs
- esdata03:/usr/share/elasticsearch/data
environment:
- node.name=es03
- cluster.name=${CLUSTER_NAME}
- cluster.initial_master_nodes=es01,es03
- discovery.seed_hosts=es01
- ELASTIC_PASSWORD=${ES_PASS}
- bootstrap.memory_lock=true
- xpack.security.enabled=true
- xpack.security.http.ssl.enabled=true
- xpack.security.http.ssl.key=certs/es03/es03.key
- xpack.security.http.ssl.certificate=certs/es03/es03.crt
- xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
- xpack.security.transport.ssl.enabled=true
- xpack.security.transport.ssl.key=certs/es03/es03.key
- xpack.security.transport.ssl.certificate=certs/es03/es03.crt
- xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
- xpack.security.transport.ssl.verification_mode=certificate
- xpack.license.self_generated.type=${LICENSE}
- xpack.ml.use_auto_machine_memory_percent=true
- cluster.routing.allocation.disk.threshold_enabled=false
- cluster.routing.allocation.disk.watermark.low="2gb"
- cluster.routing.allocation.disk.watermark.high="1gb"
- cluster.routing.allocation.disk.watermark.flood_stage="500mb"
mem_limit: ${MEM_LIMIT}
ulimits:
memlock:
soft: -1
hard: -1
healthcheck:
test:
[
"CMD-SHELL",
"curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
]
interval: 10s
timeout: 10s
retries: 120
volumes:
certs:
driver: local
esdata01:
driver: local
esdata02:
driver: local
esdata03:
driver: local

96
t/msc-actions.yaml
View File

@@ -1,96 +0,0 @@
#
# this is a mosscap action file, see: https://codeberg.org/scip/mosscap
vars:
esctl: ../esctl -c cluster.yaml
actions:
wait: |
sleep [[.time]]
docker_wait: |
for i in {1..20}; do
count=$(docker ps | grep healthy | wc -l)
if test $count -eq 3; then
echo "all containers are healthy"
exit
fi
sleep 5s
done
echo "not all containers are healthy"
docker ps
cluster_ls: |
[[.esctl]] cluster ls | grep -E domdoc.*reachable.*yes
cluster_status: |
[[.esctl]] cluster status | grep green
create_ilm: |
[[.esctl]] ilm create --hot-rollover-max-age 7d \
--hot-rollover-max-primary-shard-size 25g \
--warm-min-age 0 \
--delete-min-age 14d [[.policy]]
describe_ilm: |
[[.esctl]] ilm show [[.policy]] -t | grep -E "rollover when storage > 25g"
create_index_template: |
[[.esctl]] index template create [[.index]] -i [[.index]] \
-s number_of_shards:3 \
-s index.sort.field:@timestamp \
-r 1m --ilm-policy [[.policy]] \
user:keyword message:text @timestamp:date
describe_index_template: |
[[.esctl]] index template show [[.index]] | grep -E "index.sort.field.*@timestamp"
create_index: |
[[.esctl]] index create -s 2 -r 2 [[.index]]
describe_index: |
[[.esctl]] index show [[.index]] | grep -E "fields.*@timestamp"
index_ls: |
[[.esctl]] index ls | grep [[.index]]
index_template_rm: |
[[.esctl]] index template rm [[.index]]
index_rm: |
[[.esctl]] index rm [[.index]]
timestamp: |
date --iso-8601=second
date: |
date +%Y-%m-%d
doc_add: |
[[.esctl]] doc add -i [[.index]] '{"user": "[[.user]]", "message":"[[.message]]", "@timestamp":"[[.ts]]"}'
search_any: |
[[.esctl]] search -i [[.index]] -l 1 | jq .source.message
search_date: |
[[.esctl]] search -i [[.index]] -l 1 | jq '.source."@timestamp"' | grep [[.today]]
search_count: |
[[.esctl]] search -i [[.index]] -l 5000 | wc -l | grep 1000
search_content: |
[[.esctl]] search -i [[.index]] [[.content_pattern]] -l 1 | jq .source.message
search_content_and: |
[[.esctl]] search -i [[.index]] [[.content_and_pattern]] -l 1 | jq .source.message
nodes: |
[[.esctl]] node ls | grep es01
describe_node: |
[[.esctl]] node show [[.node]] | grep "Node version"
license: |
[[.esctl]] license show | grep -E "Expires.*never"

14
t/msc-clean.yaml
View File

@@ -1,14 +0,0 @@
#
# this is a mosscap item file, see: https://codeberg.org/scip/mosscap
items:
- name: docker/domdoc
vars:
index: test
tasks:
- name: delete_index
action: index_rm
- name: delete_template
action: index_template_rm

48
t/msc-cluster.yaml
View File

@@ -1,48 +0,0 @@
#
# this is a mosscap item file, see: https://codeberg.org/scip/mosscap
statefile: cluster.state
items:
- name: docker/domdoc
vars:
index: test
node: es01
policy: testpolicy
tasks:
- name: ls
action: cluster_ls
- name: license
action: license
- name: status
action: cluster_status
- name: ilm
action: create_ilm
- name: check_ilm
action: describe_ilm
- name: template
action: create_index_template
- name: check_template
action: describe_index_template
- name: index
action: create_index
- name: indexls
action: index_ls
- name: describe
action: describe_index
- name: nodes
action: nodes
- name: check_node
action: describe_node

12
t/msc-docker.yaml
View File

@@ -1,12 +0,0 @@
#
# this is a mosscap item file, see: https://codeberg.org/scip/mosscap
items:
- name: docker/domdoc
vars:
index: test
time: 5s
tasks:
- name: wait
action: docker_wait

21
t/msc-docs.yaml
View File

@@ -1,21 +0,0 @@
#
# this is a mosscap generator item file, see: https://codeberg.org/scip/mosscap
vars:
index: test
tasks:
- name: ts
action: timestamp
- name: add
action: doc_add
args:
ts: ts.result
generate:
user: $name
# will become something like:
# "content34 iBAXQ2Va Ko5gMwwu FEH2O99B"
# so we can search for "content3*" and get multiple matches
message: "content$int8 $rand make$int8 $rand"

31
t/msc-search.yaml
View File

@@ -1,31 +0,0 @@
#
# this is a mosscap item file, see: https://codeberg.org/scip/mosscap
items:
- name: docker/domdoc
vars:
index: test
time: 10s
content_pattern: "content3*"
content_and_pattern: "content3* make"
tasks:
- name: today
action: date
- name: search_any
action: search_any
- name: date
action: search_date
args:
today: today.result
- name: count
action: search_count
- name: content
action: search_content
- name: content_and
action: search_content_and