Compare commits

..

18 Commits

Author SHA1 Message Date
54610cdbb4 fix #93 crash index template, check nil ptr 2026-07-15 13:48:46 +02:00
T. von Dein
97a509da9e adopt latest golang enhancements (#91) 2026-07-13 14:33:41 +02:00
c5a21bd677 add esql sample 2026-07-10 15:14:23 +02:00
T. von Dein
f61fda0697 add feature esql search (#90) 2026-07-10 15:07:57 +02:00
T. von Dein
15dbf7ada0 fix api header spec (#89) 2026-07-10 13:58:56 +02:00
T. von Dein
311a8474d6 feature/node-usage (#88) 2026-07-10 13:37:59 +02:00
5644046c78 api repl: print output directly if not json 2026-07-10 11:28:25 +02:00
T. von Dein
9a42b86d62 add support for human readable /_cat endpoints: 'api repl -H' (#87) 2026-07-10 11:21:45 +02:00
T. von Dein
53abe666e8 fix 'node show' and 'shard explain' crashes (#86) 2026-07-09 13:41:07 +02:00
1e9e419e9b fix crash in ByteSize conversion 2026-07-08 15:07:01 +02:00
T. von Dein
c128e32ca1 add more important node stats (#82) 2026-07-08 15:03:31 +02:00
2969522913 rename var 2026-07-08 15:00:21 +02:00
51480e6b35 add 'index du <index>' 2026-07-08 14:58:07 +02:00
99bf703997 add table.AddRowLate() to add rows after sorting 2026-07-08 14:57:46 +02:00
T. von Dein
76ad6c7afa remove workaround to load config in root.Before() (#83)
see: https://github.com/urfave/cli/issues/2348
2026-07-08 14:13:47 +02:00
T. von Dein
8614e09004 show green and failed indices in status, rename red to failed in index ls (#81) 2026-07-08 10:23:01 +02:00
03cfeb4336 fix go tool 2026-07-08 09:34:58 +02:00
T. von Dein
3d3dfb7a42 fix #79: remove compatibility header (#80) 2026-07-08 09:33:22 +02:00
46 changed files with 1703 additions and 233 deletions

View File

@@ -16,7 +16,6 @@ builds:
goos:
- linux
- darwin
tool: "go1.26.4"
changelog:
sort: asc

View File

@@ -67,10 +67,10 @@ test: clean buildlocal
testlint: test lint
lint:
lint-basic:
golangci-lint run --enable-only errcheck,govet,ineffassign,staticcheck,unused
lint-full:
lint:
golangci-lint run --show-stats=false
testfuzzy: clean

View File

@@ -29,6 +29,7 @@ Features:
logical condition (OR, AND), use PIT, limit datetime (ES date math
can be used), etc. It is however not yet possible to create
recursive searches like: `(cond1 AND cond2) OR (cond3 OR cond4)`.
- Search using ES|QL language: `esctl searchql`.
- Cross cluster replication (ccr): view, pause, resume, delete
replication. You can also manage follower configuration.
- Index management: manage aliases, create, modify, delete indices,
@@ -352,6 +353,30 @@ $ esctl search -i foo* -F title=zeitbuchung message=pause | jq
}
```
You can also search using [ES|QL](https://www.elastic.co/docs/reference/query-languages/esql/esql-getting-started):
```console
$ esctl searchql "from hyperdrive | sort @timestamp | limit 5"
@TIMESTAMP MESSAGE TAG
2026-06-24T08:24:56.000Z arosu loop
2026-06-24T08:24:58.000Z hami loop
2026-06-24T08:24:59.000Z ishininu loop
2026-06-24T08:25:00.000Z uyomoruron loop
2026-06-24T08:25:02.000Z ishimime loop
```
There are several output modes (json, yaml, csv), to get esql output as CSV:
```console
$ esctl searchql "from hyperdrive | sort @timestamp | limit 5" -o csv
@timestamp,message,tag
2026-06-24T08:24:56.000Z,arosu,loop
2026-06-24T08:24:58.000Z,hami,loop
2026-06-24T08:24:59.000Z,ishininu,loop
2026-06-24T08:25:00.000Z,uyomoruron,loop
2026-06-24T08:25:02.000Z,ishimime,loop
```
To check which field mappings are available for an index:
```console
$ esctl index show foo2
@@ -538,6 +563,7 @@ index - manage indicies
close - close an index
fields - show info about field capabilities
ilm - show ilm status
du - show index disk usage
alias - manage index aliases
create - create an index alias
list - list index aliases
@@ -555,6 +581,7 @@ node - manage nodes
list - list nodes
show - show details about a node
clients - show node http clients
usage - show node usage stats
role - manage roles
list - list roles
show - show details about a role
@@ -574,6 +601,7 @@ version - show esctl version information
debug - developer only
help-jsonpath - show jsonpath help
help-usage - show overview of all available commands
help-esql - show esql help
```
# Development

View File

@@ -24,10 +24,11 @@ import (
"github.com/go-openapi/swag/loading"
)
//go:embed *.json
//go:embed *.json *.md
var AssetFS embed.FS
var OpenAPI *loads.Document
var EsQlCheatSheet string
func LoadAssetOpenApi() {
doc, err := loads.Spec(
@@ -41,3 +42,12 @@ func LoadAssetOpenApi() {
OpenAPI = doc
}
func LoadEsql() {
md, err := AssetFS.ReadFile("esql.md")
if err != nil {
panic(err)
}
EsQlCheatSheet = string(md)
}

951
assets/esql.md Normal file
View File

@@ -0,0 +1,951 @@
<!-- courtesy https://github.com/linkan-per/ES-QL-Cheat-Sheet by |linkan-per -->
# ES|QL (Elasticsearch Query Language) Cheat Sheet
## Table of Contents
1. [Introduction](#introduction)
2. [Basic Query Structure](#basic-query-structure)
3. [Source Commands](#source-commands)
4. [Processing Commands](#processing-commands)
5. [Data Selection & Filtering](#data-selection--filtering)
6. [Aggregations & Statistics](#aggregations--statistics)
7. [String Functions](#string-functions)
8. [Mathematical Functions](#mathematical-functions)
9. [Date/Time Functions](#datetime-functions)
10. [Type Conversion Functions](#type-conversion-functions)
11. [Conditional Functions](#conditional-functions)
12. [Array Functions](#array-functions)
13. [Join Operations (LOOKUP)](#join-operations-lookup)
14. [Sorting & Limiting](#sorting--limiting)
15. [Grouping & Aggregating](#grouping--aggregating)
16. [Advanced Patterns](#advanced-patterns)
---
## Introduction
ES|QL is Elasticsearch's new query language designed for data exploration, analysis, and transformation. It uses a pipe (`|`) syntax to chain commands together.
**Basic Syntax:**
```
FROM <data-source>
| <processing-command>
| <processing-command>
| ...
```
---
## Basic Query Structure
### Simple Query
```esql
FROM logs-*
| LIMIT 10
```
### Query with Multiple Commands
```esql
FROM employees
| WHERE department == "Engineering"
| KEEP name, salary, hire_date
| SORT salary DESC
| LIMIT 5
```
---
## Source Commands
### FROM - Specify Data Source
```esql
// From a single index
FROM logs-2024
// From multiple indices with wildcard
FROM logs-*, metrics-*
// From specific indices
FROM index1, index2, index3
// With metadata
FROM logs-* METADATA _id, _index
```
### ROW - Generate Inline Data
```esql
// Create a single row
ROW name = "John", age = 30, city = "NYC"
// Multiple rows
ROW a = 1, b = "x"
| EVAL c = a * 10
```
---
## Processing Commands
### KEEP - Select Specific Fields
```esql
FROM employees
| KEEP name, department, salary
// Keep with pattern
FROM logs-*
| KEEP @timestamp, message, host.*
```
### DROP - Remove Specific Fields
```esql
FROM employees
| DROP password, ssn, internal_notes
// Drop with pattern
FROM logs-*
| DROP *.keyword
```
### RENAME - Rename Fields
```esql
FROM employees
| RENAME emp_name AS name, emp_dept AS department
// Multiple renames
FROM logs-*
| RENAME source.ip AS src_ip, destination.ip AS dst_ip
```
---
## Data Selection & Filtering
### WHERE - Filter Rows
```esql
// Equality
FROM employees
| WHERE department == "Sales"
// Comparison operators
FROM products
| WHERE price > 100 AND stock < 50
// IS NULL / IS NOT NULL
FROM logs-*
| WHERE error_code IS NOT NULL
// IN operator
FROM employees
| WHERE department IN ("Sales", "Marketing", "HR")
// LIKE operator (wildcards)
FROM logs-*
| WHERE message LIKE "*error*"
// RLIKE operator (regex)
FROM logs-*
| WHERE message RLIKE "error|exception|failure"
// NOT operator
FROM employees
| WHERE NOT department == "IT"
// Multiple conditions
FROM orders
| WHERE status == "completed"
AND total_amount > 1000
AND order_date >= "2024-01-01"
```
---
## Aggregations & Statistics
### STATS - Aggregate Functions
#### COUNT
```esql
// Count all rows
FROM logs-*
| STATS count = COUNT()
// Count distinct
FROM employees
| STATS unique_departments = COUNT_DISTINCT(department)
// Count by group
FROM logs-*
| STATS event_count = COUNT() BY log_level
```
#### SUM, AVG, MIN, MAX
```esql
FROM sales
| STATS
total_revenue = SUM(amount),
avg_sale = AVG(amount),
min_sale = MIN(amount),
max_sale = MAX(amount)
// With grouping
FROM sales
| STATS
total = SUM(amount),
average = AVG(amount)
BY product_category
```
#### MEDIAN, PERCENTILE
```esql
FROM response_times
| STATS
median_time = MEDIAN(duration),
p95 = PERCENTILE(duration, 95),
p99 = PERCENTILE(duration, 99)
```
#### Multiple Aggregations
```esql
FROM orders
| STATS
order_count = COUNT(),
total_revenue = SUM(amount),
avg_order_value = AVG(amount),
unique_customers = COUNT_DISTINCT(customer_id)
BY region, product_category
```
---
## String Functions
### CONCAT - Concatenate Strings
```esql
FROM employees
| EVAL full_name = CONCAT(first_name, " ", last_name)
// With separator
FROM logs-*
| EVAL log_info = CONCAT(level, ": ", message)
```
### SUBSTRING - Extract Substring
```esql
FROM employees
| EVAL first_initial = SUBSTRING(first_name, 0, 1)
// Extract with length
FROM products
| EVAL short_code = SUBSTRING(product_id, 0, 5)
```
### LENGTH - String Length
```esql
FROM messages
| EVAL message_length = LENGTH(message)
| WHERE message_length > 100
```
### TRIM, LTRIM, RTRIM - Remove Whitespace
```esql
FROM user_input
| EVAL cleaned = TRIM(input_field)
| EVAL left_trimmed = LTRIM(input_field)
| EVAL right_trimmed = RTRIM(input_field)
```
### UPPER, LOWER - Case Conversion
```esql
FROM employees
| EVAL name_upper = UPPER(name)
| EVAL email_lower = LOWER(email)
```
### REPLACE - Replace String
```esql
FROM logs-*
| EVAL cleaned_message = REPLACE(message, "ERROR", "Warning")
```
### SPLIT - Split String into Array
```esql
FROM logs-*
| EVAL tags_array = SPLIT(tags, ",")
```
### STARTS_WITH, ENDS_WITH
```esql
FROM files
| WHERE STARTS_WITH(filename, "log_")
| WHERE ENDS_WITH(filename, ".txt")
```
---
## Mathematical Functions
### Basic Operations
```esql
FROM sales
| EVAL
total = price * quantity,
discount_price = price * 0.9,
tax = price * 0.08
// Multiple operations
FROM metrics
| EVAL
sum_val = field1 + field2,
diff_val = field1 - field2,
product = field1 * field2,
ratio = field1 / field2,
remainder = field1 % field2
```
### ABS - Absolute Value
```esql
FROM transactions
| EVAL abs_amount = ABS(transaction_amount)
```
### ROUND, FLOOR, CEIL
```esql
FROM measurements
| EVAL
rounded = ROUND(value, 2),
floored = FLOOR(value),
ceiled = CEIL(value)
```
### POW - Power
```esql
FROM data
| EVAL squared = POW(value, 2)
| EVAL cubed = POW(value, 3)
```
### SQRT - Square Root
```esql
FROM measurements
| EVAL sqrt_value = SQRT(value)
```
### LOG, LOG10
```esql
FROM data
| EVAL
natural_log = LOG(value),
log_base_10 = LOG10(value)
```
### GREATEST, LEAST
```esql
FROM comparisons
| EVAL
max_val = GREATEST(val1, val2, val3),
min_val = LEAST(val1, val2, val3)
```
---
## Date/Time Functions
### NOW - Current Timestamp
```esql
FROM logs-*
| EVAL current_time = NOW()
```
### DATE_EXTRACT - Extract Date Parts
```esql
FROM events
| EVAL
year = DATE_EXTRACT("year", @timestamp),
month = DATE_EXTRACT("month", @timestamp),
day = DATE_EXTRACT("day", @timestamp),
hour = DATE_EXTRACT("hour", @timestamp),
minute = DATE_EXTRACT("minute", @timestamp),
day_of_week = DATE_EXTRACT("day_of_week", @timestamp)
```
### DATE_FORMAT - Format Date
```esql
FROM events
| EVAL formatted_date = DATE_FORMAT("yyyy-MM-dd", @timestamp)
| EVAL custom_format = DATE_FORMAT("MMM dd, yyyy HH:mm", @timestamp)
```
### DATE_TRUNC - Truncate Date
```esql
FROM logs-*
| EVAL
hour_bucket = DATE_TRUNC("hour", @timestamp),
day_bucket = DATE_TRUNC("day", @timestamp),
month_bucket = DATE_TRUNC("month", @timestamp)
```
### DATE_DIFF - Date Difference
```esql
FROM orders
| EVAL days_since_order = DATE_DIFF("days", order_date, NOW())
| EVAL hours_to_delivery = DATE_DIFF("hours", order_date, delivery_date)
```
### DATE_PARSE - Parse String to Date
```esql
FROM data
| EVAL parsed_date = DATE_PARSE("yyyy-MM-dd", date_string)
```
---
## Type Conversion Functions
### TO_STRING - Convert to String
```esql
FROM data
| EVAL id_string = TO_STRING(id)
| EVAL amount_string = TO_STRING(amount)
```
### TO_INTEGER, TO_LONG - Convert to Integer
```esql
FROM data
| EVAL age_int = TO_INTEGER(age_string)
| EVAL id_long = TO_LONG(id_string)
```
### TO_DOUBLE - Convert to Double
```esql
FROM data
| EVAL price_double = TO_DOUBLE(price_string)
```
### TO_BOOLEAN - Convert to Boolean
```esql
FROM data
| EVAL is_active = TO_BOOLEAN(active_string)
```
### TO_DATETIME - Convert to DateTime
```esql
FROM data
| EVAL timestamp = TO_DATETIME(date_string)
```
### TO_IP - Convert to IP Address
```esql
FROM logs-*
| EVAL ip_address = TO_IP(ip_string)
```
---
## Conditional Functions
### CASE - Conditional Logic
```esql
FROM employees
| EVAL salary_grade = CASE(
salary < 50000, "Entry",
salary < 80000, "Mid",
salary < 120000, "Senior",
"Executive"
)
// With multiple conditions
FROM orders
| EVAL order_status = CASE(
status == "pending" AND days_old > 7, "Overdue",
status == "pending", "Processing",
status == "shipped", "In Transit",
status == "delivered", "Completed",
"Unknown"
)
```
### COALESCE - Return First Non-Null Value
```esql
FROM data
| EVAL display_name = COALESCE(nickname, first_name, username, "Unknown")
```
### IF - Simple Conditional
```esql
FROM products
| EVAL stock_status =
CASE(stock > 0, "Available", "Out of Stock")
// Nested conditions
FROM employees
| EVAL bonus = CASE(
performance_rating >= 4.5, salary * 0.15,
performance_rating >= 3.5, salary * 0.10,
performance_rating >= 2.5, salary * 0.05,
0
)
```
---
## Array Functions
### MV_COUNT - Count Array Elements
```esql
FROM logs-*
| EVAL tag_count = MV_COUNT(tags)
| WHERE tag_count > 3
```
### MV_AVG, MV_SUM, MV_MIN, MV_MAX - Array Aggregations
```esql
FROM metrics
| EVAL
avg_value = MV_AVG(values),
total = MV_SUM(values),
min_value = MV_MIN(values),
max_value = MV_MAX(values)
```
### MV_CONCAT - Concatenate Array Elements
```esql
FROM logs-*
| EVAL all_tags = MV_CONCAT(tags, ", ")
```
### MV_DEDUPE - Remove Duplicates from Array
```esql
FROM data
| EVAL unique_values = MV_DEDUPE(values)
```
### MV_FIRST, MV_LAST - Get First/Last Element
```esql
FROM logs-*
| EVAL first_tag = MV_FIRST(tags)
| EVAL last_tag = MV_LAST(tags)
```
### MV_SLICE - Extract Array Slice
```esql
FROM data
| EVAL first_three = MV_SLICE(values, 0, 3)
```
---
## Join Operations (LOOKUP)
ES|QL uses ENRICH (similar to LOOKUP/JOIN) to join data from enrich policies.
### Prerequisites: Create Enrich Policy
First, create an enrich policy in Kibana Dev Tools:
```json
PUT /_enrich/policy/user_lookup
{
"match": {
"indices": "users",
"match_field": "user_id",
"enrich_fields": ["username", "email", "department"]
}
}
POST /_enrich/policy/user_lookup/_execute
```
### ENRICH - Join/Lookup Data
```esql
FROM logs-*
| ENRICH user_lookup ON user_id
| KEEP @timestamp, user_id, username, email, message
// With field renaming
FROM transactions
| ENRICH product_lookup ON product_id WITH product_name, category, price
| KEEP transaction_id, product_name, category, quantity, price
// Multiple enrichments
FROM orders
| ENRICH customer_lookup ON customer_id WITH customer_name, customer_tier
| ENRICH product_lookup ON product_id WITH product_name, product_category
| KEEP order_id, customer_name, product_name, order_amount
```
### Complex Join Example
```esql
FROM orders
| ENRICH customer_lookup ON customer_id
WITH customer_name, customer_email, customer_segment
| ENRICH product_lookup ON product_id
WITH product_name, product_category, product_price
| EVAL total_price = quantity * product_price
| WHERE customer_segment == "Premium"
| STATS
total_orders = COUNT(),
total_revenue = SUM(total_price)
BY customer_name, product_category
| SORT total_revenue DESC
```
---
## Sorting & Limiting
### SORT - Order Results
```esql
// Ascending order (default)
FROM employees
| SORT salary
// Descending order
FROM employees
| SORT salary DESC
// Multiple fields
FROM employees
| SORT department ASC, salary DESC
// With nulls first/last
FROM data
| SORT value DESC NULLS FIRST
```
### LIMIT - Limit Results
```esql
// Get first 10 rows
FROM logs-*
| LIMIT 10
// Top 5 highest salaries
FROM employees
| SORT salary DESC
| LIMIT 5
// Pagination (skip and limit)
FROM products
| SORT price
| LIMIT 20 // Results 0-19
```
### HEAD - Get First N Rows (Alias for LIMIT)
```esql
FROM logs-*
| HEAD 100
```
---
## Grouping & Aggregating
### GROUP BY with STATS
```esql
// Single field grouping
FROM sales
| STATS total_sales = SUM(amount) BY region
// Multiple field grouping
FROM orders
| STATS
order_count = COUNT(),
total_revenue = SUM(amount)
BY region, product_category, sales_rep
// Time-based grouping
FROM logs-*
| EVAL hour = DATE_TRUNC("hour", @timestamp)
| STATS event_count = COUNT() BY hour, log_level
| SORT hour DESC
```
### Complex Aggregation Example
```esql
FROM sales_data
| WHERE order_date >= "2024-01-01"
| EVAL month = DATE_TRUNC("month", order_date)
| STATS
total_orders = COUNT(),
total_revenue = SUM(amount),
avg_order_value = AVG(amount),
unique_customers = COUNT_DISTINCT(customer_id),
max_order = MAX(amount),
min_order = MIN(amount)
BY month, region, product_category
| EVAL revenue_per_customer = total_revenue / unique_customers
| WHERE total_orders > 100
| SORT month DESC, total_revenue DESC
| LIMIT 50
```
---
## Advanced Patterns
### Window Functions Pattern
```esql
// Running total by group
FROM sales
| SORT date
| STATS
daily_sales = SUM(amount),
order_count = COUNT()
BY date, region
| SORT region, date
```
### Pivoting Data
```esql
// Count by status and priority
FROM tickets
| STATS ticket_count = COUNT() BY status, priority
| SORT status, priority
```
### Finding Duplicates
```esql
FROM users
| STATS count = COUNT() BY email
| WHERE count > 1
| SORT count DESC
```
### Time Series Analysis
```esql
FROM metrics-*
| EVAL
hour = DATE_TRUNC("hour", @timestamp),
day = DATE_EXTRACT("day", @timestamp)
| STATS
avg_cpu = AVG(cpu_percent),
max_cpu = MAX(cpu_percent),
avg_memory = AVG(memory_percent)
BY hour, host
| WHERE avg_cpu > 80
| SORT hour DESC
```
### Percentage Calculations
```esql
FROM sales
| STATS
total_sales = SUM(amount),
count = COUNT()
BY product_category
| EVAL percentage = ROUND(total_sales / SUM(total_sales) * 100, 2)
| SORT percentage DESC
```
### Top N per Group
```esql
// Top 3 products per category by sales
FROM sales
| STATS total_sales = SUM(amount) BY product_category, product_name
| SORT product_category, total_sales DESC
// Note: ES|QL doesn't have native PARTITION BY,
// so you may need to process this in multiple queries or use aggregations
```
### Data Cleaning
```esql
FROM raw_data
| EVAL
// Clean whitespace
cleaned_name = TRIM(name),
// Standardize case
email_lower = LOWER(email),
// Replace values
status = REPLACE(status, "N/A", "Unknown"),
// Handle nulls
age = COALESCE(age, 0),
// Validate ranges
valid_age = CASE(age < 0 OR age > 150, NULL, age)
| WHERE cleaned_name IS NOT NULL
| DROP name, email
| RENAME cleaned_name AS name, email_lower AS email
```
### Cohort Analysis
```esql
FROM user_events
| EVAL
signup_month = DATE_TRUNC("month", signup_date),
event_month = DATE_TRUNC("month", event_date)
| STATS
active_users = COUNT_DISTINCT(user_id)
BY signup_month, event_month
| SORT signup_month, event_month
```
### Anomaly Detection Pattern
```esql
FROM metrics-*
| EVAL hour = DATE_TRUNC("hour", @timestamp)
| STATS
avg_value = AVG(value),
stddev = SQRT(AVG(POW(value - AVG(value), 2)))
BY hour
| EVAL
upper_bound = avg_value + (2 * stddev),
lower_bound = avg_value - (2 * stddev)
```
---
## Complete Real-World Examples
### Example 1: User Activity Dashboard
```esql
FROM user_logs-*
| WHERE @timestamp >= NOW() - 7 days
| ENRICH user_lookup ON user_id WITH username, user_tier
| EVAL day = DATE_TRUNC("day", @timestamp)
| STATS
daily_active_users = COUNT_DISTINCT(user_id),
total_sessions = COUNT(),
avg_session_duration = AVG(session_duration)
BY day, user_tier
| EVAL avg_duration_minutes = ROUND(avg_session_duration / 60, 2)
| SORT day DESC, user_tier
```
### Example 2: E-commerce Sales Report
```esql
FROM orders
| WHERE order_date >= "2024-01-01"
| ENRICH customer_lookup ON customer_id
WITH customer_name, customer_segment
| ENRICH product_lookup ON product_id
WITH product_name, product_category, cost_price
| EVAL
profit = (price - cost_price) * quantity,
month = DATE_TRUNC("month", order_date)
| STATS
total_orders = COUNT(),
total_revenue = SUM(price * quantity),
total_profit = SUM(profit),
avg_order_value = AVG(price * quantity),
unique_customers = COUNT_DISTINCT(customer_id)
BY month, product_category, customer_segment
| EVAL profit_margin = ROUND(total_profit / total_revenue * 100, 2)
| WHERE total_revenue > 10000
| SORT month DESC, total_revenue DESC
| LIMIT 100
```
### Example 3: Security Log Analysis
```esql
FROM security-logs-*
| WHERE @timestamp >= NOW() - 24 hours
| WHERE event_type IN ("login_failed", "suspicious_activity")
| EVAL hour = DATE_TRUNC("hour", @timestamp)
| STATS
event_count = COUNT(),
unique_ips = COUNT_DISTINCT(source_ip),
unique_users = COUNT_DISTINCT(username)
BY hour, event_type, country
| WHERE event_count > 100
| SORT hour DESC, event_count DESC
```
### Example 4: Application Performance Monitoring
```esql
FROM apm-*
| WHERE @timestamp >= NOW() - 1 hour
| EVAL
response_category = CASE(
response_time < 100, "Fast",
response_time < 500, "Medium",
response_time < 1000, "Slow",
"Very Slow"
),
minute = DATE_TRUNC("minute", @timestamp)
| STATS
request_count = COUNT(),
avg_response = AVG(response_time),
p95_response = PERCENTILE(response_time, 95),
p99_response = PERCENTILE(response_time, 99),
error_count = COUNT() WHERE status_code >= 400
BY minute, endpoint, response_category
| EVAL error_rate = ROUND(error_count / request_count * 100, 2)
| WHERE request_count > 10
| SORT minute DESC, avg_response DESC
```
---
## Tips & Best Practices
1. **Use KEEP instead of SELECT** - More explicit about which fields to retain
2. **Filter early with WHERE** - Reduce data processing by filtering before aggregations
3. **Use DATE_TRUNC for time bucketing** - Essential for time series analysis
4. **Leverage ENRICH for joins** - Pre-create enrich policies for frequently joined data
5. **Use EVAL for calculated fields** - Create derived fields before aggregation
6. **Combine multiple conditions in WHERE** - More efficient than multiple WHERE clauses
7. **Use STATS with BY for grouping** - Replaces traditional GROUP BY
8. **Sort after aggregation** - More efficient than sorting before
9. **Use LIMIT to control output size** - Especially important for large datasets
10. **Use metadata fields when needed** - Access _id, _index with METADATA keyword
---
## Common Patterns Cheat Sheet
```esql
// Count by field
FROM index | STATS count = COUNT() BY field
// Top N
FROM index | STATS value = SUM(amount) BY category | SORT value DESC | LIMIT 10
// Time series
FROM index | EVAL bucket = DATE_TRUNC("hour", @timestamp) | STATS count = COUNT() BY bucket
// Percentage of total
FROM index | STATS total = SUM(amount) BY category | EVAL pct = total / SUM(total) * 100
// Filter nulls
FROM index | WHERE field IS NOT NULL
// String matching
FROM index | WHERE field LIKE "*pattern*"
// Date range
FROM index | WHERE @timestamp >= NOW() - 7 days
// Multiple aggregations
FROM index | STATS count = COUNT(), sum = SUM(val), avg = AVG(val) BY group
// Conditional aggregation
FROM index | STATS error_count = COUNT() WHERE status == "error" BY service
```
---
## Comparison with Traditional SQL
| SQL | ES|QL |
|-----|-------|
| SELECT * | FROM index |
| SELECT field1, field2 | FROM index \| KEEP field1, field2 |
| WHERE condition | WHERE condition (same) |
| GROUP BY field | STATS ... BY field |
| ORDER BY field | SORT field |
| LIMIT 10 | LIMIT 10 (same) |
| COUNT(*) | STATS count = COUNT() |
| SUM(field) | STATS total = SUM(field) |
| AVG(field) | STATS avg = AVG(field) |
| JOIN | ENRICH (using enrich policies) |
| CASE WHEN | CASE(...) |
| CONCAT(a, b) | CONCAT(a, b) (same) |
---
## Resources
- Official ES|QL Documentation: https://www.elastic.co/guide/en/elasticsearch/reference/current/esql.html
- ES|QL Functions Reference: https://www.elastic.co/guide/en/elasticsearch/reference/current/esql-functions.html
- Enrich Processor: https://www.elastic.co/guide/en/elasticsearch/reference/current/enrich-processor.html
---
*Last Updated: 2024*
*ES|QL is actively evolving - check official documentation for latest features*

View File

@@ -72,7 +72,7 @@ func ApiShow(conf *cfg.Config) *cli.Command {
},
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Capi)
complete(conf, cmd, Capi)
},
}
}
@@ -91,6 +91,12 @@ func ApiRepl(conf *cfg.Config) *cli.Command {
Aliases: []string{"p"},
Sources: cli.EnvVars("PAGER", "ES_JSON_PAGER"),
},
&cli.BoolFlag{
Name: "human-readable-cat",
Usage: "enable human readable /_cat output",
Destination: &conf.HumanCat,
Aliases: []string{"H"},
},
},
Action: func(ctx context.Context, cmd *cli.Command) error {

View File

@@ -59,7 +59,7 @@ func CcrStatus(conf *cfg.Config) *cli.Command {
},
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Ccluster)
complete(conf, cmd, Ccluster)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
@@ -115,7 +115,7 @@ func CcrRemoteInfo(conf *cfg.Config) *cli.Command {
UsageText: "info [options] [<index>]",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cindex)
complete(conf, cmd, Cindex)
},
Action: func(ctx context.Context, cmd *cli.Command) error {

View File

@@ -60,7 +60,7 @@ func CcrFollowerRenew(conf *cfg.Config) *cli.Command {
},
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cindex)
complete(conf, cmd, Cindex)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
@@ -82,7 +82,7 @@ func CcrFollowerResume(conf *cfg.Config) *cli.Command {
UsageText: "resume [options] <index>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cindex)
complete(conf, cmd, Cindex)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
@@ -104,7 +104,7 @@ func CcrFollowerPause(conf *cfg.Config) *cli.Command {
UsageText: "pause [options] <index>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cindex)
complete(conf, cmd, Cindex)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
@@ -126,7 +126,7 @@ func CcrFollowerUnfollow(conf *cfg.Config) *cli.Command {
UsageText: "unfollow [options] <index>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cindex)
complete(conf, cmd, Cindex)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
@@ -158,7 +158,7 @@ func CcrFollowerAdd(conf *cfg.Config) *cli.Command {
},
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cindex)
complete(conf, cmd, Cindex)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
@@ -181,7 +181,7 @@ func CcrFollowerDelete(conf *cfg.Config) *cli.Command {
UsageText: "delete <index>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cindex)
complete(conf, cmd, Cindex)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
@@ -205,7 +205,7 @@ func CcrFollowerShow(conf *cfg.Config) *cli.Command {
UsageText: "show <index>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cindex)
complete(conf, cmd, Cindex)
},
Action: func(ctx context.Context, cmd *cli.Command) error {

View File

@@ -83,7 +83,7 @@ func ClusterSwitch(conf *cfg.Config) *cli.Command {
Aliases: []string{"ctx"},
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Ccluster)
complete(conf, cmd, Ccluster)
},
Action: func(ctx context.Context, cmd *cli.Command) error {

View File

@@ -50,7 +50,7 @@ func ClusterRerouteMove(conf *cfg.Config) *cli.Command {
UsageText: "move [options] <index>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cindex)
complete(conf, cmd, Cindex)
},
Flags: []cli.Flag{
@@ -95,7 +95,7 @@ func ClusterRerouteAllocateReplica(conf *cfg.Config) *cli.Command {
UsageText: "allocate-replica [options] <index>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cindex)
complete(conf, cmd, Cindex)
},
Flags: []cli.Flag{
@@ -133,7 +133,7 @@ func ClusterRerouteCancel(conf *cfg.Config) *cli.Command {
UsageText: "cancel [options] <index>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cindex)
complete(conf, cmd, Cindex)
},
Flags: []cli.Flag{
@@ -185,7 +185,7 @@ func ClusterRerouteAllocatePrimary(conf *cfg.Config, stale bool) *cli.Command {
UsageText: name + " [options] <index>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cindex)
complete(conf, cmd, Cindex)
},
Flags: []cli.Flag{

View File

@@ -105,7 +105,7 @@ func ClusterSettingsSet(conf *cfg.Config) *cli.Command {
},
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cclustersettings)
complete(conf, cmd, Cclustersettings)
},
Action: func(ctx context.Context, cmd *cli.Command) error {

View File

@@ -36,18 +36,11 @@ const (
Cclustersettings
)
func complete(cmd *cli.Command, what int) {
func complete(conf *cfg.Config, cmd *cli.Command, what int) {
if cmd.NArg() > 0 {
return
}
// FIXME: config should load from root.Before(), see https://github.com/urfave/cli/issues/2348
// workaround: load it directly here
conf := cfg.NewConfig()
if err := conf.Init(); err != nil {
return
}
var (
list []string
err error

View File

@@ -91,7 +91,7 @@ func DatastreamShow(conf *cfg.Config) *cli.Command {
},
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cdatastream)
complete(conf, cmd, Cdatastream)
},
}
}
@@ -113,7 +113,7 @@ func DatastreamCreate(conf *cfg.Config) *cli.Command {
},
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cdatastream)
complete(conf, cmd, Cdatastream)
},
}
}
@@ -135,7 +135,7 @@ func DatastreamDelete(conf *cfg.Config) *cli.Command {
},
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cdatastream)
complete(conf, cmd, Cdatastream)
},
}
}
@@ -148,7 +148,7 @@ func DatastreamRollover(conf *cfg.Config) *cli.Command {
UsageText: "rollover <data stream>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cindex)
complete(conf, cmd, Cindex)
},
Flags: []cli.Flag{
@@ -205,7 +205,7 @@ func DatastreamIlm(conf *cfg.Config) *cli.Command {
UsageText: "ds ilm <name>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cdatastream)
complete(conf, cmd, Cdatastream)
},
Action: func(ctx context.Context, cmd *cli.Command) error {

View File

@@ -123,7 +123,7 @@ func IlmShow(conf *cfg.Config) *cli.Command {
},
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cilm)
complete(conf, cmd, Cilm)
},
}
}

View File

@@ -41,6 +41,7 @@ func Index(conf *cfg.Config) *cli.Command {
IndexClose(conf),
IndexFields(conf),
IndexIlm(conf),
IndexDu(conf),
// sub commands
IndexAlias(conf),
@@ -75,10 +76,10 @@ func IndexList(conf *cfg.Config) *cli.Command {
Aliases: []string{"H"},
},
&cli.BoolFlag{
Name: "reds",
Name: "failed",
Usage: "include only red failed indicies",
Destination: &conf.Failed,
Aliases: []string{"r"},
Aliases: []string{"f"},
},
&cli.StringSliceFlag{
Name: "filter",
@@ -110,7 +111,7 @@ func IndexShow(conf *cfg.Config) *cli.Command {
},
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cindex)
complete(conf, cmd, Cindex)
},
}
}
@@ -180,7 +181,7 @@ func IndexDelete(conf *cfg.Config) *cli.Command {
UsageText: "delete <index>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cindex)
complete(conf, cmd, Cindex)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
@@ -201,7 +202,7 @@ func IndexClose(conf *cfg.Config) *cli.Command {
UsageText: "close <index>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cindex)
complete(conf, cmd, Cindex)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
@@ -243,7 +244,7 @@ func IndexFields(conf *cfg.Config) *cli.Command {
},
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cindex)
complete(conf, cmd, Cindex)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
@@ -264,7 +265,7 @@ func IndexIlm(conf *cfg.Config) *cli.Command {
UsageText: "index ilm <index>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cindex)
complete(conf, cmd, Cindex)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
@@ -277,3 +278,24 @@ func IndexIlm(conf *cfg.Config) *cli.Command {
},
}
}
func IndexDu(conf *cfg.Config) *cli.Command {
return &cli.Command{
Name: "du",
Usage: "show index disk usage",
UsageText: "index du <index>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cindex)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
index := cmd.Args().Get(0)
if index == "" {
return errors.New("no index specified")
}
return es.IndexDiskusage(conf, index)
},
}
}

View File

@@ -52,7 +52,7 @@ func IndexAliasCreate(conf *cfg.Config) *cli.Command {
UsageText: "create <index> <alias>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cindex)
complete(conf, cmd, Cindex)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
@@ -76,7 +76,7 @@ func IndexAliasDelete(conf *cfg.Config) *cli.Command {
UsageText: "delete <index> <alias>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cindex)
complete(conf, cmd, Cindex)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
@@ -100,7 +100,7 @@ func IndexAliasRollover(conf *cfg.Config) *cli.Command {
UsageText: "rollover <alias>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cindex)
complete(conf, cmd, Cindex)
},
Flags: []cli.Flag{

View File

@@ -71,7 +71,7 @@ func IndexTemplateShow(conf *cfg.Config) *cli.Command {
},
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cindex)
complete(conf, cmd, Cindex)
},
}
}
@@ -215,7 +215,7 @@ func IndexTemplateDelete(conf *cfg.Config) *cli.Command {
},
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cindex)
complete(conf, cmd, Cindex)
},
}
}

View File

@@ -36,6 +36,7 @@ func Node(conf *cfg.Config) *cli.Command {
NodeList(conf),
NodeShow(conf),
NodeClients(conf),
NodeUsage(conf),
},
}
}
@@ -60,7 +61,7 @@ func NodeShow(conf *cfg.Config) *cli.Command {
UsageText: "show [options] <node>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cnode)
complete(conf, cmd, Cnode)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
@@ -90,7 +91,7 @@ func NodeClients(conf *cfg.Config) *cli.Command {
},
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Cnode)
complete(conf, cmd, Cnode)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
@@ -103,3 +104,19 @@ func NodeClients(conf *cfg.Config) *cli.Command {
},
}
}
func NodeUsage(conf *cfg.Config) *cli.Command {
return &cli.Command{
Name: "usage",
Usage: "show node usage stats",
UsageText: "usage [options] [<node>]",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(conf, cmd, Cnode)
},
Action: func(ctx context.Context, cmd *cli.Command) error {
return es.NodeUsage(conf, cmd.Args().Get(0))
},
}
}

View File

@@ -68,7 +68,7 @@ func RoleShow(conf *cfg.Config) *cli.Command {
UsageText: "show [options] <role>",
ShellComplete: func(ctx context.Context, cmd *cli.Command) {
complete(cmd, Crole)
complete(conf, cmd, Crole)
},
Action: func(ctx context.Context, cmd *cli.Command) error {

View File

@@ -21,13 +21,17 @@ import (
"fmt"
golog "log"
"os"
"runtime/debug"
"runtime/pprof"
"strings"
"codeberg.org/scip/esctl/assets"
"codeberg.org/scip/esctl/pkg/cfg"
"codeberg.org/scip/esctl/pkg/es"
"codeberg.org/scip/esctl/pkg/log"
"codeberg.org/scip/esctl/pkg/printer"
markdown "github.com/MichaelMure/go-term-markdown"
"github.com/urfave/cli/v3"
)
@@ -115,6 +119,12 @@ func Main() int {
Usage: "enable HTTP debugging",
Destination: &conf.DebugHTTP,
},
&cli.BoolFlag{
Name: "debug-goroutines",
Value: false,
Usage: "enable goroutine debugging",
Destination: &conf.DebugGoRoutines,
},
&cli.BoolFlag{
Name: "align-ints",
Aliases: []string{"I"},
@@ -141,7 +151,7 @@ func Main() int {
Name: "output",
Aliases: []string{"o"},
Value: "",
Usage: "output mode (tsv, json, yaml) default: tsv",
Usage: "output mode (tsv, csv, json, yaml) default: tsv",
Destination: &conf.Output,
},
&cli.StringFlag{
@@ -164,6 +174,7 @@ func Main() int {
Node(conf),
Roles(conf),
Search(conf),
SearchQL(conf),
Shard(conf),
Snapshot(conf),
Task(conf),
@@ -171,6 +182,7 @@ func Main() int {
Debug(conf),
HelpJsonPath(conf),
HelpUsage(conf),
HelpEsQl(conf),
},
Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) {
@@ -226,12 +238,34 @@ func HelpJsonPath(conf *cfg.Config) *cli.Command {
}
}
func HelpEsQl(conf *cfg.Config) *cli.Command {
return &cli.Command{
Name: "help-esql",
Usage: "show esql help",
Action: func(ctx context.Context, cmd *cli.Command) error {
assets.LoadEsql()
width := cfg.GetTermWidth()
printer.Pager("esql cheat sheet", string(markdown.Render(assets.EsQlCheatSheet, width, cfg.DefaultMargin)))
return nil
},
}
}
func Version(conf *cfg.Config) *cli.Command {
return &cli.Command{
Name: "version",
Usage: "show esctl version information",
Action: func(ctx context.Context, cmd *cli.Command) error {
info, _ := debug.ReadBuildInfo()
if strings.Contains(info.Main.Version, "+dirty") {
cfg.COMMIT += "+dirty"
}
_, err := fmt.Printf(versionFmt,
cfg.Version, cfg.BUILD, cfg.BRANCH, cfg.COMMIT, cfg.GOVERSION, cfg.APIVERSION)

View File

@@ -18,6 +18,7 @@ package cmd
import (
"context"
"strings"
"codeberg.org/scip/esctl/pkg/cfg"
"codeberg.org/scip/esctl/pkg/es"
@@ -45,6 +46,11 @@ https://www.elastic.co/docs/reference/elasticsearch/rest-apis/common-options#dat
For timestamp formats refer to:
https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/mapping-date-format`
const QlUsage = `ES|QL documentation:
https://www.elastic.co/docs/reference/query-languages/esql/esql-getting-started
See also: esctl help-esql`
func Search(conf *cfg.Config) *cli.Command {
return &cli.Command{
Name: "search",
@@ -149,3 +155,19 @@ func Search(conf *cfg.Config) *cli.Command {
},
}
}
func SearchQL(conf *cfg.Config) *cli.Command {
return &cli.Command{
Name: "searchql",
Aliases: []string{"/"},
Usage: "search using ES/QL language",
UsageText: "search <ES/QL term>",
CustomHelpTemplate: addReference(QlUsage),
Action: func(ctx context.Context, cmd *cli.Command) error {
args := cmd.Args()
return es.SearchQL(conf, strings.Join(args.Slice(), " "))
},
}
}

View File

@@ -194,18 +194,18 @@ func (cluster *Cluster) getDefaultOptions() []elasticsearch.Option {
}
func (cluster *Cluster) getTransport() elastictransport.Option {
transport := &http.Transport{
transport := new(http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
})
if cluster.DebugHTTP {
return elastictransport.WithTransport(
&DebugTransport{Transport: transport},
new(DebugTransport{Transport: transport}),
)
}
return elastictransport.WithTransport(
&CompatibilityTransport{Transport: transport},
new(CompatibilityTransport{Transport: transport}),
)
}

View File

@@ -28,7 +28,7 @@ import (
)
const (
Version string = `v0.0.26`
Version string = `v0.0.27`
)
var (
@@ -82,25 +82,28 @@ type Config struct {
SortBy string // sort: -k
Ascending bool // sort: -a
All bool // doc + node ls: -a
Exclude string // cluster compare: -e (regexp)
All, Verbose bool // cluster status: -a -v
Verbose bool // cluster status: -v
Persistent, Transient, Default bool // cluster settings set: -p -t -D
Force bool // ccr follower renew: -f
DebugHTTP bool // root: --debug-http
Separator string // role diff: -s
NotDeployed bool // role diff: -n
Undefined bool // role diff: -u
Diff bool // role diff: -D
Hidden bool // ds ls: -H
DebugHTTP bool // root: --debug-http
DebugGoRoutines bool // root: --debug-goroutines
Separator string // role diff: -s
NotDeployed bool // role diff: -n
Undefined bool // role diff: -u
Diff bool // role diff: -D
Hidden bool // ds ls: -H
// rollover
MaxAge string
MaxDocs, MaxShardSize, MaxShardDocs int // roll over
DryRun bool // rollover: -n
Tag string // api ls: -t
Tag string // api ls: -t
HumanCat bool // api repl: -H
Ilm Ilm // ilm create
@@ -111,7 +114,7 @@ type Config struct {
}
func NewConfig() *Config {
return &Config{Clusters: map[string]*Cluster{}}
return new(Config{Clusters: map[string]*Cluster{}})
}
func getDefaultPath() string {
@@ -214,7 +217,7 @@ func (conf *Config) LoadConfig() error {
return fmt.Errorf("failed to read config file: %w", err)
}
newconf := &Config{}
newconf := new(Config{})
err = yaml.Unmarshal(data, newconf)
if err != nil {

View File

@@ -53,10 +53,7 @@ func (t *DebugTransport) RoundTrip(req *http.Request) (*http.Response, error) {
contentline = buf.String()
}
if req.Header.Get("Accept") != "" {
req.Header.Del("Accept")
req.Header.Add("Accept", "application/json")
}
req = fixRequestHeaders(req)
slog.Info("req", "host", req.URL.Host, "uri", req.URL.Path,
"body", content, "bodyline", contentline,
@@ -66,20 +63,31 @@ func (t *DebugTransport) RoundTrip(req *http.Request) (*http.Response, error) {
return t.Transport.RoundTrip(req)
}
// Fixes https://codeberg.org/scip/esctl/issues/79:
// the API client has this Accept header hardcoded everywhere:
// req.Header.Set("Accept", "application/vnd.elasticsearch+json;compatible-with=9")
// While this works pretty well with ES9, it doesn't with ES8. So, we replace
// this header with a new one without the compatibility part.
type CompatibilityTransport struct {
Transport http.RoundTripper
}
func (t *CompatibilityTransport) RoundTrip(req *http.Request) (*http.Response, error) {
req = fixRequestHeaders(req)
return t.Transport.RoundTrip(req)
}
// Fixes https://codeberg.org/scip/esctl/issues/79:
// the API client has this Accept header hardcoded everywhere:
// req.Header.Set("Accept", "application/vnd.elasticsearch+json;compatible-with=9")
// While this works pretty well with ES9, it doesn't with ES8. So, we replace
// this header with a new one without the compatibility part.
func fixRequestHeaders(req *http.Request) *http.Request {
if req.Header.Get("Accept") != "" {
req.Header.Del("Accept")
req.Header.Add("Accept", "application/json")
}
return t.Transport.RoundTrip(req)
if req.Header.Get("Content-Type") != "" {
req.Header.Del("Content-Type")
req.Header.Add("Content-Type", "application/json")
}
return req
}

View File

@@ -28,6 +28,7 @@ import (
"io"
"log"
"log/slog"
"maps"
"net/http"
"os"
"os/exec"
@@ -45,22 +46,25 @@ import (
)
const (
intro = `Input format: verb path [data]"
Example:
post /yourindex/_ccr/pause_follow
put /yourindex/_settings {"number_of_replicas": 1}
You can also put multiline JSON after the path like:
put /yourindex/_settings
{
"number_of_replicas": 1
}
If you do NOT supply a JSON in the first line, you need to hit ENTER
twice to complete.`
intro = `# Input format: verb path [data]"
#
# Example:
#
# post /yourindex/_ccr/pause_follow
# put /yourindex/_settings {"number_of_replicas": 1}
#
# You can also put multiline JSON after the path like:
#
# put /yourindex/_settings
# {
# "number_of_replicas": 1
# }
#
# If you do NOT supply a JSON in the first line, you need to hit ENTER
# twice to complete.
#
# Supply the flag --human-readable-cat, -H to view /_cat API calls in
# human readable form.`
)
// holds an API operation via go-openapi/spec
@@ -143,7 +147,11 @@ func ApiRepl(conf *cfg.Config) error {
fmt.Printf("failed to call API: %s\n", esErrorString(err))
}
pageJsonOutput(conf, raw)
if conf.HumanCat && strings.HasPrefix(parts[1], "/_cat") {
fmt.Println(string(raw))
} else {
pageJsonOutput(conf, raw)
}
}
//nolint:nilerr
@@ -151,7 +159,7 @@ func ApiRepl(conf *cfg.Config) error {
}
func pageJsonOutput(conf *cfg.Config, raw []byte) {
tmpconf := &cfg.Config{HaveJQ: conf.HaveJQ}
tmpconf := new(cfg.Config{HaveJQ: conf.HaveJQ})
if conf.Pager != "" {
tmpconf.HaveJQ = false
@@ -199,16 +207,16 @@ func CallAPI(conf *cfg.Config, verb, path, data string) ([]byte, error) {
verb = strings.ToUpper(verb)
// we're using port-forwards anyway
noVerifyTransport := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
noVerifyTransport := new(http.Transport{
TLSClientConfig: new(tls.Config{InsecureSkipVerify: true}),
})
client := &http.Client{Transport: noVerifyTransport}
client := new(http.Client{Transport: noVerifyTransport})
if conf.DebugHTTP {
client = &http.Client{
Transport: &cfg.DebugTransport{
Transport: noVerifyTransport}}
client = new(http.Client{
Transport: new(cfg.DebugTransport{
Transport: noVerifyTransport})})
}
req, err := http.NewRequest(verb, conf.DefaultCluster.Uri+path, bytes.NewBuffer([]byte(data)))
@@ -216,8 +224,10 @@ func CallAPI(conf *cfg.Config, verb, path, data string) ([]byte, error) {
return nil, err
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Accept", "application/json")
if !conf.HumanCat || (conf.HumanCat && !strings.HasPrefix(path, "/_cat")) {
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Accept", "application/json")
}
// make sure we have got all we need
if err := conf.DefaultCluster.CheckAuth(); err != nil {
@@ -271,7 +281,8 @@ func prettyfiJson(conf *cfg.Config, raw []byte) (string, error) {
err := json.Indent(&pretty, raw, "", "\t")
if err != nil {
return "", fmt.Errorf("json parse error: %w", err)
//nolint:nilerr
return string(raw), nil
}
return pretty.String(), nil
@@ -356,16 +367,7 @@ func ApiList(conf *cfg.Config, pattern string) error {
func ApiPathNames() []string {
assets.LoadAssetOpenApi()
paths := make([]string, len(assets.OpenAPI.Spec().Paths.Paths))
idx := 0
for path := range assets.OpenAPI.Spec().Paths.Paths {
paths[idx] = path
idx++
}
return paths
return slices.Collect(maps.Keys(assets.OpenAPI.Spec().Paths.Paths))
}
func ApiShow(conf *cfg.Config, showpath, verb string) error {
@@ -526,7 +528,7 @@ func getApiExample(op *Op) string {
// otherwise showpath+verb have to match precisely.
func matchOperation(showpath, verb string) (*Op, error) {
ops := []*Op{}
op := &Op{}
op := new(Op{})
var found bool

View File

@@ -17,6 +17,10 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package es
import (
// "encoding/json/jsontext"
// "encoding/json/v2"
"encoding/json"
"fmt"
@@ -49,11 +53,22 @@ func getHealthReport(conf *cfg.Config) (*HealthReport, error) {
return nil, err
}
report := HealthReport{}
report := new(HealthReport{})
// FIXME: use this once jsonv2 is no more experimental it already
// builds and works like intended, but golangci-lint doesn't
// recognize it with: go: unknown GOEXPERIMENT jsonv2
//
// if err := json.UnmarshalDecode(
// jsontext.NewDecoder(
// bytes.NewBuffer(raw)),
// &report); err != nil {
// return nil, fmt.Errorf("failed to unmarshal healthreport response: %w", err)
// }
if err := json.Unmarshal(raw, &report); err != nil {
return nil, fmt.Errorf("failed to unmarshal healthreport response: %w", err)
}
return &report, nil
return report, nil
}

View File

@@ -101,23 +101,39 @@ func getClusterStatus(conf *cfg.Config) (*apiResponse, error) {
es := conf.DefaultCluster.ES()
responses := make(chan apiResponse, gocount)
wg := &sync.WaitGroup{}
wg := new(sync.WaitGroup{})
wg.Add(gocount)
go getApiData(conf, es, wg, responses, "health")
go getApiData(conf, es, wg, responses, "healthreport")
go getApiData(conf, es, wg, responses, "info")
go getApiData(conf, es, wg, responses, "ccr")
go getApiData(conf, es, wg, responses, "indices")
go getApiData(conf, es, wg, responses, "tasks")
wg.Go(func() {
getApiData(conf, es, responses, "health")
})
wg.Go(func() {
getApiData(conf, es, responses, "healthreport")
})
wg.Go(func() {
getApiData(conf, es, responses, "info")
})
wg.Go(func() {
getApiData(conf, es, responses, "ccr")
})
wg.Go(func() {
getApiData(conf, es, responses, "indices")
})
wg.Go(func() {
getApiData(conf, es, responses, "tasks")
})
if conf.Verbose {
go getApiData(conf, es, wg, responses, "stats")
getApiData(conf, es, responses, "stats")
}
wg.Wait()
all := apiResponse{}
all := new(apiResponse{})
var err error
@@ -144,7 +160,7 @@ func getClusterStatus(conf *cfg.Config) (*apiResponse, error) {
}
}
return &all, err
return all, err
}
func ClusterStatus(conf *cfg.Config) error {
@@ -174,11 +190,16 @@ func ClusterStatus(conf *cfg.Config) error {
}
// look for red indices, if any
redindices := 0
failedIndices := 0
greenIndices := 0
for _, index := range *res.indices {
if *index.Health == "red" {
redindices++
switch {
case *index.Health != "green":
failedIndices++
case !strings.HasPrefix(*index.Index, "."):
// don't count internal indices
greenIndices++
}
}
@@ -205,7 +226,8 @@ func ClusterStatus(conf *cfg.Config) error {
{"Unassigned Primary Shards", res.health.UnassignedPrimaryShards},
{"Pending Tasks", res.health.NumberOfPendingTasks},
{"Nodes", res.health.NumberOfNodes},
{"Red Indices", redindices},
{"Green Indices", greenIndices},
{"Failed Indices", failedIndices},
{"Long Running Tasks", longtasks},
}
@@ -233,6 +255,10 @@ func ClusterStatus(conf *cfg.Config) error {
for resource, items := range diag.AffectedResources {
table.Entries = append(table.Entries, []any{" -> affected " + resource, strings.Join(items, ",")})
}
if diag.Action != "" {
table.AddRow(" -> suggested action to fix", diag.Action)
}
}
}
}

View File

@@ -29,12 +29,12 @@ func ClusterRerouteMove(conf *cfg.Config, index string) error {
move := conf.DefaultCluster.ES().Cluster.Reroute()
commands := esdsl.NewCommand()
moveCommand := &types.CommandMoveAction{
moveCommand := new(types.CommandMoveAction{
Shard: conf.Shards,
FromNode: conf.FromNode,
ToNode: conf.ToNode,
Index: index,
}
})
commands.CommandCaster().Move = moveCommand
@@ -52,11 +52,11 @@ func ClusterRerouteAllocateReplica(conf *cfg.Config, index string) error {
move := conf.DefaultCluster.ES().Cluster.Reroute()
commands := esdsl.NewCommand()
allocCommand := &types.CommandAllocateReplicaAction{
allocCommand := new(types.CommandAllocateReplicaAction{
Shard: conf.Shards,
Node: conf.ToNode,
Index: index,
}
})
commands.CommandCaster().AllocateReplica = allocCommand
@@ -74,12 +74,12 @@ func ClusterRerouteCancel(conf *cfg.Config, index string) error {
move := conf.DefaultCluster.ES().Cluster.Reroute()
commands := esdsl.NewCommand()
cancelCommand := &types.CommandCancelAction{
cancelCommand := new(types.CommandCancelAction{
Shard: conf.Shards,
Node: conf.ToNode,
Index: index,
AllowPrimary: &conf.AllowPrimary,
}
})
commands.CommandCaster().Cancel = cancelCommand
@@ -97,12 +97,12 @@ func ClusterRerouteAllocatePrimary(conf *cfg.Config, index string, stale bool) e
move := conf.DefaultCluster.ES().Cluster.Reroute()
commands := esdsl.NewCommand()
allocCommand := &types.CommandAllocatePrimaryAction{
allocCommand := new(types.CommandAllocatePrimaryAction{
Shard: conf.Shards,
Node: conf.ToNode,
Index: index,
AcceptDataLoss: conf.AcceptDataLoss,
}
})
if stale {
commands.CommandCaster().AllocateStalePrimary = allocCommand

View File

@@ -101,7 +101,7 @@ func DocDelete(conf *cfg.Config, queries []string) error {
return nil
}
req := &deletebyquery.Request{}
req := new(deletebyquery.Request{})
if len(queries) == 0 && conf.All {
req.Query = esdsl.NewMatchAllQuery().QueryCaster()

View File

@@ -22,6 +22,8 @@ import (
"errors"
"fmt"
"log/slog"
"maps"
"slices"
"strings"
"codeberg.org/scip/esctl/pkg/cfg"
@@ -62,13 +64,7 @@ func IlmNames(conf *cfg.Config) ([]string, error) {
return nil, fmt.Errorf("failed to get ilm policies: %w", esErrorString(err))
}
names := make([]string, len(res))
idx := 0
for name := range res {
names[idx] = name
idx++
}
names := slices.Collect(maps.Keys(res))
return names, nil
}
@@ -351,7 +347,7 @@ func IlmCreate(conf *cfg.Config, policyname string) error {
var actions types.IlmActionsVariant = esdsl.NewIlmActions()
rollover := &types.RolloverAction{}
rollover := new(types.RolloverAction{})
haveroll := false
if policy != nil {
@@ -513,8 +509,8 @@ func IlmCreate(conf *cfg.Config, policyname string) error {
phases.PhasesCaster().Delete = policy.Phases.Delete
}
put := &putlifecycle.Request{}
newpolicy := &types.IlmPolicy{}
put := new(putlifecycle.Request{})
newpolicy := new(types.IlmPolicy{})
newpolicy.IlmPolicyCaster().Phases = *phases.PhasesCaster()
put.Policy = newpolicy

View File

@@ -185,12 +185,19 @@ func virtualAge(phase *PhaseData) time.Duration {
// Retrieve all index, ilm-explain and ilm-policies in parallel
func getIlmPhaseData(conf *cfg.Config) ([]PhaseData, error) {
responses := make(chan apiResponse, 3)
wg := &sync.WaitGroup{}
wg.Add(3)
wg := new(sync.WaitGroup{})
go getApiData(conf, conf.DefaultCluster.ES(), wg, responses, "indicesbytes")
go getApiData(conf, conf.DefaultCluster.ES(), wg, responses, "explain")
go getApiData(conf, conf.DefaultCluster.ES(), wg, responses, "policies")
wg.Go(func() {
getApiData(conf, conf.DefaultCluster.ES(), responses, "indicesbytes")
})
wg.Go(func() {
getApiData(conf, conf.DefaultCluster.ES(), responses, "explain")
})
wg.Go(func() {
getApiData(conf, conf.DefaultCluster.ES(), responses, "policies")
})
wg.Wait()
@@ -331,7 +338,7 @@ func findNextPhase(policy types.IlmPolicy, currentPhase string) *NextPhase {
// phase list to determine which comes next
phases, start := registerPhases(policy, currentPhase)
nextPhase := &NextPhase{}
nextPhase := new(NextPhase{})
// finally determine which phase comes next
// exception: hot, where we look for rollover rules

View File

@@ -18,6 +18,8 @@ package es
import (
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
"regexp"
@@ -28,6 +30,7 @@ import (
"codeberg.org/scip/esctl/pkg/cfg"
"codeberg.org/scip/esctl/pkg/printer"
"github.com/charmbracelet/lipgloss"
"github.com/elastic/go-elasticsearch/v9/typedapi/cat/indices"
"github.com/elastic/go-elasticsearch/v9/typedapi/esdsl"
"github.com/elastic/go-elasticsearch/v9/typedapi/types/enums/healthstatus"
@@ -306,3 +309,79 @@ func IndexFields(conf *cfg.Config, index string) error {
return nil
}
type Diskusage struct {
Total int64 `json:"total_in_bytes"`
Points int64 `json:"points_in_bytes"`
Norms int64 `json:"norms_in_bytes"`
TermVectors int64 `json:"term_vectors_in_bytes"`
KnnVectors int64 `json:"knn_vectors_in_bytes"`
BloomFilter int64 `json:"bloom_filter_in_bytes"`
}
type IndexDiskUsage struct {
AllFields Diskusage `json:"all_fields"`
Fields map[string]Diskusage `json:"fields"`
}
type ResIndexDiskUsage map[string]IndexDiskUsage
func IndexDiskusage(conf *cfg.Config, index string) error {
var bold = lipgloss.NewStyle().Bold(true)
res, err := conf.DefaultCluster.ES().Indices.DiskUsage(index).
RunExpensiveTasks(true).
Do(context.Background())
if err != nil {
return fmt.Errorf("failed to retrieve index disk usage: %w", esErrorString(err))
}
duRes := ResIndexDiskUsage{}
if err := json.Unmarshal(res, &duRes); err != nil {
return fmt.Errorf("failed to unmarshal disk usage response: %w", err)
}
diskusage, exists := duRes[index]
if !exists {
return errors.New("no disk usage reported for index")
}
table := printer.NewTableEmpty(conf).
WithHeaders("field", "bloom filter", "norms", "points", "term vectors", "knn vectors", "total")
for name, field := range diskusage.Fields {
if strings.HasPrefix(name, "_") || strings.HasSuffix(name, ".keyword") {
continue
}
table.AddRow(
name,
printer.Bytes(field.BloomFilter),
printer.Bytes(field.Norms),
printer.Bytes(field.Points),
printer.Bytes(field.TermVectors),
printer.Bytes(field.KnnVectors),
printer.Bytes(field.Total),
)
}
all := diskusage.AllFields
table.Sort()
table.AddRowLate(
bold.Render("Summary"),
printer.Bytes(all.BloomFilter),
printer.Bytes(all.Norms),
printer.Bytes(all.Points),
printer.Bytes(all.TermVectors),
printer.Bytes(all.KnnVectors),
printer.Bytes(all.Total),
)
if err := table.Print(); err != nil {
return err
}
return nil
}

View File

@@ -51,7 +51,11 @@ func IndexTemplateList(conf *cfg.Config) error {
return fmt.Errorf("failed to unmarshal meta json data: %w", err)
}
table.Entries[idx] = []any{tpl.Name, desc, tpl.IndexTemplate.Priority}
var prio int64
if tpl.IndexTemplate.Priority != nil {
prio = *tpl.IndexTemplate.Priority
}
table.Entries[idx] = []any{tpl.Name, desc, prio}
}
table.Sort()

View File

@@ -26,7 +26,6 @@ import (
"codeberg.org/scip/esctl/pkg/cfg"
"codeberg.org/scip/esctl/pkg/printer"
"github.com/dustin/go-humanize"
)
func NodeList(conf *cfg.Config) error {
@@ -38,11 +37,11 @@ func NodeList(conf *cfg.Config) error {
slog.Debug("ES result", "nodes", nodes)
table := printer.NewTable(conf, 7, len(nodes))
table.Addheaders("name", "ip", "load1m", "load5m", "load15m", "ram %", "heap %")
table := printer.NewTableEmpty(conf).WithHeaders(
"name", "ip", "load1m", "load5m", "load15m", "ram %", "heap %")
for idx, node := range nodes {
table.Entries[idx] = []any{
for _, node := range nodes {
table.AddRow(
*node.Name,
*node.Ip,
*node.Load1M,
@@ -50,7 +49,7 @@ func NodeList(conf *cfg.Config) error {
*node.Load15M,
node.RamPercent,
node.HeapPercent,
}
)
}
table.Sort()
@@ -114,18 +113,27 @@ func NodeShow(conf *cfg.Config, nodename string) error {
}
k8snode := info.Attributes["k8s_node_name"]
rank := "none"
adsel, exists := stat.AdaptiveSelection[id]
if exists {
rank = *adsel.Rank
}
table.Entries = [][]any{
{"Id", id},
{"Name", nodename},
{"Kubernetes node", k8snode},
{"Ip address", info.Ip},
{"Node rank", *stat.AdaptiveSelection[id].Rank},
{"Node rank", rank},
{"JVM", info.Jvm.VmName + " " + info.Jvm.Version},
{"JVM Started", time.UnixMilli(info.Jvm.StartTimeInMillis)},
{"OS", info.Os.PrettyName + " " + info.Os.Version},
{"Node roles", roles},
{"Node version", info.Version},
// FIXME: not implemented upstream
// see: https://github.com/elastic/go-elasticsearch/issues/1526
// {"Allocated shards", stat.Allocations.XXX},
{"HTTP clients", *stat.Http.CurrentOpen},
{"CPUs", *info.Os.AllocatedProcessors},
{"Load 15m/5m/1m", fmt.Sprintf("%.2f/%.2f/%.2f",
@@ -134,18 +142,41 @@ func NodeShow(conf *cfg.Config, nodename string) error {
stat.Os.Cpu.LoadAverage["1m"],
)},
{"Open FD's", *stat.Process.OpenFileDescriptors},
{"Response time avg", fmt.Sprintf("%dns", *stat.AdaptiveSelection[id].AvgResponseTimeNs)},
{"HTTP sesssions current/total", fmt.Sprintf("%d/%d",
*stat.Http.CurrentOpen,
*stat.Http.TotalOpened,
)},
{"Traffic rx/tx",
printer.ByteString(*stat.Transport.RxSizeInBytes) + " / " + printer.ByteString(*stat.Transport.TxSizeInBytes)},
{"Response time avg", time.Duration(*stat.AdaptiveSelection[id].AvgResponseTimeNs)},
{"Memory usage (used/avail)",
humanize.Bytes(uint64(*stat.Os.Mem.UsedInBytes)) + " / " + humanize.Bytes(uint64(*stat.Os.Mem.TotalInBytes))},
printer.ByteString(*stat.Os.Mem.UsedInBytes) + " / " + printer.ByteString(*stat.Os.Mem.TotalInBytes)},
{"Search queries current/total", fmt.Sprintf("%d/%d",
stat.Indices.Search.QueryCurrent,
stat.Indices.Search.QueryTotal,
)},
{"Search efficiency", stat.Indices.Search.QueryTimeInMillis / stat.Indices.Search.QueryTotal},
{"Docs count", stat.Indices.Docs.Count},
{"Merges current/total", fmt.Sprintf("%d/%d",
stat.Indices.Merges.Current,
stat.Indices.Merges.Total,
)},
{"Merge docs count current/total", fmt.Sprintf("%d/%d",
stat.Indices.Merges.CurrentDocs,
stat.Indices.Merges.TotalDocs,
)},
{"Merge size current/total", fmt.Sprintf("%s/%s",
printer.ByteString(stat.Indices.Merges.CurrentSizeInBytes),
printer.ByteString(stat.Indices.Merges.TotalSizeInBytes),
)},
{"CircuitBreaker trip count", *stat.Breakers["fielddata"].Tripped},
}
if len(stat.Fs.Data) > 0 {
fs := stat.Fs.Data[0]
table.Entries = append(table.Entries, [][]any{
{"Storage usage (used/avail)",
humanize.Bytes(uint64(*fs.AvailableInBytes)) + " / " + humanize.Bytes(uint64(*fs.TotalInBytes))},
{"Storage mount", *fs.Mount},
}...)
table.AddRow("Storage usage (used/avail)",
printer.ByteString(*fs.AvailableInBytes)+" / "+printer.ByteString(*fs.TotalInBytes))
table.AddRow("Storage mount", *fs.Mount)
}
if err := table.Print(); err != nil {
@@ -204,3 +235,72 @@ func NodeClients(conf *cfg.Config, nodename string) error {
return table.Print()
}
func NodeUsage(conf *cfg.Config, nodeid string) error {
usage := conf.DefaultCluster.ES().Nodes.Usage()
if nodeid != "" {
usage.NodeId(nodeid)
}
stats, err := usage.Do(context.Background())
if err != nil {
return fmt.Errorf("failed to get node usage: %w", esErrorString(err))
}
slog.Debug("ES result", "usage", stats)
table := printer.NewTableEmpty(conf).WithHeaders(
"node",
"bulk",
"doc get",
"doc mget",
"doc update",
"index doc",
"index stats",
"search",
"msearch",
"open pit",
)
for id, actions := range stats.Nodes {
node, err := getNodeName(conf, id)
if err != nil {
return err
}
stat := actions.RestActions
table.AddRow(
node,
stat["bulk_action"],
stat["document_get_action"],
stat["document_mget_action"],
stat["document_update_action"],
stat["document_index_action"],
stat["indices_stats_action"],
stat["search_action"],
stat["msearch_action"],
stat["open_point_in_time"],
)
}
return table.Print()
}
func getNodeName(conf *cfg.Config, id string) (string, error) {
res, err := conf.DefaultCluster.ES().Nodes.Info().
Metric("os").
Do(context.Background())
if err != nil {
return "", fmt.Errorf("failed to get node info: %w", esErrorString(err))
}
for nodeid, node := range res.Nodes {
if id == nodeid {
return node.Name, nil
}
}
return "", nil
}

View File

@@ -19,7 +19,6 @@ package es
import (
"context"
"fmt"
"sync"
"codeberg.org/scip/esctl/pkg/cfg"
"github.com/elastic/go-elasticsearch/v9"
@@ -61,14 +60,7 @@ type apiResponse struct {
which int
}
func getApiData(
conf *cfg.Config,
es *elasticsearch.TypedClient,
wg *sync.WaitGroup,
reschan chan apiResponse,
which string) {
defer wg.Done()
func getApiData(conf *cfg.Config, es *elasticsearch.TypedClient, reschan chan apiResponse, which string) {
apiRes := apiResponse{}
var arerr error

View File

@@ -20,6 +20,8 @@ import (
"context"
"fmt"
"log/slog"
"maps"
"slices"
"codeberg.org/scip/esctl/pkg/cfg"
"codeberg.org/scip/esctl/pkg/printer"
@@ -33,15 +35,7 @@ func RoleNames(conf *cfg.Config) ([]string, error) {
return nil, fmt.Errorf("failed to get roles: %w", esErrorString(err))
}
roles := make([]string, len(res))
idx := 0
for name := range res {
roles[idx] = name
idx++
}
return roles, nil
return slices.Collect(maps.Keys(res)), nil
}
func RoleList(conf *cfg.Config) error {

View File

@@ -127,7 +127,7 @@ func getCsvRecord(conf *cfg.Config, csvfile, rolename string) (*Record, error) {
}()
scanner := bufio.NewScanner(fd)
record := Record{role: rolename}
record := new(Record{role: rolename})
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
@@ -150,7 +150,7 @@ func getCsvRecord(conf *cfg.Config, csvfile, rolename string) (*Record, error) {
}
}
return &record, nil
return record, nil
}
func diffRoles(conf *cfg.Config, records map[string]Record, res getrole.Response) []Register {

View File

@@ -56,7 +56,7 @@ func Search(conf *cfg.Config, queries []string) error {
return err
}
req := &search.Request{Query: queryCaster}
req := new(search.Request{Query: queryCaster})
searchEs.Request(req)
@@ -128,7 +128,7 @@ func validateSearch(conf *cfg.Config, queries []string) error {
return err
}
req := &validatequery.Request{Query: queryCaster}
req := new(validatequery.Request{Query: queryCaster})
validate.Request(req)

View File

@@ -80,7 +80,7 @@ func NewFilter(query string) (*filter, error) {
return nil, errors.New("search queries must be in the form field<sep>pattern where <sep> must be one of: = or !=")
}
flt := &filter{term: part[0], filter: part[1], criteria: criteria}
flt := new(filter{term: part[0], filter: part[1], criteria: criteria})
if strings.Contains(part[0], ",") {
// a MultiMatchQuery, match across multiple fields at once

79
pkg/es/searchql.go Normal file
View File

@@ -0,0 +1,79 @@
/*
Copyright © 2026 Thomas von Dein
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package es
import (
"context"
"encoding/json"
"fmt"
"log/slog"
"codeberg.org/scip/esctl/pkg/cfg"
"codeberg.org/scip/esctl/pkg/printer"
)
type searchQlColumn struct {
Name string `json:"name"`
Type string `json:"type"`
}
type searchQlResult struct {
Partial bool `json:"is_partial"`
Docs int `json:"documents_found"`
Columns []searchQlColumn `json:"columns"`
Values [][]any `json:"values"`
}
func SearchQL(conf *cfg.Config, querystring string) error {
query := conf.DefaultCluster.ES().Esql.Query().
Query(querystring).
DropNullColumns(true)
res, err := query.Do(context.Background())
if err != nil {
return fmt.Errorf("failed to run esql query: %w", esErrorString(err))
}
if conf.Debug {
fmt.Println(string(res))
}
qlResult := searchQlResult{}
if err = json.Unmarshal(res, &qlResult); err != nil {
return fmt.Errorf("failed to unmarshal esql result: %w", err)
}
slog.Debug("searchql result", "qlres", qlResult)
if qlResult.Docs == 0 {
return nil
}
table := printer.NewTable(conf, len(qlResult.Columns), len(qlResult.Values))
headers := make([]string, len(qlResult.Columns))
for idx, col := range qlResult.Columns {
headers[idx] = col.Name
}
table.Addheaders(headers...)
table.Entries = qlResult.Values
return table.Print()
}

View File

@@ -163,30 +163,37 @@ func ShardAllocation(conf *cfg.Config, index string) error {
slog.Debug("ES result", "explain", res)
currentNode := res.CurrentNode
table := printer.NewTable(conf, 2, 10)
table.Addheaders("shard allocation setting", "value")
roles := make([]string, len(currentNode.Roles))
for idx, role := range currentNode.Roles {
roles[idx] = role.Name
}
table.Entries = [][]any{
{"Index", index},
{"Current state", res.CurrentState},
{"Current node", currentNode.Name},
{"Current k8s node", currentNode.Attributes["k8s_node_name"]},
{"Current node address", currentNode.TransportAddress},
{"Current node id", currentNode.Id},
{"Current node weight", currentNode.WeightRanking},
{"Current node roles", roles},
{"Can rebalance cluster", res.CanRebalanceCluster.Name},
{"Can rebalance to another node", res.CanRebalanceToOtherNode.Name},
{"Can remain on current node", res.CanRemainOnCurrentNode.Name},
}
if res.CurrentNode != nil {
currentNode := res.CurrentNode
roles := make([]string, len(currentNode.Roles))
for idx, role := range currentNode.Roles {
roles[idx] = role.Name
}
table.Entries = append(table.Entries, [][]any{
{"Current node", currentNode.Name},
{"Current k8s node", currentNode.Attributes["k8s_node_name"]},
{"Current node address", currentNode.TransportAddress},
{"Current node id", currentNode.Id},
{"Current node weight", currentNode.WeightRanking},
{"Current node roles", roles},
}...)
} else {
table.AddRow("Current node", "not currently assigned to any node")
}
if res.CurrentState == "unassigned" {
table.AddRow("Unassignment reason", res.UnassignedInfo.Reason.String()+" at "+res.UnassignedInfo.At.(string))
}

View File

@@ -66,13 +66,13 @@ func SnapshotList(conf *cfg.Config) error {
snapshots := []*Snapshot{} // original snapshot names
for _, snapshot := range sres {
snap := &Snapshot{
snap := new(Snapshot{
Name: *snapshot.Id,
Status: *snapshot.Status,
Start: fmt.Sprintf("%s", snapshot.StartTime),
Forindex: indexFromSnapshot(*snapshot.Id),
Orphaned: "no",
}
})
_, exists := indicies[snap.Forindex]
if !exists {

View File

@@ -29,13 +29,13 @@ import (
const LevelNotice = slog.Level(2)
func Init(conf *cfg.Config) {
logLevel := &slog.LevelVar{}
logLevel := new(slog.LevelVar{})
opts := &yadu.Options{
opts := new(yadu.Options{
Level: logLevel,
AddSource: true,
NoColor: !isatty.IsTerminal(os.Stdout.Fd()),
}
})
buildInfo, _ := debug.ReadBuildInfo()

View File

@@ -22,10 +22,16 @@ type ByteSize struct {
size uint64
}
func Bytes(size int64) ByteSize {
return ByteSize{size: uint64(size)}
}
func (b *ByteSize) String() string {
return humanize.Bytes(b.size)
}
func Bytes(size int64) *ByteSize {
return new(ByteSize{size: uint64(size)})
}
func ByteString(size int64) string {
b := ByteSize{size: uint64(size)}
return b.String()
}

View File

@@ -38,12 +38,18 @@ func any2string(in any) string {
return strconv.Itoa(val)
case float64:
return fmt.Sprintf("%.2f", val)
case float32:
return fmt.Sprintf("%.2f", val)
case []string:
return strings.Join(val, ",")
case ByteSize:
return val.String()
case *ByteSize:
return val.String()
case time.Time:
return val.Format("2006-01-02 15:04:05")
case time.Duration:
return val.String()
case []byte:
return string(val)
case nil:

View File

@@ -30,33 +30,44 @@ import (
)
type Table struct {
Mode string // tsv, json, yaml
Headers []string
Entries [][]any
Mode string // tsv, json, yaml
Headers []string
RawHeaders []string
Entries [][]any
rows [][]string // representation used for printing
processed bool
lenHeaders []int
alignInts bool
maxwidth int
rows [][]string // representation used for printing
processed bool
lenHeaders []int
alignInts bool
maxwidth int
debugGoRoutines bool
}
func NewTable(conf *cfg.Config, columns, rows int) *Table {
table := Table{Mode: conf.Output, maxwidth: cfg.GetTermWidth()}
table := new(Table{
Mode: conf.Output,
maxwidth: cfg.GetTermWidth(),
debugGoRoutines: conf.DebugGoRoutines,
})
table.Headers = make([]string, columns)
table.RawHeaders = make([]string, columns)
table.Entries = make([][]any, rows)
table.lenHeaders = make([]int, columns)
table.alignInts = conf.AlignInts
return &table
return table
}
func NewTableEmpty(conf *cfg.Config) *Table {
table := Table{Mode: conf.Output, maxwidth: cfg.GetTermWidth()}
table := new(Table{
Mode: conf.Output,
maxwidth: cfg.GetTermWidth(),
debugGoRoutines: conf.DebugGoRoutines,
})
table.alignInts = conf.AlignInts
return &table
return table
}
func (table *Table) WithHeaders(headers ...string) *Table {
@@ -65,6 +76,7 @@ func (table *Table) WithHeaders(headers ...string) *Table {
table.Entries = [][]any{}
table.lenHeaders = make([]int, count)
table.Headers = make([]string, count)
table.RawHeaders = make([]string, count)
table.Addheaders(headers...)
@@ -72,14 +84,24 @@ func (table *Table) WithHeaders(headers ...string) *Table {
}
func (table *Table) Print() error {
var err error
switch table.Mode {
case "json":
return table.PrintJSON()
err = table.PrintJSON()
case "yaml":
return table.PrintYAML()
err = table.PrintYAML()
case "csv":
err = table.PrintCSV()
default:
return table.PrintTSV()
err = table.PrintTSV()
}
if table.debugGoRoutines {
printGoRoutineMetrics()
}
return err
}
var (
@@ -144,9 +166,11 @@ func (table *Table) PrintTSV() error {
wrapped := wrapper(entry)
// and indent it
for idx, line := range strings.Split(wrapped, "\n") {
if idx == 0 {
first := true
for line := range strings.Lines(wrapped) {
if first {
entry = line
first = false
} else {
entry += "\n " + strings.Repeat(" ", currentWidth) + line
}
@@ -178,6 +202,28 @@ func (table *Table) PrintTSV() error {
return nil
}
func (table *Table) PrintCSV() error {
table.preprocessRows()
fmt.Println(strings.Join(table.RawHeaders, ","))
for _, entries := range table.rows {
row := make([]string, len(entries))
for idx, entry := range entries {
if strings.Contains(entry, " ") || strings.Contains(entry, ",") {
row[idx] = `"` + entry + `"`
} else {
row[idx] = entry
}
}
fmt.Println(strings.Join(row, ","))
}
return nil
}
func (table *Table) Sort() {
// sanity checks
if len(table.Entries) == 0 {
@@ -199,6 +245,8 @@ func (table *Table) Addheaders(headers ...string) {
default:
table.Headers[idx] = bold(strings.ReplaceAll(strings.ToUpper(header), " ", "-"))
}
table.RawHeaders[idx] = header
}
}
@@ -206,6 +254,22 @@ func (table *Table) AddRow(fields ...any) {
table.Entries = append(table.Entries, fields)
}
func (table *Table) AddRowLate(fields ...any) {
table.AddRow(fields)
if !table.processed {
return
}
row := make([]string, len(fields))
for idx, field := range fields {
row[idx] = any2string(field)
}
table.rows = append(table.rows, row)
}
// needed for json and yaml output
func (table *Table) toMap() []map[string]any {
raw := make([]map[string]any, len(table.Entries))