add support for human readable /_cat endpoints: 'api repl -H'

see: https://github.com/urfave/cli/issues/2348

Makefile

@@ -67,10 +67,10 @@ test: clean buildlocal
 
 testlint: test lint
 
-lint:
+lint-basic:
 	golangci-lint run --enable-only errcheck,govet,ineffassign,staticcheck,unused
 
-lint-full:
+lint:
 	golangci-lint run --show-stats=false
 
 testfuzzy: clean

cmd/api.go

@@ -91,6 +91,12 @@ func ApiRepl(conf *cfg.Config) *cli.Command {
 				Aliases:     []string{"p"},
 				Sources:     cli.EnvVars("PAGER", "ES_JSON_PAGER"),
 			},
+			&cli.BoolFlag{
+				Name:        "human-readable-cat",
+				Usage:       "enable human readable /_cat output",
+				Destination: &conf.HumanCat,
+				Aliases:     []string{"H"},
+			},
 		},
 
 		Action: func(ctx context.Context, cmd *cli.Command) error {

cmd/index.go

@@ -41,6 +41,7 @@ func Index(conf *cfg.Config) *cli.Command {
 			IndexClose(conf),
 			IndexFields(conf),
 			IndexIlm(conf),
+			IndexDu(conf),
 
 			// sub commands
 			IndexAlias(conf),
@@ -277,3 +278,24 @@ func IndexIlm(conf *cfg.Config) *cli.Command {
 		},
 	}
 }
+
+func IndexDu(conf *cfg.Config) *cli.Command {
+	return &cli.Command{
+		Name:      "du",
+		Usage:     "show index disk usage",
+		UsageText: "index du <index>",
+
+		ShellComplete: func(ctx context.Context, cmd *cli.Command) {
+			complete(conf, cmd, Cindex)
+		},
+
+		Action: func(ctx context.Context, cmd *cli.Command) error {
+			index := cmd.Args().Get(0)
+			if index == "" {
+				return errors.New("no index specified")
+			}
+
+			return es.IndexDiskusage(conf, index)
+		},
+	}
+}

pkg/cfg/config.go

@@ -101,7 +101,8 @@ type Config struct {
 	MaxDocs, MaxShardSize, MaxShardDocs int  // roll over
 	DryRun                              bool // rollover: -n
 
-	Tag string // api ls: -t
+	Tag      string // api ls: -t
+	HumanCat bool   // api repl: -H
 
 	Ilm Ilm // ilm create
 

pkg/es/api.go

@@ -45,22 +45,25 @@ import (
 )
 
 const (
-	intro = `Input format: verb path [data]"
-
-Example:
-
-post /yourindex/_ccr/pause_follow
-put /yourindex/_settings {"number_of_replicas": 1}
-
-You can also put multiline JSON after the path like:
-
-put /yourindex/_settings
-{
-  "number_of_replicas": 1
-}
-
-If you do NOT  supply a JSON in the first line, you  need to hit ENTER
-twice to complete.`
+	intro = `# Input format: verb path [data]"
+# 
+# Example:
+# 
+# post /yourindex/_ccr/pause_follow
+# put /yourindex/_settings {"number_of_replicas": 1}
+# 
+# You can also put multiline JSON after the path like:
+# 
+# put /yourindex/_settings
+# {
+#   "number_of_replicas": 1
+# }
+# 
+# If you do NOT  supply a JSON in the first line, you  need to hit ENTER
+# twice to complete.
+#
+# Supply the flag --human-readable-cat, -H to view /_cat API calls in
+# human readable form.`
 )
 
 // holds an API operation via go-openapi/spec
@@ -143,7 +146,11 @@ func ApiRepl(conf *cfg.Config) error {
 			fmt.Printf("failed to call API: %s\n", esErrorString(err))
 		}
 
-		pageJsonOutput(conf, raw)
+		if conf.HumanCat && strings.HasPrefix(parts[1], "/_cat") {
+			fmt.Println(string(raw))
+		} else {
+			pageJsonOutput(conf, raw)
+		}
 	}
 
 	//nolint:nilerr
@@ -216,8 +223,10 @@ func CallAPI(conf *cfg.Config, verb, path, data string) ([]byte, error) {
 		return nil, err
 	}
 
-	req.Header.Add("Content-Type", "application/json")
-	req.Header.Add("Accept", "application/json")
+	if !conf.HumanCat && strings.HasPrefix(path, "/_cat") {
+		req.Header.Add("Content-Type", "application/json")
+		req.Header.Add("Accept", "application/json")
+	}
 
 	// make sure we have got all we need
 	if err := conf.DefaultCluster.CheckAuth(); err != nil {

pkg/es/index.go

@@ -18,6 +18,8 @@ package es
 
 import (
 	"context"
+	"encoding/json"
+	"errors"
 	"fmt"
 	"log/slog"
 	"regexp"
@@ -28,6 +30,7 @@ import (
 
 	"codeberg.org/scip/esctl/pkg/cfg"
 	"codeberg.org/scip/esctl/pkg/printer"
+	"github.com/charmbracelet/lipgloss"
 	"github.com/elastic/go-elasticsearch/v9/typedapi/cat/indices"
 	"github.com/elastic/go-elasticsearch/v9/typedapi/esdsl"
 	"github.com/elastic/go-elasticsearch/v9/typedapi/types/enums/healthstatus"
@@ -306,3 +309,79 @@ func IndexFields(conf *cfg.Config, index string) error {
 
 	return nil
 }
+
+type Diskusage struct {
+	Total       int64 `json:"total_in_bytes"`
+	Points      int64 `json:"points_in_bytes"`
+	Norms       int64 `json:"norms_in_bytes"`
+	TermVectors int64 `json:"term_vectors_in_bytes"`
+	KnnVectors  int64 `json:"knn_vectors_in_bytes"`
+	BloomFilter int64 `json:"bloom_filter_in_bytes"`
+}
+
+type IndexDiskUsage struct {
+	AllFields Diskusage            `json:"all_fields"`
+	Fields    map[string]Diskusage `json:"fields"`
+}
+
+type ResIndexDiskUsage map[string]IndexDiskUsage
+
+func IndexDiskusage(conf *cfg.Config, index string) error {
+	var bold = lipgloss.NewStyle().Bold(true)
+
+	res, err := conf.DefaultCluster.ES().Indices.DiskUsage(index).
+		RunExpensiveTasks(true).
+		Do(context.Background())
+	if err != nil {
+		return fmt.Errorf("failed to retrieve index disk usage: %w", esErrorString(err))
+	}
+
+	duRes := ResIndexDiskUsage{}
+	if err := json.Unmarshal(res, &duRes); err != nil {
+		return fmt.Errorf("failed to unmarshal disk usage response: %w", err)
+	}
+
+	diskusage, exists := duRes[index]
+	if !exists {
+		return errors.New("no disk usage reported for index")
+	}
+
+	table := printer.NewTableEmpty(conf).
+		WithHeaders("field", "bloom filter", "norms", "points", "term vectors", "knn vectors", "total")
+
+	for name, field := range diskusage.Fields {
+		if strings.HasPrefix(name, "_") || strings.HasSuffix(name, ".keyword") {
+			continue
+		}
+
+		table.AddRow(
+			name,
+			printer.Bytes(field.BloomFilter),
+			printer.Bytes(field.Norms),
+			printer.Bytes(field.Points),
+			printer.Bytes(field.TermVectors),
+			printer.Bytes(field.KnnVectors),
+			printer.Bytes(field.Total),
+		)
+	}
+
+	all := diskusage.AllFields
+
+	table.Sort()
+
+	table.AddRowLate(
+		bold.Render("Summary"),
+		printer.Bytes(all.BloomFilter),
+		printer.Bytes(all.Norms),
+		printer.Bytes(all.Points),
+		printer.Bytes(all.TermVectors),
+		printer.Bytes(all.KnnVectors),
+		printer.Bytes(all.Total),
+	)
+
+	if err := table.Print(); err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/es/node.go

@@ -26,7 +26,6 @@ import (
 
 	"codeberg.org/scip/esctl/pkg/cfg"
 	"codeberg.org/scip/esctl/pkg/printer"
-	"github.com/dustin/go-humanize"
 )
 
 func NodeList(conf *cfg.Config) error {
@@ -114,18 +113,27 @@ func NodeShow(conf *cfg.Config, nodename string) error {
 		}
 
 		k8snode := info.Attributes["k8s_node_name"]
+		rank := "none"
+
+		adsel, exists := stat.AdaptiveSelection[id]
+		if exists {
+			rank = *adsel.Rank
+		}
 
 		table.Entries = [][]any{
 			{"Id", id},
 			{"Name", nodename},
 			{"Kubernetes node", k8snode},
 			{"Ip address", info.Ip},
-			{"Node rank", *stat.AdaptiveSelection[id].Rank},
+			{"Node rank", rank},
 			{"JVM", info.Jvm.VmName + " " + info.Jvm.Version},
 			{"JVM Started", time.UnixMilli(info.Jvm.StartTimeInMillis)},
 			{"OS", info.Os.PrettyName + " " + info.Os.Version},
 			{"Node roles", roles},
 			{"Node version", info.Version},
+			// FIXME: not implemented upstream
+			// see: https://github.com/elastic/go-elasticsearch/issues/1526
+			// {"Allocated shards", stat.Allocations.XXX},
 			{"HTTP clients", *stat.Http.CurrentOpen},
 			{"CPUs", *info.Os.AllocatedProcessors},
 			{"Load 15m/5m/1m", fmt.Sprintf("%.2f/%.2f/%.2f",
@@ -134,18 +142,41 @@ func NodeShow(conf *cfg.Config, nodename string) error {
 				stat.Os.Cpu.LoadAverage["1m"],
 			)},
 			{"Open FD's", *stat.Process.OpenFileDescriptors},
-			{"Response time avg", fmt.Sprintf("%dns", *stat.AdaptiveSelection[id].AvgResponseTimeNs)},
+			{"HTTP sesssions current/total", fmt.Sprintf("%d/%d",
+				*stat.Http.CurrentOpen,
+				*stat.Http.TotalOpened,
+			)},
+			{"Traffic rx/tx",
+				printer.ByteString(*stat.Transport.RxSizeInBytes) + " / " + printer.ByteString(*stat.Transport.TxSizeInBytes)},
+			{"Response time avg", time.Duration(*stat.AdaptiveSelection[id].AvgResponseTimeNs)},
 			{"Memory usage (used/avail)",
-				humanize.Bytes(uint64(*stat.Os.Mem.UsedInBytes)) + " / " + humanize.Bytes(uint64(*stat.Os.Mem.TotalInBytes))},
+				printer.ByteString(*stat.Os.Mem.UsedInBytes) + " / " + printer.ByteString(*stat.Os.Mem.TotalInBytes)},
+			{"Search queries current/total", fmt.Sprintf("%d/%d",
+				stat.Indices.Search.QueryCurrent,
+				stat.Indices.Search.QueryTotal,
+			)},
+			{"Search efficiency", stat.Indices.Search.QueryTimeInMillis / stat.Indices.Search.QueryTotal},
+			{"Docs count", stat.Indices.Docs.Count},
+			{"Merges current/total", fmt.Sprintf("%d/%d",
+				stat.Indices.Merges.Current,
+				stat.Indices.Merges.Total,
+			)},
+			{"Merge docs count current/total", fmt.Sprintf("%d/%d",
+				stat.Indices.Merges.CurrentDocs,
+				stat.Indices.Merges.TotalDocs,
+			)},
+			{"Merge size current/total", fmt.Sprintf("%s/%s",
+				printer.ByteString(stat.Indices.Merges.CurrentSizeInBytes),
+				printer.ByteString(stat.Indices.Merges.TotalSizeInBytes),
+			)},
+			{"CircuitBreaker trip count", *stat.Breakers["fielddata"].Tripped},
 		}
 
 		if len(stat.Fs.Data) > 0 {
 			fs := stat.Fs.Data[0]
-			table.Entries = append(table.Entries, [][]any{
-				{"Storage usage (used/avail)",
-					humanize.Bytes(uint64(*fs.AvailableInBytes)) + " / " + humanize.Bytes(uint64(*fs.TotalInBytes))},
-				{"Storage mount", *fs.Mount},
-			}...)
+			table.AddRow("Storage usage (used/avail)",
+				printer.ByteString(*fs.AvailableInBytes)+" / "+printer.ByteString(*fs.TotalInBytes))
+			table.AddRow("Storage mount", *fs.Mount)
 		}
 
 		if err := table.Print(); err != nil {

pkg/es/shard.go

@@ -163,30 +163,37 @@ func ShardAllocation(conf *cfg.Config, index string) error {
 
 	slog.Debug("ES result", "explain", res)
 
-	currentNode := res.CurrentNode
-
 	table := printer.NewTable(conf, 2, 10)
 	table.Addheaders("shard allocation setting", "value")
 
-	roles := make([]string, len(currentNode.Roles))
-	for idx, role := range currentNode.Roles {
-		roles[idx] = role.Name
-	}
-
 	table.Entries = [][]any{
 		{"Index", index},
 		{"Current state", res.CurrentState},
-		{"Current node", currentNode.Name},
-		{"Current k8s node", currentNode.Attributes["k8s_node_name"]},
-		{"Current node address", currentNode.TransportAddress},
-		{"Current node id", currentNode.Id},
-		{"Current node weight", currentNode.WeightRanking},
-		{"Current node roles", roles},
 		{"Can rebalance cluster", res.CanRebalanceCluster.Name},
 		{"Can rebalance to another node", res.CanRebalanceToOtherNode.Name},
 		{"Can remain on current node", res.CanRemainOnCurrentNode.Name},
 	}
 
+	if res.CurrentNode != nil {
+		currentNode := res.CurrentNode
+		roles := make([]string, len(currentNode.Roles))
+
+		for idx, role := range currentNode.Roles {
+			roles[idx] = role.Name
+		}
+
+		table.Entries = append(table.Entries, [][]any{
+			{"Current node", currentNode.Name},
+			{"Current k8s node", currentNode.Attributes["k8s_node_name"]},
+			{"Current node address", currentNode.TransportAddress},
+			{"Current node id", currentNode.Id},
+			{"Current node weight", currentNode.WeightRanking},
+			{"Current node roles", roles},
+		}...)
+	} else {
+		table.AddRow("Current node", "not currently assigned to any node")
+	}
+
 	if res.CurrentState == "unassigned" {
 		table.AddRow("Unassignment reason", res.UnassignedInfo.Reason.String()+" at "+res.UnassignedInfo.At.(string))
 	}

pkg/printer/bytes.go

@@ -22,10 +22,16 @@ type ByteSize struct {
 	size uint64
 }
 
-func Bytes(size int64) ByteSize {
-	return ByteSize{size: uint64(size)}
-}
-
 func (b *ByteSize) String() string {
 	return humanize.Bytes(b.size)
 }
+
+func Bytes(size int64) *ByteSize {
+	return &ByteSize{size: uint64(size)}
+}
+
+func ByteString(size int64) string {
+	b := ByteSize{size: uint64(size)}
+
+	return b.String()
+}

pkg/printer/cast.go

@@ -38,12 +38,18 @@ func any2string(in any) string {
 		return strconv.Itoa(val)
 	case float64:
 		return fmt.Sprintf("%.2f", val)
+	case float32:
+		return fmt.Sprintf("%.2f", val)
 	case []string:
 		return strings.Join(val, ",")
 	case ByteSize:
 		return val.String()
+	case *ByteSize:
+		return val.String()
 	case time.Time:
 		return val.Format("2006-01-02 15:04:05")
+	case time.Duration:
+		return val.String()
 	case []byte:
 		return string(val)
 	case nil:

pkg/printer/table.go

@@ -206,6 +206,22 @@ func (table *Table) AddRow(fields ...any) {
 	table.Entries = append(table.Entries, fields)
 }
 
+func (table *Table) AddRowLate(fields ...any) {
+	table.AddRow(fields)
+
+	if !table.processed {
+		return
+	}
+
+	row := make([]string, len(fields))
+
+	for idx, field := range fields {
+		row[idx] = any2string(field)
+	}
+
+	table.rows = append(table.rows, row)
+}
+
 // needed for json and yaml output
 func (table *Table) toMap() []map[string]any {
 	raw := make([]map[string]any, len(table.Entries))