Add CCR stuff, fix search, add docs support, support index maps, refactor (#13)

pkg/es/ccr.go

@@ -0,0 +1,87 @@
+/*
+Copyright © 2026 Thomas von Dein
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+package es
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+
+	"codeberg.org/scip/esctl/pkg/cfg"
+	"github.com/elastic/go-elasticsearch/v9/typedapi/types"
+)
+
+const (
+	shard_pause = `cluster.routing.allocation.enable`
+)
+
+func CcrShardPause(conf *cfg.Config) error {
+	return ClusterSettingsSetSingle(conf, shard_pause, "none")
+}
+
+func CcrShardResume(conf *cfg.Config) error {
+	return ClusterSettingsSetSingle(conf, shard_pause, "all")
+}
+
+func CcrStatus(conf *cfg.Config, leader, follower string) error {
+	if !checkClusterIsLeader(conf, leader) {
+		if !checkClusterIsLeader(conf, follower) {
+			return errors.New("leader/follower attribution is invalid, both clusters are followers")
+		}
+
+		// reverse attribution
+		f := follower
+		follower = leader
+		leader = f
+		slog.Debug("leader/follower attribution is invalid, reversing", "leader", leader, "follower", follower)
+	}
+
+	indices := ClusterIndices{}
+
+	for _, alias := range []string{leader, follower} {
+		cat := conf.Clusters[alias].ES.Cat.Indices().
+			// we need to add custom request headers, required for older ES instances
+			Header("content-type", "application/json").
+			Header("accept", "application/json")
+
+		res, err := cat.Do(context.Background())
+		if err != nil {
+			return fmt.Errorf("failed to get indicies on %s: %s", alias, err)
+		}
+
+		indices[alias] = make(map[string]*types.IndicesRecord, len(res))
+
+		for _, index := range res {
+			indices[alias][*index.Index] = &index
+		}
+	}
+
+	if !checkClusterStatus(conf, leader, follower) {
+		return errors.New("one of the two clusters is in a failed state")
+	}
+
+	findIlmErrors(conf, leader, follower)
+
+	if findIndicesOnlyOnLeader(conf, indices, leader, follower) &&
+		findOrphanedIndices(conf, indices, leader, follower) &&
+		findFailedFollowerIndices(conf, indices, follower) {
+		fmt.Println("everything's hunky-dory.")
+	}
+
+	return nil
+}

pkg/es/ccr_follower.go

@@ -0,0 +1,106 @@
+/*
+Copyright © 2026 Thomas von Dein
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+package es
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+
+	"codeberg.org/scip/esctl/pkg/cfg"
+)
+
+func CcrFollowerAdd(conf *cfg.Config, index string) error {
+	res, err := conf.DefaultCluster.ES.Cluster.RemoteInfo().
+		Header("content-type", "application/json").
+		Header("accept", "application/json").
+		Do(context.Background())
+	if err != nil {
+		return fmt.Errorf("failed to retrieve follower info: %s", err)
+	}
+
+	remote := ""
+	for name := range res {
+		remote = name
+		break
+	}
+
+	if remote == "" {
+		return fmt.Errorf("cluster doesn't have a follower: %s", err)
+	}
+
+	create := conf.DefaultCluster.ES.Ccr.Follow(index).
+		LeaderIndex(index).
+		RemoteCluster(remote).
+		Header("content-type", "application/json").
+		Header("accept", "application/json")
+
+	if conf.Wait {
+		create.WaitForActiveShards("all")
+	}
+
+	_, err = create.Do(context.Background())
+
+	if err != nil {
+		return fmt.Errorf("failed to create follower index: %s", err)
+	}
+
+	return nil
+}
+
+func CcrFollowerShow(conf *cfg.Config, index string) error {
+	res, err := conf.DefaultCluster.ES.Ccr.FollowStats(index).
+		Header("content-type", "application/json").
+		Header("accept", "application/json").
+		Do(context.Background())
+	if err != nil {
+		return fmt.Errorf("failed to retrieve follower index info: %s", err)
+	}
+
+	slog.Debug("ES result", "follower stats", res.Indices)
+
+	if len(res.Indices) == 0 {
+		return fmt.Errorf("cluster did not return any follower stats for index %s", index)
+	}
+
+	if len(res.Indices[0].Shards) == 0 {
+		return fmt.Errorf("cluster did not return any shard stats on follower index %s", index)
+	}
+
+	follower := res.Indices[0].Shards[0]
+
+	table := NewTable(2, 9)
+	table.Addheaders("field", "value")
+
+	table.entries = [][]string{
+		{"name", index},
+		{"remote_cluster", follower.RemoteCluster},
+		{"leader_checkpoint", fmt.Sprintf("%d", follower.LeaderGlobalCheckpoint)},
+		{"follower_checkpoint", fmt.Sprintf("%d", follower.FollowerGlobalCheckpoint)},
+		{"bytes_read", fmt.Sprintf("%d", follower.BytesRead)},
+		{"failed_read_requests", fmt.Sprintf("%d", follower.FailedReadRequests)},
+		{"failed_write_requests", fmt.Sprintf("%d", follower.FailedWriteRequests)},
+		{"successful_read_requests", fmt.Sprintf("%d", follower.SuccessfulReadRequests)},
+		{"successful_write_requests", fmt.Sprintf("%d", follower.SuccessfulWriteRequests)},
+	}
+
+	if err := table.PrintMarkdown(); err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/es/cluster.go

@@ -18,7 +18,6 @@ package es
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"log/slog"
 	"slices"
@@ -47,46 +46,6 @@ type apiResponse struct {
 	which  int
 }
 
-func ClusterCompare(conf *cfg.Config, leader, follower string) error {
-	if !checkClusterFollower(conf, leader) {
-		return errors.New("leader/follower attribution is invalid, reverse cluster attribution and retry")
-	}
-
-	indices := ClusterIndices{}
-
-	for _, alias := range []string{leader, follower} {
-		cat := conf.Clusters[alias].ES.Cat.Indices().
-			// we need to add custom request headers, required for older ES instances
-			Header("content-type", "application/json").
-			Header("accept", "application/json")
-
-		res, err := cat.Do(context.Background())
-		if err != nil {
-			return fmt.Errorf("failed to get indicies on %s: %s", alias, err)
-		}
-
-		indices[alias] = make(map[string]*types.IndicesRecord, len(res))
-
-		for _, index := range res {
-			indices[alias][*index.Index] = &index
-		}
-	}
-
-	if !checkClusterStatus(conf, leader, follower) {
-		return errors.New("one of the two clusters is in a failed state")
-	}
-
-	findIlmErrors(conf, leader, follower)
-
-	if findIndicesOnlyOnLeader(conf, indices, leader, follower) &&
-		findOrphanedIndices(conf, indices, leader, follower) &&
-		findFailedFollowerIndices(conf, indices, follower) {
-		fmt.Println("everything's hunky-dory.")
-	}
-
-	return nil
-}
-
 func ClusterList(conf *cfg.Config) error {
 	table := NewTable(3, len(conf.Clusters))
 
@@ -102,8 +61,6 @@ func ClusterList(conf *cfg.Config) error {
 
 	slices.Sort(names)
 
-	idx = 0
-
 	for idx, name := range names {
 		current := name == "default" || name == conf.CurrentCluster
 		cluster := conf.Clusters[name]
@@ -118,7 +75,6 @@ func ClusterList(conf *cfg.Config) error {
 		}
 
 		table.entries[idx] = []string{name, cluster.Uri, fmt.Sprintf("%t", current)}
-		idx++
 	}
 
 	if err := table.PrintMarkdown(); err != nil {

pkg/es/cluster_settings.go

@@ -139,3 +139,22 @@ func ClusterSettingsSet(conf *cfg.Config, args cli.Args) error {
 
 	return nil
 }
+
+func ClusterSettingsSetSingle(conf *cfg.Config, setting, value string) error {
+	message, err := json.Marshal(value)
+	if err != nil {
+		return fmt.Errorf("failed to marshall persistent value <%v> to valid JSON: %s", value, err)
+	}
+
+	_, err = conf.Clusters[conf.CurrentCluster].ES.Cluster.PutSettings().
+		AddPersistent(setting, message).
+		Header("content-type", "application/json").
+		Header("accept", "application/json").
+		Do(context.Background())
+
+	if err != nil {
+		return fmt.Errorf("failed to set %s: %s", setting, err)
+	}
+
+	return nil
+}

pkg/es/cluster_util.go

@@ -34,7 +34,7 @@ const (
 	DefaultExclude = `(part|monitoring|.internal|metrics-endpoint)`
 )
 
-func checkClusterFollower(conf *cfg.Config, leader string) bool {
+func checkClusterIsLeader(conf *cfg.Config, leader string) bool {
 	stats, err := conf.Clusters[leader].ES.Ccr.Stats().
 		Header("content-type", "application/json").
 		Header("accept", "application/json").

pkg/es/doc.go

@@ -0,0 +1,55 @@
+/*
+Copyright © 2026 Thomas von Dein
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+package es
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"codeberg.org/scip/esctl/pkg/cfg"
+)
+
+// create index with:
+//
+// esctl index create foo [id:keyword user:text age:integer]
+//
+// then add a doc:
+//
+// esctl doc add foo2 '{"id":"d8d8d","user":"scip"}'
+func DocAdd(conf *cfg.Config, index, jsondoc string) error {
+	data := map[string]any{}
+
+	err := json.Unmarshal([]byte(jsondoc), &data)
+	if err != nil {
+		return fmt.Errorf("supplied document was not valid JSON: %s", err)
+	}
+
+	now := fmt.Sprintf("%d", time.Now().Unix())
+
+	_, err = conf.DefaultCluster.ES.Create(index, now).
+		Document(data).
+		Header("content-type", "application/json").
+		Header("accept", "application/json").
+		Do(context.Background())
+	if err != nil {
+		return fmt.Errorf("failed to create new doc in index %s: %s", index, err)
+	}
+
+	return nil
+}

pkg/es/index.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"log/slog"
 	"strconv"
+	"strings"
 	"time"
 
 	"codeberg.org/scip/esctl/pkg/cfg"
@@ -134,11 +135,7 @@ func IndexShow(conf *cfg.Config, index string) error {
 	return nil
 }
 
-func IndexCreate(conf *cfg.Config, index string) error {
-	if index == "" {
-		return fmt.Errorf("no index specified")
-	}
-
+func IndexCreate(conf *cfg.Config, index string, mappings []string) error {
 	settings := esdsl.NewIndexSettings()
 
 	create := conf.DefaultCluster.ES.Indices.Create(index).
@@ -156,6 +153,30 @@ func IndexCreate(conf *cfg.Config, index string) error {
 		settings = settings.NumberOfReplicas(strconv.Itoa(conf.Replicas))
 	}
 
+	if len(mappings) > 0 {
+		maps := esdsl.NewTypeMapping()
+
+		for _, mapping := range mappings {
+			parts := strings.Split(mapping, ":")
+			if len(parts) != 2 {
+				return fmt.Errorf("invalid mapping %s, expect <name:type> (type: integer, text, date, keyword)", mapping)
+			}
+
+			switch parts[1] {
+			case "text":
+				maps.AddProperty(parts[0], esdsl.NewTextProperty())
+			case "integer":
+				maps.AddProperty(parts[0], esdsl.NewIntegerNumberProperty())
+			case "date":
+				maps.AddProperty(parts[0], esdsl.NewDateProperty())
+			case "keyword":
+				maps.AddProperty(parts[0], esdsl.NewKeywordProperty())
+			}
+		}
+
+		create.Mappings(maps)
+	}
+
 	_, err := create.Settings(settings).
 		Do(context.Background())
 

pkg/es/search.go

@@ -34,9 +34,17 @@ Execute an ES search.
 	q is the actual search query given as arg to the 'search' cmd
 	additional filters can be given as -F key=value
 */
-func Search(conf *cfg.Config, q string) error {
-	query := esdsl.NewBoolQuery().
-		Must(esdsl.NewMatchQuery("message", q))
+func Search(conf *cfg.Config, queries []string) error {
+	query := esdsl.NewBoolQuery()
+
+	for _, q := range queries {
+		parts := strings.Split(q, "=")
+		if len(parts) != 2 {
+			return fmt.Errorf("search queries must be in the form field=pattern")
+		}
+
+		query.Must(esdsl.NewMatchQuery(parts[0], parts[1]))
+	}
 
 	filters := make([]types.QueryVariant, len(conf.Filter))