mirror of
https://codeberg.org/scip/esctl.git
synced 2026-08-24 05:04:18 +02:00
add index template {ls,sh} (#33)
This commit is contained in:
T. von Dein
449
pkg/es/index_template.go
Normal file
449
pkg/es/index_template.go
Normal file
@@ -0,0 +1,449 @@
|
||||
/*
|
||||
Copyright © 2026 Thomas von Dein
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package es
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"codeberg.org/scip/esctl/pkg/cfg"
|
||||
"codeberg.org/scip/esctl/pkg/printer"
|
||||
"github.com/elastic/go-elasticsearch/v9/typedapi/esdsl"
|
||||
"github.com/elastic/go-elasticsearch/v9/typedapi/types"
|
||||
)
|
||||
|
||||
// used for completion
|
||||
func IndexTemplateList(conf *cfg.Config) error {
|
||||
res, err := conf.DefaultCluster.ES.Indices.GetIndexTemplate().
|
||||
Header("content-type", "application/json").
|
||||
Header("accept", "application/json").
|
||||
Do(context.Background())
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get index templates: %s", esErrorString(err))
|
||||
}
|
||||
|
||||
slog.Debug("res", "index templates", res)
|
||||
|
||||
table := printer.NewTable(conf, 5, len(res.IndexTemplates))
|
||||
table.Addheaders("name", "description", "priority")
|
||||
|
||||
for idx, tpl := range res.IndexTemplates {
|
||||
desc, err := json.Marshal(tpl.IndexTemplate.Meta_["description"])
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to unmarshal meta json data: %w", err)
|
||||
}
|
||||
|
||||
table.Entries[idx] = []string{
|
||||
tpl.Name,
|
||||
string(desc),
|
||||
fmt.Sprintf("%d", tpl.IndexTemplate.Priority),
|
||||
}
|
||||
}
|
||||
|
||||
table.Sort()
|
||||
return table.Print()
|
||||
}
|
||||
|
||||
func IndexTemplateShow(conf *cfg.Config, tplname string) error {
|
||||
res, err := conf.DefaultCluster.ES.Indices.GetIndexTemplate().
|
||||
Name(tplname).
|
||||
Header("content-type", "application/json").
|
||||
Header("accept", "application/json").
|
||||
Do(context.Background())
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get index template: %s", esErrorString(err))
|
||||
}
|
||||
|
||||
slog.Debug("res", "index template", res)
|
||||
|
||||
if len(res.IndexTemplates) != 1 {
|
||||
return errors.New("multiple or no index template matched the pattern")
|
||||
}
|
||||
|
||||
tpl := res.IndexTemplates[0]
|
||||
|
||||
table := printer.NewTable(conf, 2, 6)
|
||||
table.Addheaders("index template property", "value")
|
||||
|
||||
desc, err := json.Marshal(tpl.IndexTemplate.Meta_["description"])
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to unmarshal meta json data: %w", err)
|
||||
}
|
||||
|
||||
hasds := tpl.IndexTemplate.DataStream != nil
|
||||
|
||||
aliases := []string{}
|
||||
for alias := range tpl.IndexTemplate.Template.Aliases {
|
||||
aliases = append(aliases, alias)
|
||||
}
|
||||
|
||||
table.Entries = [][]string{
|
||||
{"name", tpl.Name},
|
||||
{"description", string(desc)},
|
||||
{"index patterns", strings.Join(tpl.IndexTemplate.IndexPatterns, ",")},
|
||||
{"composed of", strings.Join(tpl.IndexTemplate.ComposedOf, ",")},
|
||||
{"data stream enabled", fmt.Sprintf("%t", hasds)},
|
||||
{"aliases", strings.Join(aliases, ",")},
|
||||
}
|
||||
|
||||
if err := table.Print(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
table = printer.NewTable(conf, 2, 0)
|
||||
table.Addheaders("index setting property", "value")
|
||||
|
||||
err = getIndexTemplateSettings(conf, tplname, table)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
fmt.Println()
|
||||
if err := table.Print(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
table = printer.NewTable(conf, 2, 0)
|
||||
table.Addheaders("index field mapping", "type")
|
||||
|
||||
for name, field := range tpl.IndexTemplate.Template.Mappings.Properties {
|
||||
typeval := ""
|
||||
|
||||
switch val := field.(type) {
|
||||
case *types.IntegerNumberProperty:
|
||||
typeval = val.Type
|
||||
case *types.KeywordProperty:
|
||||
typeval = val.Type
|
||||
case *types.DateProperty:
|
||||
typeval = val.Type
|
||||
}
|
||||
|
||||
table.Entries = append(table.Entries, []string{
|
||||
name, typeval,
|
||||
})
|
||||
}
|
||||
|
||||
fmt.Println()
|
||||
if err := table.Print(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func IndexTemplateCreate(conf *cfg.Config, name string, mappings []string) error {
|
||||
settings := esdsl.NewIndexSettings()
|
||||
maps := esdsl.NewIndexTemplateMapping()
|
||||
|
||||
create := conf.DefaultCluster.ES.Indices.PutIndexTemplate(name).
|
||||
Header("content-type", "application/json").
|
||||
Header("accept", "application/json")
|
||||
|
||||
if conf.Shards > 0 {
|
||||
settings = settings.NumberOfShards(strconv.Itoa(conf.Shards))
|
||||
}
|
||||
|
||||
if conf.Replicas > 0 {
|
||||
settings = settings.NumberOfReplicas(strconv.Itoa(conf.Replicas))
|
||||
}
|
||||
|
||||
if conf.Stream {
|
||||
create.DataStream(esdsl.NewDataStreamVisibility())
|
||||
|
||||
if conf.Retention != "" {
|
||||
maps.Lifecycle(
|
||||
esdsl.NewDataStreamLifecycle().
|
||||
DataRetention(
|
||||
esdsl.NewDuration().String(conf.Retention)))
|
||||
}
|
||||
}
|
||||
|
||||
if conf.AutoCreate {
|
||||
create.AllowAutoCreate(true)
|
||||
}
|
||||
|
||||
if conf.Mode != "" {
|
||||
if !slices.Contains([]string{"standard", "timeseries", "logsdb", "lookup"}, conf.Mode) {
|
||||
return errors.New("mode must be one of: standard, timeseries, logsdb or lookup")
|
||||
}
|
||||
|
||||
settings = settings.Mode(conf.Mode)
|
||||
}
|
||||
|
||||
if len(mappings) > 0 {
|
||||
typemaps := esdsl.NewTypeMapping()
|
||||
|
||||
for _, mapping := range mappings {
|
||||
parts := strings.Split(mapping, ":")
|
||||
if len(parts) != 2 {
|
||||
return fmt.Errorf(
|
||||
"invalid mapping %s, expect <name:type> (type: integer, text, date, keyword)",
|
||||
mapping)
|
||||
}
|
||||
|
||||
switch parts[1] {
|
||||
case "text":
|
||||
typemaps.AddProperty(parts[0], esdsl.NewTextProperty())
|
||||
case "integer":
|
||||
typemaps.AddProperty(parts[0], esdsl.NewIntegerNumberProperty())
|
||||
case "date":
|
||||
typemaps.AddProperty(parts[0], esdsl.NewDateProperty())
|
||||
case "keyword":
|
||||
typemaps.AddProperty(parts[0], esdsl.NewKeywordProperty())
|
||||
}
|
||||
}
|
||||
|
||||
maps.Mappings(typemaps)
|
||||
}
|
||||
|
||||
if len(conf.Components) > 0 {
|
||||
create.ComposedOf(conf.Components...)
|
||||
}
|
||||
|
||||
for _, alias := range conf.Aliases {
|
||||
maps.AddAlias(alias, esdsl.NewAlias())
|
||||
}
|
||||
|
||||
if len(conf.Meta) > 0 {
|
||||
metadata := map[string]json.RawMessage{}
|
||||
|
||||
for _, meta := range conf.Meta {
|
||||
parts := strings.Split(meta, ":")
|
||||
if len(parts) != 2 {
|
||||
return errors.New("meta data must be in the form key:value")
|
||||
}
|
||||
|
||||
msg, err := json.Marshal(parts[1])
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to json marshal metadata %s: %w", meta, err)
|
||||
}
|
||||
|
||||
metadata[parts[0]] = msg
|
||||
}
|
||||
|
||||
create.Meta_(esdsl.NewMetadata(metadata))
|
||||
}
|
||||
|
||||
if len(conf.Settings) > 0 {
|
||||
usersettings := map[string]json.RawMessage{}
|
||||
|
||||
for _, meta := range conf.Settings {
|
||||
parts := strings.Split(meta, ":")
|
||||
if len(parts) != 2 {
|
||||
return errors.New("settings data must be in the form key:value")
|
||||
}
|
||||
|
||||
msg, err := json.Marshal(parts[1])
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to json marshal metadata %s: %w", meta, err)
|
||||
}
|
||||
|
||||
usersettings[parts[0]] = msg
|
||||
}
|
||||
|
||||
settings = settings.IndexSettings(usersettings)
|
||||
}
|
||||
|
||||
maps.Settings(settings)
|
||||
create.Template(maps)
|
||||
create.IndexPatterns(conf.Patterns...)
|
||||
|
||||
_, err := create.Do(context.Background())
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create index template: %s", esErrorString(err))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// FIXME: func's too long, refactor
|
||||
// FIXME: it's not yet possible to remove items in lists (aliases, mappings, settings), just overwrite them
|
||||
func IndexTemplateModify(conf *cfg.Config, name string, mappings []string) error {
|
||||
maps := esdsl.NewIndexTemplateMapping()
|
||||
|
||||
// load existing index mapping
|
||||
res, err := conf.DefaultCluster.ES.Indices.GetIndexTemplate().
|
||||
Name(name).
|
||||
Header("content-type", "application/json").
|
||||
Header("accept", "application/json").
|
||||
Do(context.Background())
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get index template: %s", esErrorString(err))
|
||||
}
|
||||
|
||||
slog.Debug("res", "index template", res)
|
||||
|
||||
if len(res.IndexTemplates) != 1 {
|
||||
return errors.New("multiple or no index template matched the pattern")
|
||||
}
|
||||
|
||||
tpl := res.IndexTemplates[0]
|
||||
|
||||
// our modify PUT request
|
||||
modify := conf.DefaultCluster.ES.Indices.PutIndexTemplate(name).
|
||||
Header("content-type", "application/json").
|
||||
Header("accept", "application/json")
|
||||
|
||||
// load existing settings, if any
|
||||
settings := tpl.IndexTemplate.Template.Settings
|
||||
|
||||
// pre fill mappings and aliases
|
||||
maps.Mappings(tpl.IndexTemplate.Template.Mappings)
|
||||
maps.Aliases(tpl.IndexTemplate.Template.Aliases)
|
||||
|
||||
// pre fill meta, if any
|
||||
metadata := map[string]json.RawMessage{}
|
||||
for key, value := range tpl.IndexTemplate.Meta_ {
|
||||
metadata[key] = value
|
||||
}
|
||||
|
||||
// pre fill components
|
||||
modify.ComposedOf(tpl.IndexTemplate.ComposedOf...)
|
||||
|
||||
if conf.Stream {
|
||||
modify.DataStream(esdsl.NewDataStreamVisibility())
|
||||
|
||||
if conf.Retention != "" {
|
||||
maps.Lifecycle(
|
||||
esdsl.NewDataStreamLifecycle().
|
||||
DataRetention(
|
||||
esdsl.NewDuration().String(conf.Retention)))
|
||||
}
|
||||
}
|
||||
|
||||
modify.AllowAutoCreate(conf.AutoCreate)
|
||||
|
||||
if conf.Mode != "" {
|
||||
if !slices.Contains([]string{"standard", "timeseries", "logsdb", "lookup"}, conf.Mode) {
|
||||
return errors.New("mode must be one of: standard, timeseries, logsdb or lookup")
|
||||
}
|
||||
|
||||
*settings.Mode = conf.Mode
|
||||
}
|
||||
|
||||
if len(mappings) > 0 {
|
||||
typemaps := esdsl.NewTypeMapping()
|
||||
|
||||
for _, mapping := range mappings {
|
||||
parts := strings.Split(mapping, ":")
|
||||
if len(parts) != 2 {
|
||||
return fmt.Errorf(
|
||||
"invalid mapping %s, expect <name:type> (type: integer, text, date, keyword)",
|
||||
mapping)
|
||||
}
|
||||
|
||||
switch parts[1] {
|
||||
case "text":
|
||||
typemaps.AddProperty(parts[0], esdsl.NewTextProperty())
|
||||
case "integer":
|
||||
typemaps.AddProperty(parts[0], esdsl.NewIntegerNumberProperty())
|
||||
case "date":
|
||||
typemaps.AddProperty(parts[0], esdsl.NewDateProperty())
|
||||
case "keyword":
|
||||
typemaps.AddProperty(parts[0], esdsl.NewKeywordProperty())
|
||||
}
|
||||
}
|
||||
|
||||
maps.Mappings(typemaps)
|
||||
}
|
||||
|
||||
if len(conf.Components) > 0 {
|
||||
modify.ComposedOf(conf.Components...)
|
||||
}
|
||||
|
||||
for _, alias := range conf.Aliases {
|
||||
maps.AddAlias(alias, esdsl.NewAlias())
|
||||
}
|
||||
|
||||
if len(conf.Meta) > 0 {
|
||||
for _, meta := range conf.Meta {
|
||||
parts := strings.Split(meta, ":")
|
||||
if len(parts) != 2 {
|
||||
return errors.New("meta data must be in the form key:value")
|
||||
}
|
||||
|
||||
msg, err := json.Marshal(parts[1])
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to json marshal metadata %s: %w", meta, err)
|
||||
}
|
||||
|
||||
metadata[parts[0]] = msg
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
modify.Meta_(esdsl.NewMetadata(metadata))
|
||||
|
||||
if len(conf.Settings) > 0 {
|
||||
usersettings := map[string]json.RawMessage{}
|
||||
|
||||
for _, meta := range conf.Settings {
|
||||
parts := strings.Split(meta, ":")
|
||||
if len(parts) != 2 {
|
||||
return errors.New("settings data must be in the form key:value")
|
||||
}
|
||||
|
||||
msg, err := json.Marshal(parts[1])
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to json marshal metadata %s: %w", meta, err)
|
||||
}
|
||||
|
||||
usersettings[parts[0]] = msg
|
||||
}
|
||||
|
||||
settings.IndexSettings = usersettings
|
||||
}
|
||||
|
||||
patterns := tpl.IndexTemplate.IndexPatterns
|
||||
if len(conf.Patterns) > 0 {
|
||||
patterns = conf.Patterns
|
||||
}
|
||||
|
||||
maps.Settings(settings)
|
||||
modify.Template(maps)
|
||||
modify.IndexPatterns(patterns...)
|
||||
|
||||
_, err = modify.Do(context.Background())
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to modify index template: %s", esErrorString(err))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func IndexTemplateDelete(conf *cfg.Config, name string) error {
|
||||
_, err := conf.DefaultCluster.ES.Indices.DeleteIndexTemplate(name).
|
||||
Header("content-type", "application/json").
|
||||
Header("accept", "application/json").
|
||||
Do(context.Background())
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete index template: %s", esErrorString(err))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user