mirror of
https://codeberg.org/scip/esctl.git
synced 2026-08-24 14:14:23 +02:00
Compare commits
10 Commits
internal/g
...
feature/ca
| Author | SHA1 | Date | |
|---|---|---|---|
| b0d671b06f | |||
| 01247ae744 | |||
| 2fed42fbf4 | |||
| c855a7ebbe | |||
| a366cb5010 | |||
| 67faee8f1c | |||
| 4ddddb4cb0 | |||
| 035194b6f0 | |||
| 392249f483 | |||
| 90c7da98ea |
3
.gitignore
vendored
3
.gitignore
vendored
@@ -32,3 +32,6 @@ esctl
|
|||||||
*.log
|
*.log
|
||||||
|
|
||||||
cpu.profile
|
cpu.profile
|
||||||
|
|
||||||
|
t
|
||||||
|
single
|
||||||
|
|||||||
5
Makefile
5
Makefile
@@ -62,9 +62,8 @@ install: buildlocal
|
|||||||
clean:
|
clean:
|
||||||
rm -rf $(tool) coverage.out testdata t/out pkg/es/openspec.go
|
rm -rf $(tool) coverage.out testdata t/out pkg/es/openspec.go
|
||||||
|
|
||||||
test: clean
|
test: clean buildlocal
|
||||||
mkdir -p t/out
|
make -C t test
|
||||||
go test ./... $(ARGS)
|
|
||||||
|
|
||||||
testlint: test lint
|
testlint: test lint
|
||||||
|
|
||||||
|
|||||||
@@ -113,7 +113,7 @@ func (cluster *Cluster) CheckAuth() error {
|
|||||||
cluster.Pass = pass
|
cluster.Pass = pass
|
||||||
} else {
|
} else {
|
||||||
// k, try interactively
|
// k, try interactively
|
||||||
fmt.Printf("Enter password for elasticsearch user %s@%s: ", cluster.User, cluster.Name)
|
fmt.Fprintf(os.Stderr, "Enter password for elasticsearch user %s@%s: ", cluster.User, cluster.Name)
|
||||||
pass, err := term.ReadPassword(int(syscall.Stdin))
|
pass, err := term.ReadPassword(int(syscall.Stdin))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
Version string = `v0.0.25`
|
Version string = `v0.0.26`
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -121,10 +121,11 @@ func getDefaultPath() string {
|
|||||||
func (conf *Config) Init() error {
|
func (conf *Config) Init() error {
|
||||||
DefaultConfig := getDefaultPath()
|
DefaultConfig := getDefaultPath()
|
||||||
|
|
||||||
switch {
|
if conf.ConfigFile == "" && fileExists(DefaultConfig) {
|
||||||
case fileExists(DefaultConfig):
|
|
||||||
conf.ConfigFile = DefaultConfig
|
conf.ConfigFile = DefaultConfig
|
||||||
fallthrough
|
}
|
||||||
|
|
||||||
|
switch {
|
||||||
case conf.ConfigFile != "":
|
case conf.ConfigFile != "":
|
||||||
if err := conf.LoadConfig(); err != nil {
|
if err := conf.LoadConfig(); err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -102,7 +102,7 @@ func CcrRemoteInfo(conf *cfg.Config, index string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if remote == "" {
|
if remote == "" {
|
||||||
return fmt.Errorf("cluster doesn't follow any other: %s", err)
|
return errors.New("cluster doesn't follow any other")
|
||||||
}
|
}
|
||||||
|
|
||||||
mode := "follower"
|
mode := "follower"
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|||||||
package es
|
package es
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -118,12 +119,12 @@ func getClusterStatus(conf *cfg.Config) (*apiResponse, error) {
|
|||||||
|
|
||||||
all := apiResponse{}
|
all := apiResponse{}
|
||||||
|
|
||||||
|
var err error
|
||||||
|
|
||||||
for i := 0; i < gocount; i++ {
|
for i := 0; i < gocount; i++ {
|
||||||
r := <-responses
|
r := <-responses
|
||||||
|
|
||||||
if r.error != nil {
|
err = errors.Join(err, r.error)
|
||||||
return nil, r.error
|
|
||||||
}
|
|
||||||
|
|
||||||
switch r.which {
|
switch r.which {
|
||||||
case ResponseHealth:
|
case ResponseHealth:
|
||||||
@@ -143,20 +144,23 @@ func getClusterStatus(conf *cfg.Config) (*apiResponse, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return &all, nil
|
return &all, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func ClusterStatus(conf *cfg.Config) error {
|
func ClusterStatus(conf *cfg.Config) error {
|
||||||
res, err := getClusterStatus(conf)
|
res, err := getClusterStatus(conf)
|
||||||
if err != nil {
|
if err != nil && !strings.Contains(err.Error(), "current license is non-compliant") {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
slog.Debug("ES result", "cluster health", res.health)
|
slog.Debug("ES result", "cluster health", res.health)
|
||||||
|
|
||||||
isleader := len(res.ccr.AutoFollowStats.AutoFollowedClusters) == 0
|
var isleader bool
|
||||||
|
var ccrfollowing string
|
||||||
|
|
||||||
|
if res.ccr != nil {
|
||||||
|
isleader = len(res.ccr.AutoFollowStats.AutoFollowedClusters) == 0
|
||||||
|
|
||||||
ccrfollowing := ""
|
|
||||||
if len(res.ccr.AutoFollowStats.AutoFollowedClusters) > 0 {
|
if len(res.ccr.AutoFollowStats.AutoFollowedClusters) > 0 {
|
||||||
// is following another cluster
|
// is following another cluster
|
||||||
ccrfollowing = fmt.Sprintf("%s (%d/%d)",
|
ccrfollowing = fmt.Sprintf("%s (%d/%d)",
|
||||||
@@ -165,6 +169,7 @@ func ClusterStatus(conf *cfg.Config) error {
|
|||||||
res.ccr.AutoFollowStats.NumberOfFailedFollowIndices,
|
res.ccr.AutoFollowStats.NumberOfFailedFollowIndices,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// look for red indices, if any
|
// look for red indices, if any
|
||||||
redindices := 0
|
redindices := 0
|
||||||
@@ -200,7 +205,7 @@ func ClusterStatus(conf *cfg.Config) error {
|
|||||||
{"Long Running Tasks", fmt.Sprintf("%d", longtasks)},
|
{"Long Running Tasks", fmt.Sprintf("%d", longtasks)},
|
||||||
}
|
}
|
||||||
|
|
||||||
if !isleader {
|
if !isleader && res.ccr != nil {
|
||||||
table.Entries = append(table.Entries, [][]string{
|
table.Entries = append(table.Entries, [][]string{
|
||||||
{"AutoFollow (success/failed indices)", ccrfollowing},
|
{"AutoFollow (success/failed indices)", ccrfollowing},
|
||||||
{"Followed Indices", fmt.Sprintf("%d", len(res.ccr.FollowStats.Indices))},
|
{"Followed Indices", fmt.Sprintf("%d", len(res.ccr.FollowStats.Indices))},
|
||||||
|
|||||||
@@ -67,6 +67,7 @@ func IlmNames(conf *cfg.Config) ([]string, error) {
|
|||||||
|
|
||||||
for name := range res {
|
for name := range res {
|
||||||
names[idx] = name
|
names[idx] = name
|
||||||
|
idx++
|
||||||
}
|
}
|
||||||
|
|
||||||
return names, nil
|
return names, nil
|
||||||
|
|||||||
@@ -39,18 +39,28 @@ func LicenseShow(conf *cfg.Config) error {
|
|||||||
lic := res.License
|
lic := res.License
|
||||||
|
|
||||||
var maxnodes = "infinite"
|
var maxnodes = "infinite"
|
||||||
|
var maxunits = "infinite"
|
||||||
|
var expire = "never"
|
||||||
|
|
||||||
if lic.MaxNodes != nil {
|
if lic.MaxNodes != nil {
|
||||||
maxnodes = fmt.Sprintf("%d", *lic.MaxNodes)
|
maxnodes = fmt.Sprintf("%d", *lic.MaxNodes)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if lic.ExpiryDate != nil {
|
||||||
|
expire = lic.ExpiryDate.(string)
|
||||||
|
}
|
||||||
|
|
||||||
|
if lic.MaxResourceUnits != nil {
|
||||||
|
maxunits = fmt.Sprintf("%d", *lic.MaxResourceUnits)
|
||||||
|
}
|
||||||
|
|
||||||
table.Entries = [][]string{
|
table.Entries = [][]string{
|
||||||
{"UID", lic.Uid},
|
{"UID", lic.Uid},
|
||||||
{"Issued to", lic.IssuedTo},
|
{"Issued to", lic.IssuedTo},
|
||||||
{"Expires", lic.ExpiryDate.(string)},
|
{"Expires", expire},
|
||||||
{"Issued", lic.IssueDate.(string)},
|
{"Issued", lic.IssueDate.(string)},
|
||||||
{"Max nodes", maxnodes},
|
{"Max nodes", maxnodes},
|
||||||
{"Max resource units", fmt.Sprintf("%d", *lic.MaxResourceUnits)},
|
{"Max resource units", maxunits},
|
||||||
{"Type", lic.Type.Name},
|
{"Type", lic.Type.Name},
|
||||||
{"Status", lic.Status.Name},
|
{"Status", lic.Status.Name},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -52,6 +52,6 @@ func PrintDoc(conf *cfg.Config, hit types.Hit) {
|
|||||||
value := gjson.Get(docjson, conf.Path)
|
value := gjson.Get(docjson, conf.Path)
|
||||||
fmt.Println(value.String())
|
fmt.Println(value.String())
|
||||||
} else {
|
} else {
|
||||||
fmt.Print(docjson)
|
fmt.Println(docjson)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
6
t/.env
Normal file
6
t/.env
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
ES_PASS=tuscador1
|
||||||
|
STACK_VERSION=9.4.2
|
||||||
|
CLUSTER_NAME=domdoc
|
||||||
|
LICENSE=basic
|
||||||
|
ES_PORT=9200
|
||||||
|
MEM_LIMIT=1073741824
|
||||||
46
t/Makefile
Normal file
46
t/Makefile
Normal file
@@ -0,0 +1,46 @@
|
|||||||
|
.PHONY: test clean cluster docs search up down delete
|
||||||
|
|
||||||
|
# docker stuff
|
||||||
|
up:
|
||||||
|
@echo "booting up docker containers"
|
||||||
|
docker compose up -d --remove-orphans
|
||||||
|
|
||||||
|
waitup:
|
||||||
|
@echo "wait til es cluster is up and running"
|
||||||
|
mosscap msc-actions.yaml msc-docker.yaml -n -t 300
|
||||||
|
|
||||||
|
down:
|
||||||
|
@echo "shutting down docker containers"
|
||||||
|
docker compose down
|
||||||
|
|
||||||
|
delete:
|
||||||
|
@echo "delete docker volumes"
|
||||||
|
docker volume rm t_certs t_esdata01 t_esdata02 t_esdata03
|
||||||
|
|
||||||
|
clean-docker: down delete
|
||||||
|
|
||||||
|
|
||||||
|
# mosscap esctl stuff
|
||||||
|
test: up waitup cluster docs wait search
|
||||||
|
|
||||||
|
cluster:
|
||||||
|
@echo "setting up elastisearch cluster"
|
||||||
|
mosscap msc-actions.yaml msc-cluster.yaml -n
|
||||||
|
|
||||||
|
docs:
|
||||||
|
@echo "put some docs"
|
||||||
|
mosscap msc-actions.yaml msc-docs.yaml -n -c 1000 -p 100
|
||||||
|
|
||||||
|
wait:
|
||||||
|
sleep 10
|
||||||
|
|
||||||
|
search:
|
||||||
|
@echo "make some searches"
|
||||||
|
mosscap msc-actions.yaml msc-search.yaml -n -t 60
|
||||||
|
|
||||||
|
clean:
|
||||||
|
@echo "cleaning up cluster"
|
||||||
|
mosscap msc-actions.yaml msc-clean.yaml -n
|
||||||
|
rm -f *.state debug.log
|
||||||
|
|
||||||
|
|
||||||
76
t/README.md
Normal file
76
t/README.md
Normal file
@@ -0,0 +1,76 @@
|
|||||||
|
# Testing
|
||||||
|
|
||||||
|
Building regular unit tests would require to create a mock
|
||||||
|
elasticsearch cluster, which is too much a chrore for a one man
|
||||||
|
show. Running a regular elasticsearch cluster on Codeberg CI is not
|
||||||
|
economically reasonable.
|
||||||
|
|
||||||
|
Therefore I run tests manually. Here's how.
|
||||||
|
|
||||||
|
# Dependencies
|
||||||
|
|
||||||
|
The following is required:
|
||||||
|
|
||||||
|
- linux, any distro will do
|
||||||
|
- docker in a decent version
|
||||||
|
- [mosscap](https://codeberg.org/scip/mosscap) version `0.0.17` or higher.
|
||||||
|
|
||||||
|
# Docker
|
||||||
|
|
||||||
|
There's a ready to use docker compose file, which fires up an
|
||||||
|
elasticsearch cluster with 3 nodes. To start it manually just execute
|
||||||
|
`make up`.
|
||||||
|
|
||||||
|
When it's up and running it should look like this:
|
||||||
|
|
||||||
|
```console
|
||||||
|
NAMES STATUS PORTS
|
||||||
|
t-es03-1 Up 16 minutes (healthy) 9200/tcp, 9300/tcp
|
||||||
|
t-es02-1 Up 16 minutes (healthy) 9200/tcp, 9300/tcp
|
||||||
|
t-es01-1 Up 16 minutes (healthy) 0.0.0.0:9200->9200/tcp, [::]:9200->9200/tcp, 9300/tcp
|
||||||
|
```
|
||||||
|
|
||||||
|
There's an esctl config file `cluster.yaml`, which you can use to
|
||||||
|
access that elasticsearch cluster, e.g.:
|
||||||
|
|
||||||
|
```console
|
||||||
|
esctl -c cluster.yaml cluster status
|
||||||
|
DOCKER/DOMDOC STATUS
|
||||||
|
Cluster Name domdoc
|
||||||
|
ES Status green
|
||||||
|
ES Version 9.4.2
|
||||||
|
Is Leader false
|
||||||
|
Active Shards 8
|
||||||
|
Active Primary Shards 4
|
||||||
|
Unassigned Shards 0
|
||||||
|
Unassigned Primary Shards 0
|
||||||
|
Pending Tasks 0
|
||||||
|
Nodes 3
|
||||||
|
Red Indices 0
|
||||||
|
Long Running Tasks 0
|
||||||
|
```
|
||||||
|
|
||||||
|
# Automated Tests
|
||||||
|
|
||||||
|
To execute the automated tests, just execute `make test`. This boots
|
||||||
|
up the elasticsearch containers, waits until they are up and running,
|
||||||
|
creates data on it and queries it.
|
||||||
|
|
||||||
|
# Cleanup
|
||||||
|
|
||||||
|
To just clean up the elasticsearch cluster run `make clean`.
|
||||||
|
|
||||||
|
To remove everything, run `make clean-docker`.
|
||||||
|
|
||||||
|
# Reference
|
||||||
|
|
||||||
|
## Mosscap configs
|
||||||
|
|
||||||
|
| CONFIG | DESCRIPTION |
|
||||||
|
|------------------|----------------------------------------------------------------------|
|
||||||
|
| msc-actions.yaml | contains all actions, used by all item configs |
|
||||||
|
| msc-docker.yaml | single item: used to wait until elasticsearch containers are healthy |
|
||||||
|
| msc-cluster.yaml | single item: create indices etc |
|
||||||
|
| msc-docs.yaml | generator items: put docs into the index |
|
||||||
|
| msc-search.yaml | single item: do some searches |
|
||||||
|
| msc-clean.yaml | single item: clean up the elasticsearch cluster |
|
||||||
7
t/cluster.yaml
Normal file
7
t/cluster.yaml
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
# esctl config
|
||||||
|
clusters:
|
||||||
|
docker/domdoc:
|
||||||
|
uri: https://elastic:9200
|
||||||
|
user: elastic
|
||||||
|
token: ""
|
||||||
|
default: false
|
||||||
207
t/docker-compose.yaml
Normal file
207
t/docker-compose.yaml
Normal file
@@ -0,0 +1,207 @@
|
|||||||
|
services:
|
||||||
|
setup:
|
||||||
|
image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
|
||||||
|
volumes:
|
||||||
|
- certs:/usr/share/elasticsearch/config/certs
|
||||||
|
user: "0"
|
||||||
|
command: >
|
||||||
|
bash -c '
|
||||||
|
if [ x${ES_PASS} == x ]; then
|
||||||
|
echo "Set the ES_PASS environment variable in the .env file";
|
||||||
|
exit 1;
|
||||||
|
fi;
|
||||||
|
if [ ! -f config/certs/ca.zip ]; then
|
||||||
|
echo "Creating CA";
|
||||||
|
bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
|
||||||
|
unzip config/certs/ca.zip -d config/certs;
|
||||||
|
fi;
|
||||||
|
if [ ! -f config/certs/certs.zip ]; then
|
||||||
|
echo "Creating certs";
|
||||||
|
echo -ne \
|
||||||
|
"instances:\n"\
|
||||||
|
" - name: es01\n"\
|
||||||
|
" dns:\n"\
|
||||||
|
" - es01\n"\
|
||||||
|
" - localhost\n"\
|
||||||
|
" ip:\n"\
|
||||||
|
" - 127.0.0.1\n"\
|
||||||
|
" - name: es02\n"\
|
||||||
|
" dns:\n"\
|
||||||
|
" - es02\n"\
|
||||||
|
" - localhost\n"\
|
||||||
|
" ip:\n"\
|
||||||
|
" - 127.0.0.1\n"\
|
||||||
|
" - name: es03\n"\
|
||||||
|
" dns:\n"\
|
||||||
|
" - es03\n"\
|
||||||
|
" - localhost\n"\
|
||||||
|
" ip:\n"\
|
||||||
|
" - 127.0.0.1\n"\
|
||||||
|
> config/certs/instances.yml;
|
||||||
|
bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
|
||||||
|
unzip config/certs/certs.zip -d config/certs;
|
||||||
|
fi;
|
||||||
|
echo "Setting file permissions"
|
||||||
|
chown -R root:root config/certs;
|
||||||
|
find . -type d -exec chmod 750 \{\} \;;
|
||||||
|
find . -type f -exec chmod 640 \{\} \;;
|
||||||
|
echo "Waiting for Elasticsearch availability";
|
||||||
|
until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
|
||||||
|
echo "All done!";
|
||||||
|
'
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
|
||||||
|
interval: 1s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 120
|
||||||
|
|
||||||
|
es01:
|
||||||
|
depends_on:
|
||||||
|
setup:
|
||||||
|
condition: service_healthy
|
||||||
|
image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
|
||||||
|
volumes:
|
||||||
|
- certs:/usr/share/elasticsearch/config/certs
|
||||||
|
- esdata01:/usr/share/elasticsearch/data
|
||||||
|
ports:
|
||||||
|
- ${ES_PORT}:9200
|
||||||
|
environment:
|
||||||
|
- node.name=es01
|
||||||
|
- cluster.name=${CLUSTER_NAME}
|
||||||
|
- cluster.initial_master_nodes=es01
|
||||||
|
- discovery.seed_hosts=es02
|
||||||
|
- ELASTIC_PASSWORD=${ES_PASS}
|
||||||
|
- bootstrap.memory_lock=true
|
||||||
|
- xpack.security.enabled=true
|
||||||
|
- xpack.security.http.ssl.enabled=true
|
||||||
|
- xpack.security.http.ssl.key=certs/es01/es01.key
|
||||||
|
- xpack.security.http.ssl.certificate=certs/es01/es01.crt
|
||||||
|
- xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
|
||||||
|
- xpack.security.transport.ssl.enabled=true
|
||||||
|
- xpack.security.transport.ssl.key=certs/es01/es01.key
|
||||||
|
- xpack.security.transport.ssl.certificate=certs/es01/es01.crt
|
||||||
|
- xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
|
||||||
|
- xpack.security.transport.ssl.verification_mode=certificate
|
||||||
|
- xpack.license.self_generated.type=${LICENSE}
|
||||||
|
- xpack.ml.use_auto_machine_memory_percent=true
|
||||||
|
- node_roles=data_content
|
||||||
|
- xpack.searchable.snapshot.shared_cache.size="1gb"
|
||||||
|
- cluster.routing.allocation.disk.threshold_enabled=false
|
||||||
|
- cluster.routing.allocation.disk.watermark.low="2gb"
|
||||||
|
- cluster.routing.allocation.disk.watermark.high="1gb"
|
||||||
|
- cluster.routing.allocation.disk.watermark.flood_stage="500mb"
|
||||||
|
mem_limit: ${MEM_LIMIT}
|
||||||
|
ulimits:
|
||||||
|
memlock:
|
||||||
|
soft: -1
|
||||||
|
hard: -1
|
||||||
|
healthcheck:
|
||||||
|
test:
|
||||||
|
[
|
||||||
|
"CMD-SHELL",
|
||||||
|
"curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
|
||||||
|
]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 10s
|
||||||
|
retries: 120
|
||||||
|
|
||||||
|
es02:
|
||||||
|
depends_on:
|
||||||
|
- es01
|
||||||
|
image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
|
||||||
|
volumes:
|
||||||
|
- certs:/usr/share/elasticsearch/config/certs
|
||||||
|
- esdata02:/usr/share/elasticsearch/data
|
||||||
|
environment:
|
||||||
|
- node.name=es02
|
||||||
|
- cluster.name=${CLUSTER_NAME}
|
||||||
|
- cluster.initial_master_nodes=es01,es02
|
||||||
|
- discovery.seed_hosts=es01
|
||||||
|
- ELASTIC_PASSWORD=${ES_PASS}
|
||||||
|
- bootstrap.memory_lock=true
|
||||||
|
- xpack.security.enabled=true
|
||||||
|
- xpack.security.http.ssl.enabled=true
|
||||||
|
- xpack.security.http.ssl.key=certs/es02/es02.key
|
||||||
|
- xpack.security.http.ssl.certificate=certs/es02/es02.crt
|
||||||
|
- xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
|
||||||
|
- xpack.security.transport.ssl.enabled=true
|
||||||
|
- xpack.security.transport.ssl.key=certs/es02/es02.key
|
||||||
|
- xpack.security.transport.ssl.certificate=certs/es02/es02.crt
|
||||||
|
- xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
|
||||||
|
- xpack.security.transport.ssl.verification_mode=certificate
|
||||||
|
- xpack.license.self_generated.type=${LICENSE}
|
||||||
|
- xpack.ml.use_auto_machine_memory_percent=true
|
||||||
|
- cluster.routing.allocation.disk.threshold_enabled=false
|
||||||
|
- cluster.routing.allocation.disk.watermark.low="2gb"
|
||||||
|
- cluster.routing.allocation.disk.watermark.high="1gb"
|
||||||
|
- cluster.routing.allocation.disk.watermark.flood_stage="500mb"
|
||||||
|
mem_limit: ${MEM_LIMIT}
|
||||||
|
ulimits:
|
||||||
|
memlock:
|
||||||
|
soft: -1
|
||||||
|
hard: -1
|
||||||
|
healthcheck:
|
||||||
|
test:
|
||||||
|
[
|
||||||
|
"CMD-SHELL",
|
||||||
|
"curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
|
||||||
|
]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 10s
|
||||||
|
retries: 120
|
||||||
|
|
||||||
|
|
||||||
|
es03:
|
||||||
|
depends_on:
|
||||||
|
- es01
|
||||||
|
image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
|
||||||
|
volumes:
|
||||||
|
- certs:/usr/share/elasticsearch/config/certs
|
||||||
|
- esdata03:/usr/share/elasticsearch/data
|
||||||
|
environment:
|
||||||
|
- node.name=es03
|
||||||
|
- cluster.name=${CLUSTER_NAME}
|
||||||
|
- cluster.initial_master_nodes=es01,es03
|
||||||
|
- discovery.seed_hosts=es01
|
||||||
|
- ELASTIC_PASSWORD=${ES_PASS}
|
||||||
|
- bootstrap.memory_lock=true
|
||||||
|
- xpack.security.enabled=true
|
||||||
|
- xpack.security.http.ssl.enabled=true
|
||||||
|
- xpack.security.http.ssl.key=certs/es03/es03.key
|
||||||
|
- xpack.security.http.ssl.certificate=certs/es03/es03.crt
|
||||||
|
- xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
|
||||||
|
- xpack.security.transport.ssl.enabled=true
|
||||||
|
- xpack.security.transport.ssl.key=certs/es03/es03.key
|
||||||
|
- xpack.security.transport.ssl.certificate=certs/es03/es03.crt
|
||||||
|
- xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
|
||||||
|
- xpack.security.transport.ssl.verification_mode=certificate
|
||||||
|
- xpack.license.self_generated.type=${LICENSE}
|
||||||
|
- xpack.ml.use_auto_machine_memory_percent=true
|
||||||
|
- cluster.routing.allocation.disk.threshold_enabled=false
|
||||||
|
- cluster.routing.allocation.disk.watermark.low="2gb"
|
||||||
|
- cluster.routing.allocation.disk.watermark.high="1gb"
|
||||||
|
- cluster.routing.allocation.disk.watermark.flood_stage="500mb"
|
||||||
|
mem_limit: ${MEM_LIMIT}
|
||||||
|
ulimits:
|
||||||
|
memlock:
|
||||||
|
soft: -1
|
||||||
|
hard: -1
|
||||||
|
healthcheck:
|
||||||
|
test:
|
||||||
|
[
|
||||||
|
"CMD-SHELL",
|
||||||
|
"curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
|
||||||
|
]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 10s
|
||||||
|
retries: 120
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
certs:
|
||||||
|
driver: local
|
||||||
|
esdata01:
|
||||||
|
driver: local
|
||||||
|
esdata02:
|
||||||
|
driver: local
|
||||||
|
esdata03:
|
||||||
|
driver: local
|
||||||
96
t/msc-actions.yaml
Normal file
96
t/msc-actions.yaml
Normal file
@@ -0,0 +1,96 @@
|
|||||||
|
#
|
||||||
|
# this is a mosscap action file, see: https://codeberg.org/scip/mosscap
|
||||||
|
vars:
|
||||||
|
esctl: ../esctl -c cluster.yaml
|
||||||
|
|
||||||
|
actions:
|
||||||
|
wait: |
|
||||||
|
sleep [[.time]]
|
||||||
|
|
||||||
|
docker_wait: |
|
||||||
|
for i in {1..20}; do
|
||||||
|
count=$(docker ps | grep healthy | wc -l)
|
||||||
|
if test $count -eq 3; then
|
||||||
|
echo "all containers are healthy"
|
||||||
|
exit
|
||||||
|
fi
|
||||||
|
sleep 5s
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "not all containers are healthy"
|
||||||
|
docker ps
|
||||||
|
|
||||||
|
cluster_ls: |
|
||||||
|
[[.esctl]] cluster ls | grep -E domdoc.*reachable.*yes
|
||||||
|
|
||||||
|
cluster_status: |
|
||||||
|
[[.esctl]] cluster status | grep green
|
||||||
|
|
||||||
|
create_ilm: |
|
||||||
|
[[.esctl]] ilm create --hot-rollover-max-age 7d \
|
||||||
|
--hot-rollover-max-primary-shard-size 25g \
|
||||||
|
--warm-min-age 0 \
|
||||||
|
--delete-min-age 14d [[.policy]]
|
||||||
|
|
||||||
|
describe_ilm: |
|
||||||
|
[[.esctl]] ilm show [[.policy]] -t | grep -E "rollover when storage > 25g"
|
||||||
|
|
||||||
|
create_index_template: |
|
||||||
|
[[.esctl]] index template create [[.index]] -i [[.index]] \
|
||||||
|
-s number_of_shards:3 \
|
||||||
|
-s index.sort.field:@timestamp \
|
||||||
|
-r 1m --ilm-policy [[.policy]] \
|
||||||
|
user:keyword message:text @timestamp:date
|
||||||
|
|
||||||
|
describe_index_template: |
|
||||||
|
[[.esctl]] index template show [[.index]] | grep -E "index.sort.field.*@timestamp"
|
||||||
|
|
||||||
|
create_index: |
|
||||||
|
[[.esctl]] index create -s 2 -r 2 [[.index]]
|
||||||
|
|
||||||
|
describe_index: |
|
||||||
|
[[.esctl]] index show [[.index]] | grep -E "fields.*@timestamp"
|
||||||
|
|
||||||
|
index_ls: |
|
||||||
|
[[.esctl]] index ls | grep [[.index]]
|
||||||
|
|
||||||
|
index_template_rm: |
|
||||||
|
[[.esctl]] index template rm [[.index]]
|
||||||
|
|
||||||
|
index_rm: |
|
||||||
|
[[.esctl]] index rm [[.index]]
|
||||||
|
|
||||||
|
timestamp: |
|
||||||
|
date --iso-8601=second
|
||||||
|
|
||||||
|
date: |
|
||||||
|
date +%Y-%m-%d
|
||||||
|
|
||||||
|
doc_add: |
|
||||||
|
[[.esctl]] doc add -i [[.index]] '{"user": "[[.user]]", "message":"[[.message]]", "@timestamp":"[[.ts]]"}'
|
||||||
|
|
||||||
|
search_any: |
|
||||||
|
[[.esctl]] search -i [[.index]] -l 1 | jq .source.message
|
||||||
|
|
||||||
|
search_date: |
|
||||||
|
[[.esctl]] search -i [[.index]] -l 1 | jq '.source."@timestamp"' | grep [[.today]]
|
||||||
|
|
||||||
|
search_count: |
|
||||||
|
[[.esctl]] search -i [[.index]] -l 5000 | wc -l | grep 1000
|
||||||
|
|
||||||
|
search_content: |
|
||||||
|
[[.esctl]] search -i [[.index]] [[.content_pattern]] -l 1 | jq .source.message
|
||||||
|
|
||||||
|
search_content_and: |
|
||||||
|
[[.esctl]] search -i [[.index]] [[.content_and_pattern]] -l 1 | jq .source.message
|
||||||
|
|
||||||
|
nodes: |
|
||||||
|
[[.esctl]] node ls | grep es01
|
||||||
|
|
||||||
|
describe_node: |
|
||||||
|
[[.esctl]] node show [[.node]] | grep "Node version"
|
||||||
|
|
||||||
|
license: |
|
||||||
|
[[.esctl]] license show | grep -E "Expires.*never"
|
||||||
|
|
||||||
|
|
||||||
14
t/msc-clean.yaml
Normal file
14
t/msc-clean.yaml
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
#
|
||||||
|
# this is a mosscap item file, see: https://codeberg.org/scip/mosscap
|
||||||
|
items:
|
||||||
|
- name: docker/domdoc
|
||||||
|
|
||||||
|
vars:
|
||||||
|
index: test
|
||||||
|
|
||||||
|
tasks:
|
||||||
|
- name: delete_index
|
||||||
|
action: index_rm
|
||||||
|
|
||||||
|
- name: delete_template
|
||||||
|
action: index_template_rm
|
||||||
48
t/msc-cluster.yaml
Normal file
48
t/msc-cluster.yaml
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
#
|
||||||
|
# this is a mosscap item file, see: https://codeberg.org/scip/mosscap
|
||||||
|
statefile: cluster.state
|
||||||
|
|
||||||
|
items:
|
||||||
|
- name: docker/domdoc
|
||||||
|
|
||||||
|
vars:
|
||||||
|
index: test
|
||||||
|
node: es01
|
||||||
|
policy: testpolicy
|
||||||
|
|
||||||
|
tasks:
|
||||||
|
- name: ls
|
||||||
|
action: cluster_ls
|
||||||
|
|
||||||
|
- name: license
|
||||||
|
action: license
|
||||||
|
|
||||||
|
- name: status
|
||||||
|
action: cluster_status
|
||||||
|
|
||||||
|
- name: ilm
|
||||||
|
action: create_ilm
|
||||||
|
|
||||||
|
- name: check_ilm
|
||||||
|
action: describe_ilm
|
||||||
|
|
||||||
|
- name: template
|
||||||
|
action: create_index_template
|
||||||
|
|
||||||
|
- name: check_template
|
||||||
|
action: describe_index_template
|
||||||
|
|
||||||
|
- name: index
|
||||||
|
action: create_index
|
||||||
|
|
||||||
|
- name: indexls
|
||||||
|
action: index_ls
|
||||||
|
|
||||||
|
- name: describe
|
||||||
|
action: describe_index
|
||||||
|
|
||||||
|
- name: nodes
|
||||||
|
action: nodes
|
||||||
|
|
||||||
|
- name: check_node
|
||||||
|
action: describe_node
|
||||||
12
t/msc-docker.yaml
Normal file
12
t/msc-docker.yaml
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
#
|
||||||
|
# this is a mosscap item file, see: https://codeberg.org/scip/mosscap
|
||||||
|
items:
|
||||||
|
- name: docker/domdoc
|
||||||
|
|
||||||
|
vars:
|
||||||
|
index: test
|
||||||
|
time: 5s
|
||||||
|
|
||||||
|
tasks:
|
||||||
|
- name: wait
|
||||||
|
action: docker_wait
|
||||||
21
t/msc-docs.yaml
Normal file
21
t/msc-docs.yaml
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
#
|
||||||
|
# this is a mosscap generator item file, see: https://codeberg.org/scip/mosscap
|
||||||
|
vars:
|
||||||
|
index: test
|
||||||
|
|
||||||
|
tasks:
|
||||||
|
- name: ts
|
||||||
|
action: timestamp
|
||||||
|
|
||||||
|
- name: add
|
||||||
|
action: doc_add
|
||||||
|
args:
|
||||||
|
ts: ts.result
|
||||||
|
|
||||||
|
generate:
|
||||||
|
user: $name
|
||||||
|
# will become something like:
|
||||||
|
# "content34 iBAXQ2Va Ko5gMwwu FEH2O99B"
|
||||||
|
# so we can search for "content3*" and get multiple matches
|
||||||
|
message: "content$int8 $rand make$int8 $rand"
|
||||||
|
|
||||||
31
t/msc-search.yaml
Normal file
31
t/msc-search.yaml
Normal file
@@ -0,0 +1,31 @@
|
|||||||
|
#
|
||||||
|
# this is a mosscap item file, see: https://codeberg.org/scip/mosscap
|
||||||
|
items:
|
||||||
|
- name: docker/domdoc
|
||||||
|
|
||||||
|
vars:
|
||||||
|
index: test
|
||||||
|
time: 10s
|
||||||
|
content_pattern: "content3*"
|
||||||
|
content_and_pattern: "content3* make"
|
||||||
|
|
||||||
|
tasks:
|
||||||
|
- name: today
|
||||||
|
action: date
|
||||||
|
|
||||||
|
- name: search_any
|
||||||
|
action: search_any
|
||||||
|
|
||||||
|
- name: date
|
||||||
|
action: search_date
|
||||||
|
args:
|
||||||
|
today: today.result
|
||||||
|
|
||||||
|
- name: count
|
||||||
|
action: search_count
|
||||||
|
|
||||||
|
- name: content
|
||||||
|
action: search_content
|
||||||
|
|
||||||
|
- name: content_and
|
||||||
|
action: search_content_and
|
||||||
Reference in New Issue
Block a user