9 Commits

Author SHA1 Message Date
fe3951f3c2 more doc, fix format str 2024-12-22 13:28:08 +01:00
249c3f1cfb bump version 2024-12-22 13:28:08 +01:00
8687e084bf fixes and additions:
- add ANYDB_PASSWORD env var
- add config file support, including buckets dict
- finalized custom bucket support
- fine tuned info support
2024-12-22 13:28:08 +01:00
Thomas von Dein
24240b85f2 fix error checking 2024-12-22 13:28:08 +01:00
Thomas von Dein
8e400c6831 added db info command 2024-12-22 13:28:08 +01:00
Thomas von Dein
3de65aa1c3 add custom bucket support 2024-12-22 13:28:08 +01:00
be79886e89 refactor Import() 2024-12-22 11:29:12 +01:00
T.v.Dein
dc328afa44 Update TODO.md 2024-12-21 09:55:52 +01:00
T.v.Dein
cfa739ac83 Update TODO.md 2024-12-21 07:57:35 +01:00
14 changed files with 412 additions and 54 deletions

View File

@@ -66,11 +66,11 @@ clean:
rm -rf $(tool) releases coverage.out
test:
go test -v ./...
ANYDB_PASSWORD=test go test -v ./...
singletest:
@echo "Call like this: ''make singletest TEST=TestPrepareColumns MOD=lib"
go test -run $(TEST) github.com/tlinden/anydb/$(MOD)
ANYDB_PASSWORD=test go test -run $(TEST) github.com/tlinden/anydb/$(MOD)
cover-report:
go test ./... -cover -coverprofile=coverage.out

View File

@@ -95,9 +95,29 @@ anydb import -r backup.json
# you can encrypt entries. anydb asks for a passphrase
# and will do the same when you retrieve the key using the
# get command.
# get command. anydb will ask you interactively for a password
anydb set mypassword -e
# but you can provide it via an environment variable too
ANYDB_PASSWORD=foo anydb set -e secretkey blahblah
# too tiresome to add -e every time you add an entry?
# use a per bucket config
cat ~/.config/anydb/anydb.toml
[buckets.data]
encrypt = true
anydb set foo bar # will be encrypted
# speaking of buckets, you can use different buckets
anydb -b test set foo bar
# and speaking of configs, you can place a config file at these places:
# ~/.config/anydb/anydb.toml
# ~/.anydb.toml
# anydb.toml (current directory)
# or specify one using -c <filename>
# look at example.toml
# using template output mode you can freely design how to print stuff
# here, we print the values in CSV format ONLY if they have some tag
anydb ls -m template -T "{{ if .Tags }}{{ .Key }},{{ .Value }},{{ .Created}}{{ end }}"
@@ -125,6 +145,10 @@ curl localhost:8787/anydb/v1/foo
# list keys
curl localhost:8787/anydb/v1/
# sometimes you need to know some details about the current database
# add -d for more details
anydb info
# it comes with a manpage builtin
anydb man
```

View File

@@ -1,4 +1,3 @@
- repl
- mime-type => exec app + value
- custom buckets (like skate: key@bucket or key+bucket)
- encryption per bucket, one key for all entries (in that bucket)
- [edit command](https://github.com/TLINDEN/rpnc/blob/master/command.go#L249)

View File

@@ -1,4 +1,4 @@
.\" Automatically generated by Pod::Man 4.14 (Pod::Simple 3.40)
.\" Automatically generated by Pod::Man 4.14 (Pod::Simple 3.42)
.\"
.\" Standard preamble:
.\" ========================================================================
@@ -133,7 +133,7 @@
.\" ========================================================================
.\"
.IX Title "ANYDB 1"
.TH ANYDB 1 "2024-12-18" "1" "User Commands"
.TH ANYDB 1 "2024-12-22" "1" "User Commands"
.\" For nroff, turn off justification. Always turn off hyphenation; it makes
.\" way too many mistakes in technical documents.
.if n .ad l

View File

@@ -34,6 +34,7 @@ const MaxValueWidth int = 60
type DB struct {
Debug bool
Dbfile string
Bucket string
DB *bolt.DB
}
@@ -48,6 +49,19 @@ type DbEntry struct {
Size int
}
type BucketInfo struct {
Name string
Keys int
Size int
Sequence uint64
Stats bolt.BucketStats
}
type DbInfo struct {
Buckets []BucketInfo
Path string
}
// Post process an entry for list output.
// Do NOT call it during write processing!
func (entry *DbEntry) Normalize() {
@@ -75,8 +89,8 @@ type DbTag struct {
const BucketData string = "data"
func New(file string, debug bool) (*DB, error) {
return &DB{Debug: debug, Dbfile: file}, nil
func New(file string, bucket string, debug bool) (*DB, error) {
return &DB{Debug: debug, Dbfile: file, Bucket: bucket}, nil
}
func (db *DB) Open() error {
@@ -114,7 +128,7 @@ func (db *DB) List(attr *DbAttr) (DbEntries, error) {
err := db.DB.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(BucketData))
bucket := tx.Bucket([]byte(db.Bucket))
if bucket == nil {
return nil
}
@@ -182,7 +196,7 @@ func (db *DB) Set(attr *DbAttr) error {
// any tags. if so, we initialize our update struct with these
// tags unless it has new tags configured.
err := db.DB.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(BucketData))
bucket := tx.Bucket([]byte(db.Bucket))
if bucket == nil {
return nil
}
@@ -211,7 +225,7 @@ func (db *DB) Set(attr *DbAttr) error {
err = db.DB.Update(func(tx *bolt.Tx) error {
// insert data
bucket, err := tx.CreateBucketIfNotExists([]byte(BucketData))
bucket, err := tx.CreateBucketIfNotExists([]byte(db.Bucket))
if err != nil {
return fmt.Errorf("failed to create DB bucket: %w", err)
}
@@ -245,13 +259,14 @@ func (db *DB) Get(attr *DbAttr) (*DbEntry, error) {
entry := DbEntry{}
err := db.DB.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(BucketData))
bucket := tx.Bucket([]byte(db.Bucket))
if bucket == nil {
return nil
}
jsonentry := bucket.Get([]byte(attr.Key))
if jsonentry == nil {
// FIXME: shall we return a key not found error?
return nil
}
@@ -277,7 +292,7 @@ func (db *DB) Del(attr *DbAttr) error {
defer db.Close()
err := db.DB.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(BucketData))
bucket := tx.Bucket([]byte(db.Bucket))
if bucket == nil {
return nil
@@ -289,14 +304,14 @@ func (db *DB) Del(attr *DbAttr) error {
return err
}
func (db *DB) Import(attr *DbAttr) error {
func (db *DB) Import(attr *DbAttr) (string, error) {
// open json file into attr.Val
if err := attr.GetFileValue(); err != nil {
return err
return "", err
}
if attr.Val == "" {
return errors.New("empty json file")
return "", errors.New("empty json file")
}
var entries DbEntries
@@ -304,27 +319,27 @@ func (db *DB) Import(attr *DbAttr) error {
newfile := db.Dbfile + now.Format("-02.01.2006T03:04.05")
if err := json.Unmarshal([]byte(attr.Val), &entries); err != nil {
return cleanError(newfile, fmt.Errorf("failed to unmarshal json: %w", err))
return "", cleanError(newfile, fmt.Errorf("failed to unmarshal json: %w", err))
}
if fileExists(db.Dbfile) {
// backup the old file
err := os.Rename(db.Dbfile, newfile)
if err != nil {
return fmt.Errorf("failed to rename file %s to %s: %w", db.Dbfile, newfile, err)
return "", fmt.Errorf("failed to rename file %s to %s: %w", db.Dbfile, newfile, err)
}
}
// should now be a new db file
if err := db.Open(); err != nil {
return cleanError(newfile, err)
return "", cleanError(newfile, err)
}
defer db.Close()
err := db.DB.Update(func(tx *bolt.Tx) error {
// insert data
bucket, err := tx.CreateBucketIfNotExists([]byte(BucketData))
bucket, err := tx.CreateBucketIfNotExists([]byte(db.Bucket))
if err != nil {
return fmt.Errorf("failed to create bucket: %w", err)
}
@@ -345,28 +360,53 @@ func (db *DB) Import(attr *DbAttr) error {
})
if err != nil {
return cleanError(newfile, err)
return "", cleanError(newfile, err)
}
fmt.Printf("backed up database file to %s\n", newfile)
fmt.Printf("imported %d database entries\n", len(entries))
return fmt.Sprintf("backed up database file to %s\nimported %d database entries\n",
newfile, len(entries)), nil
}
func (db *DB) Info() (*DbInfo, error) {
if err := db.Open(); err != nil {
return nil, err
}
defer db.Close()
info := &DbInfo{Path: db.Dbfile}
err := db.DB.View(func(tx *bolt.Tx) error {
err := tx.ForEach(func(name []byte, bucket *bolt.Bucket) error {
stats := bucket.Stats()
binfo := BucketInfo{
Name: string(name),
Sequence: bucket.Sequence(),
Keys: stats.KeyN,
Stats: bucket.Stats(),
}
err := bucket.ForEach(func(key, entry []byte) error {
binfo.Size += len(entry) + len(key)
return nil
})
if err != nil {
return fmt.Errorf("failed to read keys: %w", err)
}
func cleanError(file string, err error) error {
// remove given [backup] file and forward the given error
os.Remove(file)
return err
}
info.Buckets = append(info.Buckets, binfo)
return nil
func fileExists(filename string) bool {
info, err := os.Stat(filename)
})
if err != nil {
// return false on any error
return false
return fmt.Errorf("failed to read from DB: %w", err)
}
return !info.IsDir()
return nil
})
return info, err
}

36
app/io.go Normal file
View File

@@ -0,0 +1,36 @@
/*
Copyright © 2024 Thomas von Dein
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package app
import "os"
func cleanError(file string, err error) error {
// remove given [backup] file and forward the given error
os.Remove(file)
return err
}
func fileExists(filename string) bool {
info, err := os.Stat(filename)
if err != nil {
// return false on any error
return false
}
return !info.IsDir()
}

View File

@@ -1,19 +1,114 @@
/*
Copyright © 2024 Thomas von Dein
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package cfg
import "github.com/tlinden/anydb/app"
import (
"fmt"
"io"
"os"
var Version string = "v0.0.4"
"github.com/pelletier/go-toml"
"github.com/tlinden/anydb/app"
"github.com/tlinden/anydb/common"
)
var Version string = "v0.0.5"
type BucketConfig struct {
Encrypt bool
}
type Config struct {
Debug bool
Dbfile string
Dbbucket string
Template string
Mode string // wide, table, yaml, json
NoHeaders bool
NoHumanize bool
Encrypt bool
DB *app.DB
File string
Tags []string
Encrypt bool // one entry
Listen string
Buckets map[string]BucketConfig // config file only
Tags []string // internal
DB *app.DB // internal
File string // internal
}
func (conf *Config) GetConfig(files []string) error {
for _, file := range files {
if err := conf.ParseConfigFile(file); err != nil {
return err
}
}
return nil
}
func (conf *Config) ParseConfigFile(file string) error {
if !common.FileExists(file) {
return nil
}
fd, err := os.OpenFile(file, os.O_RDONLY, 0600)
if err != nil {
return fmt.Errorf("failed to open config file %s: %w", file, err)
}
data, err := io.ReadAll(fd)
if err != nil {
return fmt.Errorf("failed to read from config file: %w", err)
}
add := Config{}
err = toml.Unmarshal(data, &add)
if err != nil {
return fmt.Errorf("failed to unmarshall toml: %w", err)
}
// merge new values into existing config
switch {
case add.Debug != conf.Debug:
conf.Debug = add.Debug
case add.Dbfile != "":
conf.Dbfile = add.Dbfile
case add.Dbbucket != "":
conf.Dbbucket = add.Dbbucket
case add.Template != "":
conf.Template = add.Template
case add.NoHeaders != conf.NoHeaders:
conf.NoHeaders = add.NoHeaders
case add.NoHumanize != conf.NoHumanize:
conf.NoHumanize = add.NoHumanize
case add.Encrypt != conf.Encrypt:
conf.Encrypt = add.Encrypt
case add.Listen != "":
conf.Listen = add.Listen
}
// only supported in config files
conf.Buckets = add.Buckets
// determine bucket encryption mode
for name, bucket := range conf.Buckets {
if name == conf.Dbbucket {
conf.Encrypt = bucket.Encrypt
}
}
return nil
}

View File

@@ -65,7 +65,7 @@ func Set(conf *cfg.Config) *cobra.Command {
// encrypt?
if conf.Encrypt {
pass, err := app.AskForPassword()
pass, err := getPassword()
if err != nil {
return err
}
@@ -118,7 +118,7 @@ func Get(conf *cfg.Config) *cobra.Command {
}
if entry.Encrypted {
pass, err := app.AskForPassword()
pass, err := getPassword()
if err != nil {
return err
}
@@ -279,7 +279,13 @@ func Import(conf *cfg.Config) *cobra.Command {
// errors at this stage do not cause the usage to be shown
cmd.SilenceUsage = true
return conf.DB.Import(&attr)
out, err := conf.DB.Import(&attr)
if err != nil {
return err
}
fmt.Print(out)
return nil
},
}
@@ -346,3 +352,45 @@ func Serve(conf *cfg.Config) *cobra.Command {
return cmd
}
func Info(conf *cfg.Config) *cobra.Command {
var cmd = &cobra.Command{
Use: "info",
Short: "info",
Long: `show info about database`,
RunE: func(cmd *cobra.Command, args []string) error {
// errors at this stage do not cause the usage to be shown
cmd.SilenceUsage = true
info, err := conf.DB.Info()
if err != nil {
return err
}
return output.Info(os.Stdout, conf, info)
},
}
cmd.PersistentFlags().BoolVarP(&conf.NoHumanize, "no-human", "N", false, "do not translate to human readable values")
return cmd
}
func getPassword() ([]byte, error) {
var pass []byte
envpass := os.Getenv("ANYDB_PASSWORD")
if envpass == "" {
readpass, err := app.AskForPassword()
if err != nil {
return nil, err
}
pass = readpass
} else {
pass = []byte(envpass)
}
return pass, nil
}

View File

@@ -22,6 +22,7 @@ import (
"os"
"path/filepath"
"github.com/alecthomas/repr"
"github.com/spf13/cobra"
"github.com/tlinden/anydb/app"
"github.com/tlinden/anydb/cfg"
@@ -45,22 +46,49 @@ func completion(cmd *cobra.Command, mode string) error {
func Execute() {
var (
conf cfg.Config
configfile string
ShowVersion bool
ShowCompletion string
)
home, err := os.UserHomeDir()
if err != nil {
panic(err)
}
SearchConfigs := []string{
filepath.Join(home, ".config", "anydb", "anydb.toml"),
filepath.Join(home, ".anydb.toml"),
"anydb.toml",
}
var rootCmd = &cobra.Command{
Use: "anydb <command> [options]",
Short: "anydb",
Long: `A personal key value store`,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
db, err := app.New(conf.Dbfile, conf.Debug)
db, err := app.New(conf.Dbfile, conf.Dbbucket, conf.Debug)
if err != nil {
return err
}
conf.DB = db
var configs []string
if configfile != "" {
configs = []string{configfile}
} else {
configs = SearchConfigs
}
if err := conf.GetConfig(configs); err != nil {
return err
}
if conf.Debug {
repr.Println(conf)
}
return nil
},
@@ -82,16 +110,14 @@ func Execute() {
},
}
home, err := os.UserHomeDir()
if err != nil {
panic(err)
}
// options
rootCmd.PersistentFlags().BoolVarP(&ShowVersion, "version", "v", false, "Print program version")
rootCmd.PersistentFlags().BoolVarP(&conf.Debug, "debug", "d", false, "Enable debugging")
rootCmd.PersistentFlags().StringVarP(&conf.Dbfile, "dbfile", "f",
filepath.Join(home, ".config", "anydb", "default.db"), "DB file to use")
rootCmd.PersistentFlags().StringVarP(&conf.Dbbucket, "bucket", "b",
app.BucketData, "use other bucket (default: "+app.BucketData+")")
rootCmd.PersistentFlags().StringVarP(&configfile, "config", "c", "", "toml config file")
rootCmd.AddCommand(Set(&conf))
rootCmd.AddCommand(List(&conf))
@@ -101,6 +127,7 @@ func Execute() {
rootCmd.AddCommand(Import(&conf))
rootCmd.AddCommand(Serve(&conf))
rootCmd.AddCommand(Man(&conf))
rootCmd.AddCommand(Info(&conf))
err = rootCmd.Execute()
if err != nil {

36
common/io.go Normal file
View File

@@ -0,0 +1,36 @@
/*
Copyright © 2024 Thomas von Dein
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package common
import "os"
func CleanError(file string, err error) error {
// remove given [backup] file and forward the given error
os.Remove(file)
return err
}
func FileExists(filename string) bool {
info, err := os.Stat(filename)
if err != nil {
// return false on any error
return false
}
return !info.IsDir()
}

14
example.toml Normal file
View File

@@ -0,0 +1,14 @@
# defaults
dbfile = "~/.config/anydb/default.db"
dbbucket = "data"
noheaders = false
nohumanize = false
encrypt = false
listen = "localhost:8787"
# different setups for different buckets
[buckets.data]
encrypt = true
[buckets.test]
encrypt = false

1
go.mod
View File

@@ -16,6 +16,7 @@ require (
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/rogpeppe/go-internal v1.13.1 // indirect
github.com/spf13/cobra v1.8.1 // indirect

2
go.sum
View File

@@ -45,6 +45,8 @@ github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZ
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=

View File

@@ -21,11 +21,14 @@ import (
"fmt"
"io"
"os"
"reflect"
"strings"
"github.com/dustin/go-humanize"
"github.com/tlinden/anydb/app"
"github.com/tlinden/anydb/cfg"
"golang.org/x/term"
//"github.com/alecthomas/repr"
)
func Print(writer io.Writer, conf *cfg.Config, attr *app.DbAttr, entry *app.DbEntry) error {
@@ -93,3 +96,36 @@ func WriteFile(writer io.Writer, conf *cfg.Config, attr *app.DbAttr, entry *app.
return nil
}
func Info(writer io.Writer, conf *cfg.Config, info *app.DbInfo) error {
fmt.Fprintf(writer, "Database: %s\n", info.Path)
for _, bucket := range info.Buckets {
if conf.NoHumanize {
fmt.Fprintf(
writer,
"%19s: %s\n%19s: %d\n%19s: %d\n%19s: %t\n",
"Bucket", bucket.Name,
"Size", bucket.Size,
"Keys", bucket.Keys,
"Encrypted", conf.Encrypt)
} else {
fmt.Fprintf(
writer,
"%19s: %s\n%19s: %s\n%19s: %d\n",
"Bucket", bucket.Name,
"Size", humanize.Bytes(uint64(bucket.Size)),
"Keys", bucket.Keys)
}
if conf.Debug {
val := reflect.ValueOf(&bucket.Stats).Elem()
for i := 0; i < val.NumField(); i++ {
fmt.Fprintf(writer, "%19s: %v\n", val.Type().Field(i).Name, val.Field(i))
}
}
fmt.Fprintln(writer)
}
return nil
}