From c144e99b41ba3f777e05869cbfa90173f24997f5 Mon Sep 17 00:00:00 2001 From: Thomas von Dein Date: Sun, 29 Dec 2024 13:01:16 +0100 Subject: [PATCH 1/5] switch to use protobuf for internal data structure in DB --- Makefile | 7 +- TODO.md | 32 +++++++- anydb.1 | 4 +- app/db.go | 53 ++++++------- app/dbentry.pb.go | 198 ++++++++++++++++++++++++++++++++++++++++++++++ app/dbentry.proto | 17 ++++ go.mod | 1 + go.sum | 2 + output/list.go | 6 +- 9 files changed, 282 insertions(+), 38 deletions(-) create mode 100644 app/dbentry.pb.go create mode 100644 app/dbentry.proto diff --git a/Makefile b/Makefile index 4a5b667..5380520 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ BUILD = $(shell date +%Y.%m.%d.%H%M%S) VERSION := $(if $(filter $(BRANCH), development),$(version)-$(BRANCH)-$(COMMIT)-$(BUILD),$(version)) HAVE_POD := $(shell pod2text -h 2>/dev/null) -all: $(tool).1 cmd/$(tool).go buildlocal +all: $(tool).1 cmd/$(tool).go app/dbentry.pb.go buildlocal %.1: %.pod ifdef HAVE_POD @@ -49,6 +49,11 @@ endif # awk '/SYNOPS/{f=1;next} /DESCR/{f=0} f' $*.pod | sed 's/^ //' >> cmd/$*.go # echo "\`" >> cmd/$*.go +app/dbentry.pb.go: app/dbentry.proto + protoc -I=. --go_out=app app/dbentry.proto + mv app/github.com/tlinden/anydb/app/dbentry.pb.go app/dbentry.pb.go + rm -rf app/github.com + buildlocal: go build -ldflags "-X 'github.com/tlinden/anydb/cfg.VERSION=$(VERSION)'" diff --git a/TODO.md b/TODO.md index c2df08b..476114a 100644 --- a/TODO.md +++ b/TODO.md @@ -1,5 +1,35 @@ +## Features + - repl - mime-type => exec app + value - add waitgroup to db.go funcs - RestList does not support any params? -- lc() incoming tags +- lc() incoming tags+keys + +## DB Structure + +- put tags into sub bucket see #1 +- change structure to: + +data bucket +key => {key,value[0:60],isbin:bool} + +value bucket +key => value (maybe always use []byte here) + +tags bucket +key/tag => tag/key +tag/key => tag + +So, list just uses the data bucket, no large contents. +A tag search only looksup matching tags, see #1. +Only a full text search and get would need to dig into the value bucket. + +A delete would just delete all keys from all values and then: +lookup in tags bucket for all key/*, then iterate over the values and +remove all tag/key's. Then deleting a key would not leave any residue +behind. + +However, maybe change the list command to just list everything and add +an extra find command for fulltext or tag search. Maybe still provide +filter options in list command but only filter for keys. diff --git a/anydb.1 b/anydb.1 index 4a7e1a3..7483fed 100644 --- a/anydb.1 +++ b/anydb.1 @@ -1,4 +1,4 @@ -.\" Automatically generated by Pod::Man 4.14 (Pod::Simple 3.40) +.\" Automatically generated by Pod::Man 4.14 (Pod::Simple 3.42) .\" .\" Standard preamble: .\" ======================================================================== @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "ANYDB 1" -.TH ANYDB 1 "2024-12-25" "1" "User Commands" +.TH ANYDB 1 "2024-12-29" "1" "User Commands" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff --git a/app/db.go b/app/db.go index 3bb60b9..ad75367 100644 --- a/app/db.go +++ b/app/db.go @@ -27,6 +27,8 @@ import ( "time" bolt "go.etcd.io/bbolt" + "google.golang.org/protobuf/proto" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" ) const MaxValueWidth int = 60 @@ -38,17 +40,6 @@ type DB struct { DB *bolt.DB } -type DbEntry struct { - Id string `json:"id"` - Key string `json:"key"` - Value string `json:"value"` - Encrypted bool `json:"encrypted"` - Bin []byte `json:"bin"` - Tags []string `json:"tags"` - Created time.Time `json:"created"` - Size int -} - type BucketInfo struct { Name string Keys int @@ -65,7 +56,7 @@ type DbInfo struct { // Post process an entry for list output. // Do NOT call it during write processing! func (entry *DbEntry) Normalize() { - entry.Size = len(entry.Value) + entry.Size = uint64(len(entry.Value)) if entry.Encrypted { entry.Value = "" @@ -73,7 +64,7 @@ func (entry *DbEntry) Normalize() { if len(entry.Bin) > 0 { entry.Value = "" - entry.Size = len(entry.Bin) + entry.Size = uint64(len(entry.Bin)) } if strings.Contains(entry.Value, "\n") { @@ -140,10 +131,10 @@ func (db *DB) List(attr *DbAttr) (DbEntries, error) { return nil } - err := bucket.ForEach(func(key, jsonentry []byte) error { + err := bucket.ForEach(func(key, pbentry []byte) error { var entry DbEntry - if err := json.Unmarshal(jsonentry, &entry); err != nil { - return fmt.Errorf("failed to unmarshal from json: %w", err) + if err := proto.Unmarshal(pbentry, &entry); err != nil { + return fmt.Errorf("failed to unmarshal from protobuf: %w", err) } var include bool @@ -196,7 +187,7 @@ func (db *DB) Set(attr *DbAttr) error { Bin: attr.Bin, Tags: attr.Tags, Encrypted: attr.Encrypted, - Created: time.Now(), + Created: timestamppb.Now(), } // check if the entry already exists and if yes, check if it has @@ -208,14 +199,14 @@ func (db *DB) Set(attr *DbAttr) error { return nil } - jsonentry := bucket.Get([]byte(entry.Key)) - if jsonentry == nil { + pbentry := bucket.Get([]byte(entry.Key)) + if pbentry == nil { return nil } var oldentry DbEntry - if err := json.Unmarshal(jsonentry, &oldentry); err != nil { - return fmt.Errorf("failed to unmarshal from json: %w", err) + if err := proto.Unmarshal(pbentry, &oldentry); err != nil { + return fmt.Errorf("failed to unmarshal from protobuf: %w", err) } if len(oldentry.Tags) > 0 && len(entry.Tags) == 0 { @@ -237,12 +228,12 @@ func (db *DB) Set(attr *DbAttr) error { return fmt.Errorf("failed to create DB bucket: %w", err) } - jsonentry, err := json.Marshal(entry) + pbentry, err := proto.Marshal(&entry) if err != nil { - return fmt.Errorf("failed to marshall json: %w", err) + return fmt.Errorf("failed to marshall protobuf: %w", err) } - err = bucket.Put([]byte(entry.Key), []byte(jsonentry)) + err = bucket.Put([]byte(entry.Key), []byte(pbentry)) if err != nil { return fmt.Errorf("failed to insert data: %w", err) } @@ -271,14 +262,14 @@ func (db *DB) Get(attr *DbAttr) (*DbEntry, error) { return nil } - jsonentry := bucket.Get([]byte(attr.Key)) - if jsonentry == nil { + pbentry := bucket.Get([]byte(attr.Key)) + if pbentry == nil { // FIXME: shall we return a key not found error? return nil } - if err := json.Unmarshal(jsonentry, &entry); err != nil { - return fmt.Errorf("failed to unmarshal from json: %w", err) + if err := proto.Unmarshal(pbentry, &entry); err != nil { + return fmt.Errorf("failed to unmarshal from protobuf: %w", err) } return nil @@ -352,12 +343,12 @@ func (db *DB) Import(attr *DbAttr) (string, error) { } for _, entry := range entries { - jsonentry, err := json.Marshal(entry) + pbentry, err := proto.Marshal(&entry) if err != nil { - return fmt.Errorf("failed to marshall json: %w", err) + return fmt.Errorf("failed to marshall protobuf: %w", err) } - err = bucket.Put([]byte(entry.Key), []byte(jsonentry)) + err = bucket.Put([]byte(entry.Key), []byte(pbentry)) if err != nil { return fmt.Errorf("failed to insert data into DB: %w", err) } diff --git a/app/dbentry.pb.go b/app/dbentry.pb.go new file mode 100644 index 0000000..06f7469 --- /dev/null +++ b/app/dbentry.pb.go @@ -0,0 +1,198 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.1 +// protoc v3.21.12 +// source: app/dbentry.proto + +package app + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type DbEntry struct { + state protoimpl.MessageState `protogen:"open.v1"` + Id string `protobuf:"bytes,1,opt,name=Id,proto3" json:"Id,omitempty"` + Key string `protobuf:"bytes,2,opt,name=Key,proto3" json:"Key,omitempty"` + Value string `protobuf:"bytes,3,opt,name=Value,proto3" json:"Value,omitempty"` + Bin []byte `protobuf:"bytes,4,opt,name=Bin,proto3" json:"Bin,omitempty"` + Tags []string `protobuf:"bytes,5,rep,name=Tags,proto3" json:"Tags,omitempty"` + Created *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=Created,proto3" json:"Created,omitempty"` + Size uint64 `protobuf:"varint,7,opt,name=Size,proto3" json:"Size,omitempty"` + Encrypted bool `protobuf:"varint,8,opt,name=Encrypted,proto3" json:"Encrypted,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DbEntry) Reset() { + *x = DbEntry{} + mi := &file_app_dbentry_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DbEntry) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DbEntry) ProtoMessage() {} + +func (x *DbEntry) ProtoReflect() protoreflect.Message { + mi := &file_app_dbentry_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DbEntry.ProtoReflect.Descriptor instead. +func (*DbEntry) Descriptor() ([]byte, []int) { + return file_app_dbentry_proto_rawDescGZIP(), []int{0} +} + +func (x *DbEntry) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *DbEntry) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *DbEntry) GetValue() string { + if x != nil { + return x.Value + } + return "" +} + +func (x *DbEntry) GetBin() []byte { + if x != nil { + return x.Bin + } + return nil +} + +func (x *DbEntry) GetTags() []string { + if x != nil { + return x.Tags + } + return nil +} + +func (x *DbEntry) GetCreated() *timestamppb.Timestamp { + if x != nil { + return x.Created + } + return nil +} + +func (x *DbEntry) GetSize() uint64 { + if x != nil { + return x.Size + } + return 0 +} + +func (x *DbEntry) GetEncrypted() bool { + if x != nil { + return x.Encrypted + } + return false +} + +var File_app_dbentry_proto protoreflect.FileDescriptor + +var file_app_dbentry_proto_rawDesc = []byte{ + 0x0a, 0x11, 0x61, 0x70, 0x70, 0x2f, 0x64, 0x62, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x12, 0x03, 0x61, 0x70, 0x70, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xcf, 0x01, 0x0a, 0x07, 0x44, 0x62, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x02, 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x4b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x10, 0x0a, + 0x03, 0x42, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x42, 0x69, 0x6e, 0x12, + 0x12, 0x0a, 0x04, 0x54, 0x61, 0x67, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x54, + 0x61, 0x67, 0x73, 0x12, 0x34, 0x0a, 0x07, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x18, 0x06, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x52, 0x07, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x53, 0x69, 0x7a, + 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1c, 0x0a, + 0x09, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x09, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x42, 0x1e, 0x5a, 0x1c, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x6c, 0x69, 0x6e, 0x64, 0x65, + 0x6e, 0x2f, 0x61, 0x6e, 0x79, 0x64, 0x62, 0x2f, 0x61, 0x70, 0x70, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, +} + +var ( + file_app_dbentry_proto_rawDescOnce sync.Once + file_app_dbentry_proto_rawDescData = file_app_dbentry_proto_rawDesc +) + +func file_app_dbentry_proto_rawDescGZIP() []byte { + file_app_dbentry_proto_rawDescOnce.Do(func() { + file_app_dbentry_proto_rawDescData = protoimpl.X.CompressGZIP(file_app_dbentry_proto_rawDescData) + }) + return file_app_dbentry_proto_rawDescData +} + +var file_app_dbentry_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_app_dbentry_proto_goTypes = []any{ + (*DbEntry)(nil), // 0: app.DbEntry + (*timestamppb.Timestamp)(nil), // 1: google.protobuf.Timestamp +} +var file_app_dbentry_proto_depIdxs = []int32{ + 1, // 0: app.DbEntry.Created:type_name -> google.protobuf.Timestamp + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_app_dbentry_proto_init() } +func file_app_dbentry_proto_init() { + if File_app_dbentry_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_app_dbentry_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_app_dbentry_proto_goTypes, + DependencyIndexes: file_app_dbentry_proto_depIdxs, + MessageInfos: file_app_dbentry_proto_msgTypes, + }.Build() + File_app_dbentry_proto = out.File + file_app_dbentry_proto_rawDesc = nil + file_app_dbentry_proto_goTypes = nil + file_app_dbentry_proto_depIdxs = nil +} diff --git a/app/dbentry.proto b/app/dbentry.proto new file mode 100644 index 0000000..8904334 --- /dev/null +++ b/app/dbentry.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; +package app; + +import "google/protobuf/timestamp.proto"; + +option go_package = "github.com/tlinden/anydb/app"; + +message DbEntry { + string Id = 1; + string Key = 2; + string Value = 3; + bytes Bin = 4; + repeated string Tags = 5; + google.protobuf.Timestamp Created = 6; + uint64 Size = 7; + bool Encrypted = 8; +} diff --git a/go.mod b/go.mod index 676bba0..4e4d98a 100644 --- a/go.mod +++ b/go.mod @@ -29,4 +29,5 @@ require ( golang.org/x/sys v0.28.0 // indirect golang.org/x/term v0.27.0 // indirect golang.org/x/tools v0.22.0 // indirect + google.golang.org/protobuf v1.36.1 // indirect ) diff --git a/go.sum b/go.sum index 0b195c4..29263e6 100644 --- a/go.sum +++ b/go.sum @@ -89,6 +89,8 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= +google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/output/list.go b/output/list.go index eeaadf4..1d29efe 100644 --- a/output/list.go +++ b/output/list.go @@ -102,8 +102,8 @@ func ListTable(writer io.Writer, conf *cfg.Config, entries app.DbEntries) error table.Append([]string{ row.Key, strings.Join(row.Tags, ","), - strconv.Itoa(row.Size), - row.Created.Format("02.01.2006T03:04.05"), + strconv.FormatUint(row.Size, 10), + row.Created.AsTime().Format("02.01.2006T03:04.05"), row.Value, }) default: @@ -112,7 +112,7 @@ func ListTable(writer io.Writer, conf *cfg.Config, entries app.DbEntries) error strings.Join(row.Tags, ","), humanize.Bytes(uint64(row.Size)), //row.Created.Format("02.01.2006T03:04.05"), - humanize.Time(row.Created), + humanize.Time(row.Created.AsTime()), row.Value, }) } From a4b6a3cfdf9e890cfc4772552cbe5d5a020414e5 Mon Sep 17 00:00:00 2001 From: Thomas von Dein Date: Sun, 29 Dec 2024 18:29:43 +0100 Subject: [PATCH 2/5] restructured data storage, values now have their own sub bucket --- TODO.md | 2 + app/attr.go | 43 ++++++++-- app/crypto.go | 33 +++----- app/db.go | 204 ++++++++++++++++++++++++++++++++++++---------- app/dbentry.pb.go | 66 +++++++++------ app/dbentry.proto | 15 ++-- cfg/config.go | 2 +- cmd/crud.go | 65 +++++++++++++-- cmd/extra.go | 14 +--- cmd/root.go | 1 + output/list.go | 11 +-- output/single.go | 25 +++--- 12 files changed, 330 insertions(+), 151 deletions(-) diff --git a/TODO.md b/TODO.md index 476114a..f13e0ca 100644 --- a/TODO.md +++ b/TODO.md @@ -33,3 +33,5 @@ behind. However, maybe change the list command to just list everything and add an extra find command for fulltext or tag search. Maybe still provide filter options in list command but only filter for keys. + +DONE: most of the above, except the tag stuff. manpage needs update and tests. diff --git a/app/attr.go b/app/attr.go index 7f09087..ead3293 100644 --- a/app/attr.go +++ b/app/attr.go @@ -20,17 +20,19 @@ import ( "fmt" "io" "os" + "strings" "unicode/utf8" ) type DbAttr struct { Key string - Val string - Bin []byte + Preview string + Val []byte Args []string Tags []string File string Encrypted bool + Binary bool } func (attr *DbAttr) ParseKV() error { @@ -43,7 +45,7 @@ func (attr *DbAttr) ParseKV() error { } case 2: attr.Key = attr.Args[0] - attr.Val = attr.Args[1] + attr.Val = []byte(attr.Args[1]) if attr.Args[1] == "-" { attr.File = "-" @@ -51,7 +53,29 @@ func (attr *DbAttr) ParseKV() error { } if attr.File != "" { - return attr.GetFileValue() + if err := attr.GetFileValue(); err != nil { + return err + } + } + + if attr.Binary { + attr.Preview = "" + } else { + if len(attr.Val) > MaxValueWidth { + attr.Preview = string(attr.Val)[0:MaxValueWidth] + "..." + + if strings.Contains(attr.Preview, "\n") { + parts := strings.Split(attr.Preview, "\n") + if len(parts) > 0 { + attr.Preview = parts[0] + } + } + } else { + attr.Preview = string(attr.Val) + } + } + if attr.Encrypted { + attr.Preview = "" } return nil @@ -82,11 +106,12 @@ func (attr *DbAttr) GetFileValue() error { } // poor man's text file test - sdata := string(data) - if utf8.ValidString(sdata) { - attr.Val = sdata + attr.Val = data + + if utf8.ValidString(string(data)) { + attr.Binary = false } else { - attr.Bin = data + attr.Binary = true } } else { // read from console stdin @@ -101,7 +126,7 @@ func (attr *DbAttr) GetFileValue() error { data += input + "\n" } - attr.Val = data + attr.Val = []byte(data) } return nil diff --git a/app/crypto.go b/app/crypto.go index c9ee9d9..4c65ea9 100644 --- a/app/crypto.go +++ b/app/crypto.go @@ -18,7 +18,6 @@ package app import ( "crypto/rand" - "encoding/base64" "errors" "fmt" "os" @@ -104,7 +103,7 @@ func GetRandom(size int, capacity int) ([]byte, error) { // modifying it. // // The cipher text consists of: -// base64(password-salt) + base64(12 byte nonce + ciphertext + 16 byte mac) +// password-salt) + (12 byte nonce + ciphertext + 16 byte mac) func Encrypt(pass []byte, attr *DbAttr) error { key, err := DeriveKey(pass, nil) if err != nil { @@ -116,25 +115,17 @@ func Encrypt(pass []byte, attr *DbAttr) error { return fmt.Errorf("failed to create AEAD cipher: %w", err) } - var plain []byte - if attr.Val != "" { - plain = []byte(attr.Val) - } else { - plain = attr.Bin - } - - total := aead.NonceSize() + len(plain) + aead.Overhead() + total := aead.NonceSize() + len(attr.Val) + aead.Overhead() nonce, err := GetRandom(aead.NonceSize(), total) if err != nil { return err } - cipher := aead.Seal(nonce, nonce, plain, nil) + cipher := aead.Seal(nonce, nonce, attr.Val, nil) - attr.Bin = nil - attr.Val = base64.RawStdEncoding.EncodeToString(key.Salt) + - base64.RawStdEncoding.EncodeToString(cipher) + attr.Val = append(attr.Val, key.Salt...) + attr.Val = append(attr.Val, cipher...) attr.Encrypted = true @@ -142,21 +133,17 @@ func Encrypt(pass []byte, attr *DbAttr) error { } // Do the reverse -func Decrypt(pass []byte, cipherb64 string) ([]byte, error) { - salt, err := base64.RawStdEncoding.Strict().DecodeString(cipherb64[0:B64SaltLen]) - if err != nil { - return nil, fmt.Errorf("failed to encode to base64: %w", err) +func Decrypt(pass []byte, cipherb []byte) ([]byte, error) { + if len(cipherb) < B64SaltLen { + return nil, fmt.Errorf("encrypted cipher block too small") } - key, err := DeriveKey(pass, salt) + key, err := DeriveKey(pass, cipherb[0:B64SaltLen]) if err != nil { return nil, err } - cipher, err := base64.RawStdEncoding.Strict().DecodeString(cipherb64[B64SaltLen:]) - if err != nil { - return nil, fmt.Errorf("failed to encode to base64: %w", err) - } + cipher := cipherb[B64SaltLen:] aead, err := chacha20poly1305.New(key.Key) if err != nil { diff --git a/app/db.go b/app/db.go index ad75367..e18d766 100644 --- a/app/db.go +++ b/app/db.go @@ -53,32 +53,6 @@ type DbInfo struct { Path string } -// Post process an entry for list output. -// Do NOT call it during write processing! -func (entry *DbEntry) Normalize() { - entry.Size = uint64(len(entry.Value)) - - if entry.Encrypted { - entry.Value = "" - } - - if len(entry.Bin) > 0 { - entry.Value = "" - entry.Size = uint64(len(entry.Bin)) - } - - if strings.Contains(entry.Value, "\n") { - parts := strings.Split(entry.Value, "\n") - if len(parts) > 0 { - entry.Value = parts[0] - } - } - - if len(entry.Value) > MaxValueWidth { - entry.Value = entry.Value[0:MaxValueWidth] + "..." - } -} - type DbEntries []DbEntry type DbTag struct { @@ -126,7 +100,12 @@ func (db *DB) List(attr *DbAttr) (DbEntries, error) { err := db.DB.View(func(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte(db.Bucket)) + root := tx.Bucket([]byte(db.Bucket)) + if root == nil { + return nil + } + + bucket := root.Bucket([]byte("meta")) if bucket == nil { return nil } @@ -141,8 +120,7 @@ func (db *DB) List(attr *DbAttr) (DbEntries, error) { switch { case filter != nil: - if filter.MatchString(entry.Value) || - filter.MatchString(entry.Key) || + if filter.MatchString(entry.Key) || filter.MatchString(strings.Join(entry.Tags, " ")) { include = true } @@ -183,18 +161,24 @@ func (db *DB) Set(attr *DbAttr) error { entry := DbEntry{ Key: attr.Key, - Value: attr.Val, - Bin: attr.Bin, + Binary: attr.Binary, Tags: attr.Tags, Encrypted: attr.Encrypted, Created: timestamppb.Now(), + Size: uint64(len(attr.Val)), + Preview: attr.Preview, } // check if the entry already exists and if yes, check if it has // any tags. if so, we initialize our update struct with these // tags unless it has new tags configured. err := db.DB.View(func(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte(db.Bucket)) + root := tx.Bucket([]byte(db.Bucket)) + if root == nil { + return nil + } + + bucket := root.Bucket([]byte("meta")) if bucket == nil { return nil } @@ -221,23 +205,43 @@ func (db *DB) Set(attr *DbAttr) error { return err } + // marshall our data + pbentry, err := proto.Marshal(&entry) + if err != nil { + return fmt.Errorf("failed to marshall protobuf: %w", err) + } + err = db.DB.Update(func(tx *bolt.Tx) error { - // insert data - bucket, err := tx.CreateBucketIfNotExists([]byte(db.Bucket)) + // create root bucket + root, err := tx.CreateBucketIfNotExists([]byte(db.Bucket)) if err != nil { return fmt.Errorf("failed to create DB bucket: %w", err) } - pbentry, err := proto.Marshal(&entry) + // create meta bucket + bucket, err := root.CreateBucketIfNotExists([]byte("meta")) if err != nil { - return fmt.Errorf("failed to marshall protobuf: %w", err) + return fmt.Errorf("failed to create DB meta sub bucket: %w", err) } + // write meta data err = bucket.Put([]byte(entry.Key), []byte(pbentry)) if err != nil { return fmt.Errorf("failed to insert data: %w", err) } + // create data bucket + databucket, err := root.CreateBucketIfNotExists([]byte("data")) + if err != nil { + return fmt.Errorf("failed to create DB data sub bucket: %w", err) + } + + // write value + err = databucket.Put([]byte(entry.Key), attr.Val) + if err != nil { + return fmt.Errorf("failed to insert data: %w", err) + } + return nil }) @@ -257,21 +261,35 @@ func (db *DB) Get(attr *DbAttr) (*DbEntry, error) { entry := DbEntry{} err := db.DB.View(func(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte(db.Bucket)) + root := tx.Bucket([]byte(db.Bucket)) + if root == nil { + return nil + } + + bucket := root.Bucket([]byte("meta")) if bucket == nil { return nil } pbentry := bucket.Get([]byte(attr.Key)) if pbentry == nil { - // FIXME: shall we return a key not found error? - return nil + return fmt.Errorf("no such key: %s", attr.Key) } if err := proto.Unmarshal(pbentry, &entry); err != nil { return fmt.Errorf("failed to unmarshal from protobuf: %w", err) } + databucket := root.Bucket([]byte("data")) + if databucket == nil { + return fmt.Errorf("failed to retrieve data sub bucket") + } + + entry.Value = databucket.Get([]byte(attr.Key)) + if len(entry.Value) == 0 { + return fmt.Errorf("no such key: %s", attr.Key) + } + return nil }) @@ -308,7 +326,7 @@ func (db *DB) Import(attr *DbAttr) (string, error) { return "", err } - if attr.Val == "" { + if len(attr.Val) == 0 { return "", errors.New("empty json file") } @@ -336,10 +354,16 @@ func (db *DB) Import(attr *DbAttr) (string, error) { defer db.Close() err := db.DB.Update(func(tx *bolt.Tx) error { - // insert data - bucket, err := tx.CreateBucketIfNotExists([]byte(db.Bucket)) + // create root bucket + root, err := tx.CreateBucketIfNotExists([]byte(db.Bucket)) if err != nil { - return fmt.Errorf("failed to create bucket: %w", err) + return fmt.Errorf("failed to create DB bucket: %w", err) + } + + // create meta bucket + bucket, err := root.CreateBucketIfNotExists([]byte("meta")) + if err != nil { + return fmt.Errorf("failed to create DB meta sub bucket: %w", err) } for _, entry := range entries { @@ -348,10 +372,23 @@ func (db *DB) Import(attr *DbAttr) (string, error) { return fmt.Errorf("failed to marshall protobuf: %w", err) } + // write meta data err = bucket.Put([]byte(entry.Key), []byte(pbentry)) if err != nil { return fmt.Errorf("failed to insert data into DB: %w", err) } + + // create data bucket + databucket, err := root.CreateBucketIfNotExists([]byte("data")) + if err != nil { + return fmt.Errorf("failed to create DB data sub bucket: %w", err) + } + + // write value + err = databucket.Put([]byte(entry.Key), entry.Value) + if err != nil { + return fmt.Errorf("failed to insert data: %w", err) + } } return nil @@ -408,3 +445,84 @@ func (db *DB) Info() (*DbInfo, error) { return info, err } + +func (db *DB) Find(attr *DbAttr) (DbEntries, error) { + if err := db.Open(); err != nil { + return nil, err + } + defer db.Close() + + var entries DbEntries + var filter *regexp.Regexp + + if len(attr.Args) > 0 { + filter = regexp.MustCompile(attr.Args[0]) + } + + err := db.DB.View(func(tx *bolt.Tx) error { + + root := tx.Bucket([]byte(db.Bucket)) + if root == nil { + return nil + } + + bucket := root.Bucket([]byte("meta")) + if bucket == nil { + return nil + } + + databucket := root.Bucket([]byte("data")) + if databucket == nil { + return fmt.Errorf("failed to retrieve data sub bucket") + } + + err := bucket.ForEach(func(key, pbentry []byte) error { + var entry DbEntry + if err := proto.Unmarshal(pbentry, &entry); err != nil { + return fmt.Errorf("failed to unmarshal from protobuf: %w", err) + } + + entry.Value = databucket.Get([]byte(entry.Key)) + + var include bool + + switch { + case filter != nil: + if filter.MatchString(entry.Key) || + filter.MatchString(strings.Join(entry.Tags, " ")) { + include = true + } + + if !entry.Binary && !include { + if filter.MatchString(string(entry.Value)) { + include = true + } + } + case len(attr.Tags) > 0: + for _, search := range attr.Tags { + for _, tag := range entry.Tags { + if tag == search { + include = true + break + } + } + + if include { + break + } + } + default: + include = true + } + + if include { + entries = append(entries, entry) + } + + return nil + }) + + return err + }) + return entries, err +} diff --git a/app/dbentry.pb.go b/app/dbentry.pb.go index 06f7469..61642fc 100644 --- a/app/dbentry.pb.go +++ b/app/dbentry.pb.go @@ -1,3 +1,5 @@ +// -*-c++-*- + // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.1 @@ -25,12 +27,13 @@ type DbEntry struct { state protoimpl.MessageState `protogen:"open.v1"` Id string `protobuf:"bytes,1,opt,name=Id,proto3" json:"Id,omitempty"` Key string `protobuf:"bytes,2,opt,name=Key,proto3" json:"Key,omitempty"` - Value string `protobuf:"bytes,3,opt,name=Value,proto3" json:"Value,omitempty"` - Bin []byte `protobuf:"bytes,4,opt,name=Bin,proto3" json:"Bin,omitempty"` - Tags []string `protobuf:"bytes,5,rep,name=Tags,proto3" json:"Tags,omitempty"` - Created *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=Created,proto3" json:"Created,omitempty"` - Size uint64 `protobuf:"varint,7,opt,name=Size,proto3" json:"Size,omitempty"` - Encrypted bool `protobuf:"varint,8,opt,name=Encrypted,proto3" json:"Encrypted,omitempty"` + Preview string `protobuf:"bytes,3,opt,name=Preview,proto3" json:"Preview,omitempty"` + Tags []string `protobuf:"bytes,4,rep,name=Tags,proto3" json:"Tags,omitempty"` + Created *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=Created,proto3" json:"Created,omitempty"` + Size uint64 `protobuf:"varint,6,opt,name=Size,proto3" json:"Size,omitempty"` + Encrypted bool `protobuf:"varint,7,opt,name=Encrypted,proto3" json:"Encrypted,omitempty"` + Binary bool `protobuf:"varint,8,opt,name=Binary,proto3" json:"Binary,omitempty"` + Value []byte `protobuf:"bytes,9,opt,name=Value,proto3" json:"Value,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -79,20 +82,13 @@ func (x *DbEntry) GetKey() string { return "" } -func (x *DbEntry) GetValue() string { +func (x *DbEntry) GetPreview() string { if x != nil { - return x.Value + return x.Preview } return "" } -func (x *DbEntry) GetBin() []byte { - if x != nil { - return x.Bin - } - return nil -} - func (x *DbEntry) GetTags() []string { if x != nil { return x.Tags @@ -121,26 +117,42 @@ func (x *DbEntry) GetEncrypted() bool { return false } +func (x *DbEntry) GetBinary() bool { + if x != nil { + return x.Binary + } + return false +} + +func (x *DbEntry) GetValue() []byte { + if x != nil { + return x.Value + } + return nil +} + var File_app_dbentry_proto protoreflect.FileDescriptor var file_app_dbentry_proto_rawDesc = []byte{ 0x0a, 0x11, 0x61, 0x70, 0x70, 0x2f, 0x64, 0x62, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x03, 0x61, 0x70, 0x70, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, - 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xcf, 0x01, 0x0a, 0x07, 0x44, 0x62, + 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xef, 0x01, 0x0a, 0x07, 0x44, 0x62, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x4b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x10, 0x0a, - 0x03, 0x42, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x42, 0x69, 0x6e, 0x12, - 0x12, 0x0a, 0x04, 0x54, 0x61, 0x67, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x54, - 0x61, 0x67, 0x73, 0x12, 0x34, 0x0a, 0x07, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x18, 0x06, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x52, 0x07, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x53, 0x69, 0x7a, - 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1c, 0x0a, - 0x09, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x09, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x42, 0x1e, 0x5a, 0x1c, 0x67, + 0x28, 0x09, 0x52, 0x03, 0x4b, 0x65, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x50, 0x72, 0x65, 0x76, 0x69, + 0x65, 0x77, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x50, 0x72, 0x65, 0x76, 0x69, 0x65, + 0x77, 0x12, 0x12, 0x0a, 0x04, 0x54, 0x61, 0x67, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x04, 0x54, 0x61, 0x67, 0x73, 0x12, 0x34, 0x0a, 0x07, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x52, 0x07, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x53, + 0x69, 0x7a, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x12, + 0x1c, 0x0a, 0x09, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x18, 0x07, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x09, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x12, 0x16, 0x0a, + 0x06, 0x42, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x42, + 0x69, 0x6e, 0x61, 0x72, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x09, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x1e, 0x5a, 0x1c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x6c, 0x69, 0x6e, 0x64, 0x65, 0x6e, 0x2f, 0x61, 0x6e, 0x79, 0x64, 0x62, 0x2f, 0x61, 0x70, 0x70, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, diff --git a/app/dbentry.proto b/app/dbentry.proto index 8904334..fa8d113 100644 --- a/app/dbentry.proto +++ b/app/dbentry.proto @@ -1,3 +1,5 @@ +// -*-c++-*- + syntax = "proto3"; package app; @@ -8,10 +10,11 @@ option go_package = "github.com/tlinden/anydb/app"; message DbEntry { string Id = 1; string Key = 2; - string Value = 3; - bytes Bin = 4; - repeated string Tags = 5; - google.protobuf.Timestamp Created = 6; - uint64 Size = 7; - bool Encrypted = 8; + string Preview = 3; + repeated string Tags = 4; + google.protobuf.Timestamp Created = 5; + uint64 Size = 6; + bool Encrypted = 7; + bool Binary = 8; + bytes Value = 9; } diff --git a/cfg/config.go b/cfg/config.go index 47ff633..e932ed6 100644 --- a/cfg/config.go +++ b/cfg/config.go @@ -26,7 +26,7 @@ import ( "github.com/tlinden/anydb/common" ) -var Version string = "v0.0.7" +var Version string = "v0.1.0" type BucketConfig struct { Encrypt bool diff --git a/cmd/crud.go b/cmd/crud.go index fb72b2e..d2b208a 100644 --- a/cmd/crud.go +++ b/cmd/crud.go @@ -20,7 +20,6 @@ import ( "errors" "os" "strings" - "unicode/utf8" "github.com/spf13/cobra" "github.com/tlinden/anydb/app" @@ -124,12 +123,7 @@ func Get(conf *cfg.Config) *cobra.Command { return err } - if utf8.ValidString(string(clear)) { - entry.Value = string(clear) - } else { - entry.Bin = clear - } - + entry.Value = clear entry.Encrypted = false } @@ -188,7 +182,7 @@ func List(conf *cfg.Config) *cobra.Command { ) var cmd = &cobra.Command{ - Use: "list [] [-t ] [-m ] [-n -N] [-T ] [-i]", + Use: "list [] [-m ] [-n -N] [-T ] [-i]", Short: "List database contents", Long: `List database contents`, RunE: func(cmd *cobra.Command, args []string) error { @@ -221,6 +215,58 @@ func List(conf *cfg.Config) *cobra.Command { }, } + cmd.PersistentFlags().StringVarP(&conf.Mode, "mode", "m", "", "output format (table|wide|json|template), wide is a verbose table. (default 'table')") + cmd.PersistentFlags().StringVarP(&conf.Template, "template", "T", "", "go template for '-m template'") + cmd.PersistentFlags().BoolVarP(&wide, "wide-output", "l", false, "output mode: wide") + cmd.PersistentFlags().BoolVarP(&conf.NoHeaders, "no-headers", "n", false, "omit headers in tables") + cmd.PersistentFlags().BoolVarP(&conf.NoHumanize, "no-human", "N", false, "do not translate to human readable values") + cmd.PersistentFlags().BoolVarP(&conf.CaseInsensitive, "case-insensitive", "i", false, "filter case insensitive") + + cmd.Aliases = append(cmd.Aliases, "ls") + + return cmd +} + +func Find(conf *cfg.Config) *cobra.Command { + var ( + attr app.DbAttr + wide bool + ) + + var cmd = &cobra.Command{ + Use: "find | -t [-m ] [-n -N] [-T ] [-i]", + Short: "Find database contents", + Long: `Find database contents`, + RunE: func(cmd *cobra.Command, args []string) error { + // errors at this stage do not cause the usage to be shown + cmd.SilenceUsage = true + + if len(args) > 0 { + if conf.CaseInsensitive { + attr.Args = []string{"(?i)" + args[0]} + } else { + attr.Args = args + } + } + + // turn comma list into slice, if needed + if len(attr.Tags) == 1 && strings.Contains(attr.Tags[0], ",") { + attr.Tags = strings.Split(attr.Tags[0], ",") + } + + if wide { + conf.Mode = "wide" + } + + entries, err := conf.DB.Find(&attr) + if err != nil { + return err + } + + return output.List(os.Stdout, conf, entries) + }, + } + cmd.PersistentFlags().StringVarP(&conf.Mode, "mode", "m", "", "output format (table|wide|json|template), wide is a verbose table. (default 'table')") cmd.PersistentFlags().StringVarP(&conf.Template, "template", "T", "", "go template for '-m template'") cmd.PersistentFlags().BoolVarP(&wide, "wide-output", "l", false, "output mode: wide") @@ -230,7 +276,8 @@ func List(conf *cfg.Config) *cobra.Command { cmd.PersistentFlags().StringArrayVarP(&attr.Tags, "tags", "t", nil, "tags, multiple allowed") cmd.Aliases = append(cmd.Aliases, "/") - cmd.Aliases = append(cmd.Aliases, "ls") + cmd.Aliases = append(cmd.Aliases, "f") + cmd.Aliases = append(cmd.Aliases, "search") return cmd } diff --git a/cmd/extra.go b/cmd/extra.go index 8c889e0..795f06f 100644 --- a/cmd/extra.go +++ b/cmd/extra.go @@ -23,7 +23,6 @@ import ( "io" "os" "os/exec" - "unicode/utf8" "github.com/spf13/cobra" "github.com/tlinden/anydb/app" @@ -199,7 +198,7 @@ func Edit(conf *cfg.Config) *cobra.Command { return err } - if len(entry.Value) == 0 && len(entry.Bin) > 0 { + if len(entry.Value) == 0 && entry.Binary { return errors.New("key contains binary uneditable content") } @@ -216,12 +215,7 @@ func Edit(conf *cfg.Config) *cobra.Command { return err } - if utf8.ValidString(string(clear)) { - entry.Value = string(clear) - } else { - entry.Bin = clear - } - + entry.Value = clear entry.Encrypted = false } @@ -231,7 +225,7 @@ func Edit(conf *cfg.Config) *cobra.Command { // save file to a temp file, call the editor with it, read // it back in and compare the content with the original // one - newcontent, err := editContent(editor, entry.Value) + newcontent, err := editContent(editor, string(entry.Value)) if err != nil { return err } @@ -241,7 +235,7 @@ func Edit(conf *cfg.Config) *cobra.Command { Key: attr.Key, Tags: attr.Tags, Encrypted: attr.Encrypted, - Val: newcontent, + Val: []byte(newcontent), } // encrypt if needed diff --git a/cmd/root.go b/cmd/root.go index c6d1f57..d448eb8 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -122,6 +122,7 @@ func Execute() { // CRUD rootCmd.AddCommand(Set(&conf)) rootCmd.AddCommand(List(&conf)) + rootCmd.AddCommand(Find(&conf)) rootCmd.AddCommand(Get(&conf)) rootCmd.AddCommand(Del(&conf)) diff --git a/output/list.go b/output/list.go index 1d29efe..cc4e479 100644 --- a/output/list.go +++ b/output/list.go @@ -65,8 +65,6 @@ func ListTemplate(writer io.Writer, conf *cfg.Config, entries app.DbEntries) err buf := bytes.Buffer{} for _, row := range entries { - row.Normalize() - buf.Reset() err = tmpl.Execute(&buf, row) if err != nil { @@ -94,8 +92,6 @@ func ListTable(writer io.Writer, conf *cfg.Config, entries app.DbEntries) error } for _, row := range entries { - row.Normalize() - if conf.Mode == "wide" { switch conf.NoHumanize { case true: @@ -104,21 +100,20 @@ func ListTable(writer io.Writer, conf *cfg.Config, entries app.DbEntries) error strings.Join(row.Tags, ","), strconv.FormatUint(row.Size, 10), row.Created.AsTime().Format("02.01.2006T03:04.05"), - row.Value, + row.Preview, }) default: table.Append([]string{ row.Key, strings.Join(row.Tags, ","), humanize.Bytes(uint64(row.Size)), - //row.Created.Format("02.01.2006T03:04.05"), humanize.Time(row.Created.AsTime()), - row.Value, + row.Preview, }) } } else { - table.Append([]string{row.Key, row.Value}) + table.Append([]string{row.Key, row.Preview}) } } diff --git a/output/single.go b/output/single.go index 7294f83..2e08c45 100644 --- a/output/single.go +++ b/output/single.go @@ -22,7 +22,6 @@ import ( "io" "os" "reflect" - "strings" "github.com/dustin/go-humanize" "github.com/tlinden/anydb/app" @@ -40,16 +39,15 @@ func Print(writer io.Writer, conf *cfg.Config, attr *app.DbAttr, entry *app.DbEn switch conf.Mode { case "simple", "": - if len(entry.Bin) > 0 { + if entry.Binary { if isatty { fmt.Println("binary data omitted") } else { - os.Stdout.Write(entry.Bin) + os.Stdout.Write(entry.Value) } } else { - fmt.Print(entry.Value) - - if !strings.HasSuffix(entry.Value, "\n") { + fmt.Print(string(entry.Value)) + if entry.Value[entry.Size-1] != '\n' { // always add a terminal newline fmt.Println() } @@ -87,17 +85,14 @@ func WriteFile(writer io.Writer, conf *cfg.Config, attr *app.DbAttr, entry *app. fileHandle = fd } - if len(entry.Bin) > 0 { + if entry.Binary { // binary file content - _, err = fileHandle.Write(entry.Bin) - } else { - val := entry.Value - if !strings.HasSuffix(val, "\n") { - // always add a terminal newline - val += "\n" - } + _, err = fileHandle.Write(entry.Value) - _, err = fileHandle.Write([]byte(val)) + if entry.Value[entry.Size-1] != '\n' { + // always add a terminal newline + _, err = fileHandle.Write([]byte{'\n'}) + } } if err != nil { From bb5c268ca81d53555ab50dacb3c06e3227f92f87 Mon Sep 17 00:00:00 2001 From: Thomas von Dein Date: Sun, 29 Dec 2024 23:29:23 +0100 Subject: [PATCH 3/5] todo --- TODO.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/TODO.md b/TODO.md index f13e0ca..fa278e2 100644 --- a/TODO.md +++ b/TODO.md @@ -35,3 +35,5 @@ an extra find command for fulltext or tag search. Maybe still provide filter options in list command but only filter for keys. DONE: most of the above, except the tag stuff. manpage needs update and tests. + +maybe stitch the find command and just add -f (full text search) to list. From 1eb5efae0c1961016516206adf9d16d50aafea38 Mon Sep 17 00:00:00 2001 From: Thomas von Dein Date: Mon, 30 Dec 2024 12:12:02 +0100 Subject: [PATCH 4/5] finalized conversion to protobuf: - fixed import+export - generalized file options - always store keys as lowercase - fixed+enhanced docs - fixed tests --- README.md | 10 ++++-- anydb.1 | 10 +++--- anydb.pod | 35 ++++++++++--------- app/attr.go | 16 +++++---- app/db.go | 87 +++++++++++++++++++++++------------------------- cfg/config.go | 1 + cmd/anydb.go | 8 ++--- cmd/crud.go | 60 +++------------------------------ cmd/extra.go | 16 +++++---- cmd/root.go | 1 - output/single.go | 7 ++-- rest/handlers.go | 2 +- t/files.txtar | 17 ++++++++++ t/restore.txtar | 4 +-- t/workflow.txtar | 6 ++-- 15 files changed, 128 insertions(+), 152 deletions(-) diff --git a/README.md b/README.md index edb336b..27afcbc 100644 --- a/README.md +++ b/README.md @@ -76,10 +76,14 @@ anydb set foo bar -t note,important anydb list -t important # beside tags filtering you can also use regexps for searching +# note, by default the list command only searches through keys anydb list '[a-z]+\d' +# do a full text search +anydb list '[a-z]+\d' -s + # anydb also supports a wide output -anydb list -o wide +anydb list -m wide KEY TAGS SIZE AGE VALUE blah important 4 B 7 seconds ago haha foo 3 B 15 seconds ago bar @@ -90,13 +94,13 @@ anydb ls -l anydb / # other outputs are possible as well -anydb list -o json +anydb list -m json # you can backup your database anydb export -o backup.json # and import it somewhere else -anydb import -r backup.json +anydb import -i backup.json # you can encrypt entries. anydb asks for a passphrase # and will do the same when you retrieve the key using the diff --git a/anydb.1 b/anydb.1 index 7483fed..bc9f696 100644 --- a/anydb.1 +++ b/anydb.1 @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "ANYDB 1" -.TH ANYDB 1 "2024-12-29" "1" "User Commands" +.TH ANYDB 1 "2024-12-30" "1" "User Commands" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l @@ -356,18 +356,18 @@ Usage: .PP .Vb 2 \& Usage: -\& anydb list [] [\-t ] [\-m ] [\-n \-N] [\-T ] [\-i] [flags] +\& anydb list [ | \-t ] [\-m ] [\-nNif] [\-T ] [flags] \& \& Aliases: -\& list, /, ls +\& list, ls, /, find, search \& \& Flags: \& \-i, \-\-case\-insensitive filter case insensitive \& \-h, \-\-help help for list -\& \-m, \-\-mode string output format (table|wide|json|template), -\& wide is a verbose table. (default \*(Aqtable\*(Aq) +\& \-m, \-\-mode string output format (table|wide|json|template), wide is a verbose table. (default \*(Aqtable\*(Aq) \& \-n, \-\-no\-headers omit headers in tables \& \-N, \-\-no\-human do not translate to human readable values +\& \-s, \-\-search\-fulltext perform a full text search \& \-t, \-\-tags stringArray tags, multiple allowed \& \-T, \-\-template string go template for \*(Aq\-m template\*(Aq \& \-l, \-\-wide\-output output mode: wide diff --git a/anydb.pod b/anydb.pod index 6c8d874..5368701 100644 --- a/anydb.pod +++ b/anydb.pod @@ -206,18 +206,18 @@ The B subcommand displays a list of all database entries. Usage: Usage: - anydb list [] [-t ] [-m ] [-n -N] [-T ] [-i] [flags] + anydb list [ | -t ] [-m ] [-nNif] [-T ] [flags] Aliases: - list, /, ls + list, ls, /, find, search Flags: -i, --case-insensitive filter case insensitive -h, --help help for list - -m, --mode string output format (table|wide|json|template), - wide is a verbose table. (default 'table') + -m, --mode string output format (table|wide|json|template), wide is a verbose table. (default 'table') -n, --no-headers omit headers in tables -N, --no-human do not translate to human readable values + -s, --search-fulltext perform a full text search -t, --tags stringArray tags, multiple allowed -T, --template string go template for '-m template' -l, --wide-output output mode: wide @@ -254,6 +254,10 @@ features. If you want to search case insensitive, add the option C<-i>. +By default anydb only searches through the keys. If you want to search +through the values as well, then use the C<-s> option, which enables +full-text search. + You can - as with the B command - use other output modes. The default mode is "table". The "wide" mode is, as already mentioned, a more detailed table. Also supported is "json" mode and "template" @@ -323,7 +327,7 @@ the B subcommand. Usage: Usage: - anydb export [-o ] [flags] + anydb export -o [flags] Aliases: export, dump, backup @@ -332,12 +336,12 @@ Usage: -h, --help help for export -o, --output string output to file -The database dump is a JSON representation of the whole database and -will be printed to STDOUT by default. Redirect it to a file or use the -C<-o> option: +The database dump is a JSON representation of the whole database and +will be printed to the file specified with the C<-o> option. If you +specify "-" as the filename, it will be written to STDIN. - anydb export > dump.json anydb export -o dump.json + anydb export -o - > dump.json Please note, that encrypted values will not be decrypted. This might change in a future version of anydb. @@ -350,7 +354,7 @@ dump. Usage: Usage: - anydb import [] [flags] + anydb import -i [flags] Aliases: import, restore @@ -360,12 +364,13 @@ Usage: -h, --help help for import -t, --tags stringArray tags, multiple allowed -By default the C subcommand reads the JSON contents from -STDIN. You might pipe the dump into it or use the option C<-r>: +The C subcommand reads the JSON contents from +the file specified with the C<-i> option. If you specify "-" as the +filename, it will be read from STDIN. - anydb import < dump.json - anydb import -r dump.json - cat dump.json | anydb import + anydb import -i - < dump.json + anydb import -i dump.json + cat dump.json | anydb import -i - If there is already a database, it will be saved by appending a timestamp and a new database with the contents of the dump will be diff --git a/app/attr.go b/app/attr.go index ead3293..27501ba 100644 --- a/app/attr.go +++ b/app/attr.go @@ -35,16 +35,18 @@ type DbAttr struct { Binary bool } +// check if value is to be read from a file or stdin, setup preview +// text according to flags, lowercase key func (attr *DbAttr) ParseKV() error { + attr.Key = strings.ToLower(attr.Args[0]) + switch len(attr.Args) { case 1: // 1 arg = key + read from file or stdin - attr.Key = attr.Args[0] if attr.File == "" { attr.File = "-" } case 2: - attr.Key = attr.Args[0] attr.Val = []byte(attr.Args[1]) if attr.Args[1] == "-" { @@ -58,9 +60,12 @@ func (attr *DbAttr) ParseKV() error { } } - if attr.Binary { + switch { + case attr.Binary: + attr.Preview = "" + case attr.Encrypted: attr.Preview = "" - } else { + default: if len(attr.Val) > MaxValueWidth { attr.Preview = string(attr.Val)[0:MaxValueWidth] + "..." @@ -74,9 +79,6 @@ func (attr *DbAttr) ParseKV() error { attr.Preview = string(attr.Val) } } - if attr.Encrypted { - attr.Preview = "" - } return nil } diff --git a/app/db.go b/app/db.go index e18d766..eafaa73 100644 --- a/app/db.go +++ b/app/db.go @@ -85,7 +85,7 @@ func (db *DB) Close() error { return db.DB.Close() } -func (db *DB) List(attr *DbAttr) (DbEntries, error) { +func (db *DB) List(attr *DbAttr, fulltext bool) (DbEntries, error) { if err := db.Open(); err != nil { return nil, err } @@ -99,7 +99,6 @@ func (db *DB) List(attr *DbAttr) (DbEntries, error) { } err := db.DB.View(func(tx *bolt.Tx) error { - root := tx.Bucket([]byte(db.Bucket)) if root == nil { return nil @@ -110,12 +109,19 @@ func (db *DB) List(attr *DbAttr) (DbEntries, error) { return nil } + databucket := root.Bucket([]byte("data")) + if databucket == nil { + return fmt.Errorf("failed to retrieve data sub bucket") + } + err := bucket.ForEach(func(key, pbentry []byte) error { var entry DbEntry if err := proto.Unmarshal(pbentry, &entry); err != nil { return fmt.Errorf("failed to unmarshal from protobuf: %w", err) } + entry.Value = databucket.Get([]byte(entry.Key)) // empty is ok + var include bool switch { @@ -124,6 +130,12 @@ func (db *DB) List(attr *DbAttr) (DbEntries, error) { filter.MatchString(strings.Join(entry.Tags, " ")) { include = true } + + if !entry.Binary && !include && fulltext { + if filter.MatchString(string(entry.Value)) { + include = true + } + } case len(attr.Tags) > 0: for _, search := range attr.Tags { for _, tag := range entry.Tags { @@ -261,35 +273,48 @@ func (db *DB) Get(attr *DbAttr) (*DbEntry, error) { entry := DbEntry{} err := db.DB.View(func(tx *bolt.Tx) error { + // root bucket root := tx.Bucket([]byte(db.Bucket)) if root == nil { return nil } + // get meta sub bucket bucket := root.Bucket([]byte("meta")) if bucket == nil { return nil } + // retrieve meta data pbentry := bucket.Get([]byte(attr.Key)) if pbentry == nil { return fmt.Errorf("no such key: %s", attr.Key) } + // put into struct if err := proto.Unmarshal(pbentry, &entry); err != nil { return fmt.Errorf("failed to unmarshal from protobuf: %w", err) } + // get data sub bucket databucket := root.Bucket([]byte("data")) if databucket == nil { return fmt.Errorf("failed to retrieve data sub bucket") } - entry.Value = databucket.Get([]byte(attr.Key)) - if len(entry.Value) == 0 { + // retrieve actual data value + value := databucket.Get([]byte(attr.Key)) + if len(value) == 0 { return fmt.Errorf("no such key: %s", attr.Key) } + // we need to make a copy of it, otherwise we'll get an + // "unexpected fault address" error + vc := make([]byte, len(value)) + copy(vc, value) + + entry.Value = vc + return nil }) @@ -446,78 +471,50 @@ func (db *DB) Info() (*DbInfo, error) { return info, err } -func (db *DB) Find(attr *DbAttr) (DbEntries, error) { +func (db *DB) Getall(attr *DbAttr) (DbEntries, error) { if err := db.Open(); err != nil { return nil, err } defer db.Close() var entries DbEntries - var filter *regexp.Regexp - - if len(attr.Args) > 0 { - filter = regexp.MustCompile(attr.Args[0]) - } err := db.DB.View(func(tx *bolt.Tx) error { - + // root bucket root := tx.Bucket([]byte(db.Bucket)) if root == nil { return nil } + // get meta sub bucket bucket := root.Bucket([]byte("meta")) if bucket == nil { return nil } + // get data sub bucket databucket := root.Bucket([]byte("data")) if databucket == nil { return fmt.Errorf("failed to retrieve data sub bucket") } + // iterate over all db entries in meta sub bucket err := bucket.ForEach(func(key, pbentry []byte) error { var entry DbEntry if err := proto.Unmarshal(pbentry, &entry); err != nil { return fmt.Errorf("failed to unmarshal from protobuf: %w", err) } - entry.Value = databucket.Get([]byte(entry.Key)) + // retrieve the value from the data sub bucket + value := databucket.Get([]byte(entry.Key)) - var include bool + // we need to make a copy of it, otherwise we'll get an + // "unexpected fault address" error + vc := make([]byte, len(value)) + copy(vc, value) - switch { - case filter != nil: - if filter.MatchString(entry.Key) || - filter.MatchString(strings.Join(entry.Tags, " ")) { - include = true - } - - if !entry.Binary && !include { - if filter.MatchString(string(entry.Value)) { - include = true - } - } - case len(attr.Tags) > 0: - for _, search := range attr.Tags { - for _, tag := range entry.Tags { - if tag == search { - include = true - break - } - } - - if include { - break - } - } - default: - include = true - } - - if include { - entries = append(entries, entry) - } + entry.Value = vc + entries = append(entries, entry) return nil }) diff --git a/cfg/config.go b/cfg/config.go index e932ed6..3297a31 100644 --- a/cfg/config.go +++ b/cfg/config.go @@ -42,6 +42,7 @@ type Config struct { NoHumanize bool Encrypt bool // one entry CaseInsensitive bool + Fulltext bool Listen string Buckets map[string]BucketConfig // config file only diff --git a/cmd/anydb.go b/cmd/anydb.go index d8c36da..07ae3bc 100644 --- a/cmd/anydb.go +++ b/cmd/anydb.go @@ -187,18 +187,18 @@ SUBCOMMANDS Usage: Usage: - anydb list [] [-t ] [-m ] [-n -N] [-T ] [-i] [flags] + anydb list [ | -t ] [-m ] [-nNif] [-T ] [flags] Aliases: - list, /, ls + list, ls, /, find, search Flags: -i, --case-insensitive filter case insensitive -h, --help help for list - -m, --mode string output format (table|wide|json|template), - wide is a verbose table. (default 'table') + -m, --mode string output format (table|wide|json|template), wide is a verbose table. (default 'table') -n, --no-headers omit headers in tables -N, --no-human do not translate to human readable values + -s, --search-fulltext perform a full text search -t, --tags stringArray tags, multiple allowed -T, --template string go template for '-m template' -l, --wide-output output mode: wide diff --git a/cmd/crud.go b/cmd/crud.go index d2b208a..00bdeab 100644 --- a/cmd/crud.go +++ b/cmd/crud.go @@ -182,7 +182,7 @@ func List(conf *cfg.Config) *cobra.Command { ) var cmd = &cobra.Command{ - Use: "list [] [-m ] [-n -N] [-T ] [-i]", + Use: "list [ | -t ] [-m ] [-nNif] [-T ]", Short: "List database contents", Long: `List database contents`, RunE: func(cmd *cobra.Command, args []string) error { @@ -206,59 +206,7 @@ func List(conf *cfg.Config) *cobra.Command { conf.Mode = "wide" } - entries, err := conf.DB.List(&attr) - if err != nil { - return err - } - - return output.List(os.Stdout, conf, entries) - }, - } - - cmd.PersistentFlags().StringVarP(&conf.Mode, "mode", "m", "", "output format (table|wide|json|template), wide is a verbose table. (default 'table')") - cmd.PersistentFlags().StringVarP(&conf.Template, "template", "T", "", "go template for '-m template'") - cmd.PersistentFlags().BoolVarP(&wide, "wide-output", "l", false, "output mode: wide") - cmd.PersistentFlags().BoolVarP(&conf.NoHeaders, "no-headers", "n", false, "omit headers in tables") - cmd.PersistentFlags().BoolVarP(&conf.NoHumanize, "no-human", "N", false, "do not translate to human readable values") - cmd.PersistentFlags().BoolVarP(&conf.CaseInsensitive, "case-insensitive", "i", false, "filter case insensitive") - - cmd.Aliases = append(cmd.Aliases, "ls") - - return cmd -} - -func Find(conf *cfg.Config) *cobra.Command { - var ( - attr app.DbAttr - wide bool - ) - - var cmd = &cobra.Command{ - Use: "find | -t [-m ] [-n -N] [-T ] [-i]", - Short: "Find database contents", - Long: `Find database contents`, - RunE: func(cmd *cobra.Command, args []string) error { - // errors at this stage do not cause the usage to be shown - cmd.SilenceUsage = true - - if len(args) > 0 { - if conf.CaseInsensitive { - attr.Args = []string{"(?i)" + args[0]} - } else { - attr.Args = args - } - } - - // turn comma list into slice, if needed - if len(attr.Tags) == 1 && strings.Contains(attr.Tags[0], ",") { - attr.Tags = strings.Split(attr.Tags[0], ",") - } - - if wide { - conf.Mode = "wide" - } - - entries, err := conf.DB.Find(&attr) + entries, err := conf.DB.List(&attr, conf.Fulltext) if err != nil { return err } @@ -273,10 +221,12 @@ func Find(conf *cfg.Config) *cobra.Command { cmd.PersistentFlags().BoolVarP(&conf.NoHeaders, "no-headers", "n", false, "omit headers in tables") cmd.PersistentFlags().BoolVarP(&conf.NoHumanize, "no-human", "N", false, "do not translate to human readable values") cmd.PersistentFlags().BoolVarP(&conf.CaseInsensitive, "case-insensitive", "i", false, "filter case insensitive") + cmd.PersistentFlags().BoolVarP(&conf.Fulltext, "search-fulltext", "s", false, "perform a full text search") cmd.PersistentFlags().StringArrayVarP(&attr.Tags, "tags", "t", nil, "tags, multiple allowed") + cmd.Aliases = append(cmd.Aliases, "ls") cmd.Aliases = append(cmd.Aliases, "/") - cmd.Aliases = append(cmd.Aliases, "f") + cmd.Aliases = append(cmd.Aliases, "find") cmd.Aliases = append(cmd.Aliases, "search") return cmd diff --git a/cmd/extra.go b/cmd/extra.go index 795f06f..a2c20f6 100644 --- a/cmd/extra.go +++ b/cmd/extra.go @@ -37,16 +37,16 @@ func Export(conf *cfg.Config) *cobra.Command { ) var cmd = &cobra.Command{ - Use: "export [-o ]", - Short: "Export database to json", - Long: `Export database to json`, + Use: "export -o ", + Short: "Export database to json file", + Long: `Export database to json file`, RunE: func(cmd *cobra.Command, args []string) error { // errors at this stage do not cause the usage to be shown cmd.SilenceUsage = true conf.Mode = "json" - entries, err := conf.DB.List(&attr) + entries, err := conf.DB.Getall(&attr) if err != nil { return err } @@ -55,7 +55,8 @@ func Export(conf *cfg.Config) *cobra.Command { }, } - cmd.PersistentFlags().StringVarP(&attr.File, "output", "o", "", "output to file") + cmd.PersistentFlags().StringVarP(&attr.File, "output-file", "o", "", "filename or - for STDIN") + cmd.MarkPersistentFlagRequired("output-file") cmd.Aliases = append(cmd.Aliases, "dump") cmd.Aliases = append(cmd.Aliases, "backup") @@ -69,7 +70,7 @@ func Import(conf *cfg.Config) *cobra.Command { ) var cmd = &cobra.Command{ - Use: "import []", + Use: "import -i ", Short: "Import database dump", Long: `Import database dump`, RunE: func(cmd *cobra.Command, args []string) error { @@ -86,7 +87,8 @@ func Import(conf *cfg.Config) *cobra.Command { }, } - cmd.PersistentFlags().StringVarP(&attr.File, "file", "r", "", "Filename or - for STDIN") + cmd.PersistentFlags().StringVarP(&attr.File, "import-file", "i", "", "filename or - for STDIN") + cmd.MarkPersistentFlagRequired("import-file") cmd.PersistentFlags().StringArrayVarP(&attr.Tags, "tags", "t", nil, "tags, multiple allowed") cmd.Aliases = append(cmd.Aliases, "restore") diff --git a/cmd/root.go b/cmd/root.go index d448eb8..c6d1f57 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -122,7 +122,6 @@ func Execute() { // CRUD rootCmd.AddCommand(Set(&conf)) rootCmd.AddCommand(List(&conf)) - rootCmd.AddCommand(Find(&conf)) rootCmd.AddCommand(Get(&conf)) rootCmd.AddCommand(Del(&conf)) diff --git a/output/single.go b/output/single.go index 2e08c45..7bbe74f 100644 --- a/output/single.go +++ b/output/single.go @@ -75,7 +75,6 @@ func WriteFile(writer io.Writer, conf *cfg.Config, attr *app.DbAttr, entry *app. if attr.File == "-" { fileHandle = os.Stdout } else { - fd, err := os.OpenFile(attr.File, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755) if err != nil { return fmt.Errorf("failed to open file %s for writing: %w", attr.File, err) @@ -85,10 +84,10 @@ func WriteFile(writer io.Writer, conf *cfg.Config, attr *app.DbAttr, entry *app. fileHandle = fd } - if entry.Binary { - // binary file content - _, err = fileHandle.Write(entry.Value) + // actually write file content + _, err = fileHandle.Write(entry.Value) + if !entry.Binary { if entry.Value[entry.Size-1] != '\n' { // always add a terminal newline _, err = fileHandle.Write([]byte{'\n'}) diff --git a/rest/handlers.go b/rest/handlers.go index 83f4cd2..c375545 100644 --- a/rest/handlers.go +++ b/rest/handlers.go @@ -54,7 +54,7 @@ func RestList(c *fiber.Ctx, conf *cfg.Config) error { } // get list - entries, err := conf.DB.List(attr) + entries, err := conf.DB.List(attr, false) if err != nil { return JsonStatus(c, fiber.StatusForbidden, "Unable to list keys: "+err.Error()) diff --git a/t/files.txtar b/t/files.txtar index 7ab329a..58bbaec 100644 --- a/t/files.txtar +++ b/t/files.txtar @@ -32,3 +32,20 @@ stdout 50 # look if it's inside the db exec anydb -f test.db ls stdout datum.*binary-content + +# do the same thing with text content, start with a new text entry +exec anydb -f test.db set feed alpha + +# which we write to a file +exec anydb -f test.db get feed -o out2.txt +exists out2.txt + +# check if its filled (5 bytes + newline) +exec ls -l out2.txt +stdout 6 + +# compare content +exec cat out2.txt +stdout alpha + + diff --git a/t/restore.txtar b/t/restore.txtar index f2cfe9c..e57f90e 100644 --- a/t/restore.txtar +++ b/t/restore.txtar @@ -23,10 +23,10 @@ exec anydb -f test.db export -o backup.json stdout 'database contents exported to backup.json' # import into new db -exec anydb -f new.db import -r backup.json +exec anydb -f new.db import -i backup.json stdout 'imported.*entries' # check contents -exec anydb -f new.db list +exec anydb -f new.db list bar -s stdout foo.*bar diff --git a/t/workflow.txtar b/t/workflow.txtar index cf10205..7caabd6 100644 --- a/t/workflow.txtar +++ b/t/workflow.txtar @@ -37,12 +37,12 @@ exec anydb -f test.db list -t flower ! stdout bar # list with filter -exec anydb -f test.db list b.r +exec anydb -f test.db list b.r -s stdout bar # list with -i filter -exec anydb -f test.db list -i mucha -stdout MUCHA +exec anydb -f test.db list -is mucha +stdout mucha # get single entry exec anydb -f test.db get color From 659e3472bb8d8cff79e32bbb57483fc4cca3e93e Mon Sep 17 00:00:00 2001 From: Thomas von Dein Date: Mon, 30 Dec 2024 12:23:13 +0100 Subject: [PATCH 5/5] fixed copylocks linter warning --- anydb.1 | 27 ++++++++++++++++----------- app/db.go | 9 +++++---- cmd/anydb.go | 25 +++++++++++++++---------- cmd/extra.go | 8 ++++++-- output/single.go | 4 ++-- 5 files changed, 44 insertions(+), 29 deletions(-) diff --git a/anydb.1 b/anydb.1 index bc9f696..367a284 100644 --- a/anydb.1 +++ b/anydb.1 @@ -409,6 +409,10 @@ features. .PP If you want to search case insensitive, add the option \f(CW\*(C`\-i\*(C'\fR. .PP +By default anydb only searches through the keys. If you want to search +through the values as well, then use the \f(CW\*(C`\-s\*(C'\fR option, which enables +full-text search. +.PP You can \- as with the \fBget\fR command \- use other output modes. The default mode is \*(L"table\*(R". The \*(L"wide\*(R" mode is, as already mentioned, a more detailed table. Also supported is \*(L"json\*(R" mode and \*(L"template\*(R" @@ -478,7 +482,7 @@ Usage: .PP .Vb 2 \& Usage: -\& anydb export [\-o ] [flags] +\& anydb export \-o [flags] \& \& Aliases: \& export, dump, backup @@ -488,13 +492,13 @@ Usage: \& \-o, \-\-output string output to file .Ve .PP -The database dump is a \s-1JSON\s0 representation of the whole database and -will be printed to \s-1STDOUT\s0 by default. Redirect it to a file or use the -\&\f(CW\*(C`\-o\*(C'\fR option: +The database dump is a \s-1JSON\s0 representation of the whole database and +will be printed to the file specified with the \f(CW\*(C`\-o\*(C'\fR option. If you +specify \*(L"\-\*(R" as the filename, it will be written to \s-1STDIN.\s0 .PP .Vb 2 -\& anydb export > dump.json \& anydb export \-o dump.json +\& anydb export \-o \- > dump.json .Ve .PP Please note, that encrypted values will not be decrypted. This might @@ -508,7 +512,7 @@ Usage: .PP .Vb 2 \& Usage: -\& anydb import [] [flags] +\& anydb import \-i [flags] \& \& Aliases: \& import, restore @@ -519,13 +523,14 @@ Usage: \& \-t, \-\-tags stringArray tags, multiple allowed .Ve .PP -By default the \f(CW\*(C`import\*(C'\fR subcommand reads the \s-1JSON\s0 contents from -\&\s-1STDIN.\s0 You might pipe the dump into it or use the option \f(CW\*(C`\-r\*(C'\fR: +The \f(CW\*(C`import\*(C'\fR subcommand reads the \s-1JSON\s0 contents from +the file specified with the \f(CW\*(C`\-i\*(C'\fR option. If you specify \*(L"\-\*(R" as the +filename, it will be read from \s-1STDIN.\s0 .PP .Vb 3 -\& anydb import < dump.json -\& anydb import \-r dump.json -\& cat dump.json | anydb import +\& anydb import \-i \- < dump.json +\& anydb import \-i dump.json +\& cat dump.json | anydb import \-i \- .Ve .PP If there is already a database, it will be saved by appending a diff --git a/app/db.go b/app/db.go index eafaa73..27b1850 100644 --- a/app/db.go +++ b/app/db.go @@ -53,7 +53,7 @@ type DbInfo struct { Path string } -type DbEntries []DbEntry +type DbEntries []*DbEntry type DbTag struct { Keys []string `json:"key"` @@ -154,7 +154,7 @@ func (db *DB) List(attr *DbAttr, fulltext bool) (DbEntries, error) { } if include { - entries = append(entries, entry) + entries = append(entries, &entry) } return nil @@ -162,6 +162,7 @@ func (db *DB) List(attr *DbAttr, fulltext bool) (DbEntries, error) { return err }) + return entries, err } @@ -392,7 +393,7 @@ func (db *DB) Import(attr *DbAttr) (string, error) { } for _, entry := range entries { - pbentry, err := proto.Marshal(&entry) + pbentry, err := proto.Marshal(entry) if err != nil { return fmt.Errorf("failed to marshall protobuf: %w", err) } @@ -514,7 +515,7 @@ func (db *DB) Getall(attr *DbAttr) (DbEntries, error) { copy(vc, value) entry.Value = vc - entries = append(entries, entry) + entries = append(entries, &entry) return nil }) diff --git a/cmd/anydb.go b/cmd/anydb.go index 07ae3bc..6a11cd0 100644 --- a/cmd/anydb.go +++ b/cmd/anydb.go @@ -234,6 +234,10 @@ SUBCOMMANDS If you want to search case insensitive, add the option "-i". + By default anydb only searches through the keys. If you want to search + through the values as well, then use the "-s" option, which enables + full-text search. + You can - as with the get command - use other output modes. The default mode is "table". The "wide" mode is, as already mentioned, a more detailed table. Also supported is "json" mode and "template" mode. For @@ -293,7 +297,7 @@ SUBCOMMANDS Usage: Usage: - anydb export [-o ] [flags] + anydb export -o [flags] Aliases: export, dump, backup @@ -303,11 +307,11 @@ SUBCOMMANDS -o, --output string output to file The database dump is a JSON representation of the whole database and - will be printed to STDOUT by default. Redirect it to a file or use the - "-o" option: + will be printed to the file specified with the "-o" option. If you + specify "-" as the filename, it will be written to STDIN. - anydb export > dump.json anydb export -o dump.json + anydb export -o - > dump.json Please note, that encrypted values will not be decrypted. This might change in a future version of anydb. @@ -319,7 +323,7 @@ SUBCOMMANDS Usage: Usage: - anydb import [] [flags] + anydb import -i [flags] Aliases: import, restore @@ -329,12 +333,13 @@ SUBCOMMANDS -h, --help help for import -t, --tags stringArray tags, multiple allowed - By default the "import" subcommand reads the JSON contents from STDIN. - You might pipe the dump into it or use the option "-r": + The "import" subcommand reads the JSON contents from the file specified + with the "-i" option. If you specify "-" as the filename, it will be + read from STDIN. - anydb import < dump.json - anydb import -r dump.json - cat dump.json | anydb import + anydb import -i - < dump.json + anydb import -i dump.json + cat dump.json | anydb import -i - If there is already a database, it will be saved by appending a timestamp and a new database with the contents of the dump will be diff --git a/cmd/extra.go b/cmd/extra.go index a2c20f6..d5629ef 100644 --- a/cmd/extra.go +++ b/cmd/extra.go @@ -56,7 +56,9 @@ func Export(conf *cfg.Config) *cobra.Command { } cmd.PersistentFlags().StringVarP(&attr.File, "output-file", "o", "", "filename or - for STDIN") - cmd.MarkPersistentFlagRequired("output-file") + if err := cmd.MarkPersistentFlagRequired("output-file"); err != nil { + panic(err) + } cmd.Aliases = append(cmd.Aliases, "dump") cmd.Aliases = append(cmd.Aliases, "backup") @@ -88,8 +90,10 @@ func Import(conf *cfg.Config) *cobra.Command { } cmd.PersistentFlags().StringVarP(&attr.File, "import-file", "i", "", "filename or - for STDIN") - cmd.MarkPersistentFlagRequired("import-file") cmd.PersistentFlags().StringArrayVarP(&attr.Tags, "tags", "t", nil, "tags, multiple allowed") + if err := cmd.MarkPersistentFlagRequired("import-file"); err != nil { + panic(err) + } cmd.Aliases = append(cmd.Aliases, "restore") diff --git a/output/single.go b/output/single.go index 7bbe74f..d0dd036 100644 --- a/output/single.go +++ b/output/single.go @@ -60,9 +60,9 @@ func Print(writer io.Writer, conf *cfg.Config, attr *app.DbAttr, entry *app.DbEn fmt.Println(string(jsonentry)) case "wide": - return ListTable(writer, conf, app.DbEntries{*entry}) + return ListTable(writer, conf, app.DbEntries{entry}) case "template": - return ListTemplate(writer, conf, app.DbEntries{*entry}) + return ListTemplate(writer, conf, app.DbEntries{entry}) } return nil