From c144e99b41ba3f777e05869cbfa90173f24997f5 Mon Sep 17 00:00:00 2001 From: Thomas von Dein Date: Sun, 29 Dec 2024 13:01:16 +0100 Subject: [PATCH] switch to use protobuf for internal data structure in DB --- Makefile | 7 +- TODO.md | 32 +++++++- anydb.1 | 4 +- app/db.go | 53 ++++++------- app/dbentry.pb.go | 198 ++++++++++++++++++++++++++++++++++++++++++++++ app/dbentry.proto | 17 ++++ go.mod | 1 + go.sum | 2 + output/list.go | 6 +- 9 files changed, 282 insertions(+), 38 deletions(-) create mode 100644 app/dbentry.pb.go create mode 100644 app/dbentry.proto diff --git a/Makefile b/Makefile index 4a5b667..5380520 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ BUILD = $(shell date +%Y.%m.%d.%H%M%S) VERSION := $(if $(filter $(BRANCH), development),$(version)-$(BRANCH)-$(COMMIT)-$(BUILD),$(version)) HAVE_POD := $(shell pod2text -h 2>/dev/null) -all: $(tool).1 cmd/$(tool).go buildlocal +all: $(tool).1 cmd/$(tool).go app/dbentry.pb.go buildlocal %.1: %.pod ifdef HAVE_POD @@ -49,6 +49,11 @@ endif # awk '/SYNOPS/{f=1;next} /DESCR/{f=0} f' $*.pod | sed 's/^ //' >> cmd/$*.go # echo "\`" >> cmd/$*.go +app/dbentry.pb.go: app/dbentry.proto + protoc -I=. --go_out=app app/dbentry.proto + mv app/github.com/tlinden/anydb/app/dbentry.pb.go app/dbentry.pb.go + rm -rf app/github.com + buildlocal: go build -ldflags "-X 'github.com/tlinden/anydb/cfg.VERSION=$(VERSION)'" diff --git a/TODO.md b/TODO.md index c2df08b..476114a 100644 --- a/TODO.md +++ b/TODO.md @@ -1,5 +1,35 @@ +## Features + - repl - mime-type => exec app + value - add waitgroup to db.go funcs - RestList does not support any params? -- lc() incoming tags +- lc() incoming tags+keys + +## DB Structure + +- put tags into sub bucket see #1 +- change structure to: + +data bucket +key => {key,value[0:60],isbin:bool} + +value bucket +key => value (maybe always use []byte here) + +tags bucket +key/tag => tag/key +tag/key => tag + +So, list just uses the data bucket, no large contents. +A tag search only looksup matching tags, see #1. +Only a full text search and get would need to dig into the value bucket. + +A delete would just delete all keys from all values and then: +lookup in tags bucket for all key/*, then iterate over the values and +remove all tag/key's. Then deleting a key would not leave any residue +behind. + +However, maybe change the list command to just list everything and add +an extra find command for fulltext or tag search. Maybe still provide +filter options in list command but only filter for keys. diff --git a/anydb.1 b/anydb.1 index 4a7e1a3..7483fed 100644 --- a/anydb.1 +++ b/anydb.1 @@ -1,4 +1,4 @@ -.\" Automatically generated by Pod::Man 4.14 (Pod::Simple 3.40) +.\" Automatically generated by Pod::Man 4.14 (Pod::Simple 3.42) .\" .\" Standard preamble: .\" ======================================================================== @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "ANYDB 1" -.TH ANYDB 1 "2024-12-25" "1" "User Commands" +.TH ANYDB 1 "2024-12-29" "1" "User Commands" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff --git a/app/db.go b/app/db.go index 3bb60b9..ad75367 100644 --- a/app/db.go +++ b/app/db.go @@ -27,6 +27,8 @@ import ( "time" bolt "go.etcd.io/bbolt" + "google.golang.org/protobuf/proto" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" ) const MaxValueWidth int = 60 @@ -38,17 +40,6 @@ type DB struct { DB *bolt.DB } -type DbEntry struct { - Id string `json:"id"` - Key string `json:"key"` - Value string `json:"value"` - Encrypted bool `json:"encrypted"` - Bin []byte `json:"bin"` - Tags []string `json:"tags"` - Created time.Time `json:"created"` - Size int -} - type BucketInfo struct { Name string Keys int @@ -65,7 +56,7 @@ type DbInfo struct { // Post process an entry for list output. // Do NOT call it during write processing! func (entry *DbEntry) Normalize() { - entry.Size = len(entry.Value) + entry.Size = uint64(len(entry.Value)) if entry.Encrypted { entry.Value = "" @@ -73,7 +64,7 @@ func (entry *DbEntry) Normalize() { if len(entry.Bin) > 0 { entry.Value = "" - entry.Size = len(entry.Bin) + entry.Size = uint64(len(entry.Bin)) } if strings.Contains(entry.Value, "\n") { @@ -140,10 +131,10 @@ func (db *DB) List(attr *DbAttr) (DbEntries, error) { return nil } - err := bucket.ForEach(func(key, jsonentry []byte) error { + err := bucket.ForEach(func(key, pbentry []byte) error { var entry DbEntry - if err := json.Unmarshal(jsonentry, &entry); err != nil { - return fmt.Errorf("failed to unmarshal from json: %w", err) + if err := proto.Unmarshal(pbentry, &entry); err != nil { + return fmt.Errorf("failed to unmarshal from protobuf: %w", err) } var include bool @@ -196,7 +187,7 @@ func (db *DB) Set(attr *DbAttr) error { Bin: attr.Bin, Tags: attr.Tags, Encrypted: attr.Encrypted, - Created: time.Now(), + Created: timestamppb.Now(), } // check if the entry already exists and if yes, check if it has @@ -208,14 +199,14 @@ func (db *DB) Set(attr *DbAttr) error { return nil } - jsonentry := bucket.Get([]byte(entry.Key)) - if jsonentry == nil { + pbentry := bucket.Get([]byte(entry.Key)) + if pbentry == nil { return nil } var oldentry DbEntry - if err := json.Unmarshal(jsonentry, &oldentry); err != nil { - return fmt.Errorf("failed to unmarshal from json: %w", err) + if err := proto.Unmarshal(pbentry, &oldentry); err != nil { + return fmt.Errorf("failed to unmarshal from protobuf: %w", err) } if len(oldentry.Tags) > 0 && len(entry.Tags) == 0 { @@ -237,12 +228,12 @@ func (db *DB) Set(attr *DbAttr) error { return fmt.Errorf("failed to create DB bucket: %w", err) } - jsonentry, err := json.Marshal(entry) + pbentry, err := proto.Marshal(&entry) if err != nil { - return fmt.Errorf("failed to marshall json: %w", err) + return fmt.Errorf("failed to marshall protobuf: %w", err) } - err = bucket.Put([]byte(entry.Key), []byte(jsonentry)) + err = bucket.Put([]byte(entry.Key), []byte(pbentry)) if err != nil { return fmt.Errorf("failed to insert data: %w", err) } @@ -271,14 +262,14 @@ func (db *DB) Get(attr *DbAttr) (*DbEntry, error) { return nil } - jsonentry := bucket.Get([]byte(attr.Key)) - if jsonentry == nil { + pbentry := bucket.Get([]byte(attr.Key)) + if pbentry == nil { // FIXME: shall we return a key not found error? return nil } - if err := json.Unmarshal(jsonentry, &entry); err != nil { - return fmt.Errorf("failed to unmarshal from json: %w", err) + if err := proto.Unmarshal(pbentry, &entry); err != nil { + return fmt.Errorf("failed to unmarshal from protobuf: %w", err) } return nil @@ -352,12 +343,12 @@ func (db *DB) Import(attr *DbAttr) (string, error) { } for _, entry := range entries { - jsonentry, err := json.Marshal(entry) + pbentry, err := proto.Marshal(&entry) if err != nil { - return fmt.Errorf("failed to marshall json: %w", err) + return fmt.Errorf("failed to marshall protobuf: %w", err) } - err = bucket.Put([]byte(entry.Key), []byte(jsonentry)) + err = bucket.Put([]byte(entry.Key), []byte(pbentry)) if err != nil { return fmt.Errorf("failed to insert data into DB: %w", err) } diff --git a/app/dbentry.pb.go b/app/dbentry.pb.go new file mode 100644 index 0000000..06f7469 --- /dev/null +++ b/app/dbentry.pb.go @@ -0,0 +1,198 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.1 +// protoc v3.21.12 +// source: app/dbentry.proto + +package app + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type DbEntry struct { + state protoimpl.MessageState `protogen:"open.v1"` + Id string `protobuf:"bytes,1,opt,name=Id,proto3" json:"Id,omitempty"` + Key string `protobuf:"bytes,2,opt,name=Key,proto3" json:"Key,omitempty"` + Value string `protobuf:"bytes,3,opt,name=Value,proto3" json:"Value,omitempty"` + Bin []byte `protobuf:"bytes,4,opt,name=Bin,proto3" json:"Bin,omitempty"` + Tags []string `protobuf:"bytes,5,rep,name=Tags,proto3" json:"Tags,omitempty"` + Created *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=Created,proto3" json:"Created,omitempty"` + Size uint64 `protobuf:"varint,7,opt,name=Size,proto3" json:"Size,omitempty"` + Encrypted bool `protobuf:"varint,8,opt,name=Encrypted,proto3" json:"Encrypted,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DbEntry) Reset() { + *x = DbEntry{} + mi := &file_app_dbentry_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DbEntry) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DbEntry) ProtoMessage() {} + +func (x *DbEntry) ProtoReflect() protoreflect.Message { + mi := &file_app_dbentry_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DbEntry.ProtoReflect.Descriptor instead. +func (*DbEntry) Descriptor() ([]byte, []int) { + return file_app_dbentry_proto_rawDescGZIP(), []int{0} +} + +func (x *DbEntry) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *DbEntry) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *DbEntry) GetValue() string { + if x != nil { + return x.Value + } + return "" +} + +func (x *DbEntry) GetBin() []byte { + if x != nil { + return x.Bin + } + return nil +} + +func (x *DbEntry) GetTags() []string { + if x != nil { + return x.Tags + } + return nil +} + +func (x *DbEntry) GetCreated() *timestamppb.Timestamp { + if x != nil { + return x.Created + } + return nil +} + +func (x *DbEntry) GetSize() uint64 { + if x != nil { + return x.Size + } + return 0 +} + +func (x *DbEntry) GetEncrypted() bool { + if x != nil { + return x.Encrypted + } + return false +} + +var File_app_dbentry_proto protoreflect.FileDescriptor + +var file_app_dbentry_proto_rawDesc = []byte{ + 0x0a, 0x11, 0x61, 0x70, 0x70, 0x2f, 0x64, 0x62, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x12, 0x03, 0x61, 0x70, 0x70, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xcf, 0x01, 0x0a, 0x07, 0x44, 0x62, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x02, 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x4b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x10, 0x0a, + 0x03, 0x42, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x42, 0x69, 0x6e, 0x12, + 0x12, 0x0a, 0x04, 0x54, 0x61, 0x67, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x54, + 0x61, 0x67, 0x73, 0x12, 0x34, 0x0a, 0x07, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x18, 0x06, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x52, 0x07, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x53, 0x69, 0x7a, + 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1c, 0x0a, + 0x09, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x09, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x42, 0x1e, 0x5a, 0x1c, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x6c, 0x69, 0x6e, 0x64, 0x65, + 0x6e, 0x2f, 0x61, 0x6e, 0x79, 0x64, 0x62, 0x2f, 0x61, 0x70, 0x70, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, +} + +var ( + file_app_dbentry_proto_rawDescOnce sync.Once + file_app_dbentry_proto_rawDescData = file_app_dbentry_proto_rawDesc +) + +func file_app_dbentry_proto_rawDescGZIP() []byte { + file_app_dbentry_proto_rawDescOnce.Do(func() { + file_app_dbentry_proto_rawDescData = protoimpl.X.CompressGZIP(file_app_dbentry_proto_rawDescData) + }) + return file_app_dbentry_proto_rawDescData +} + +var file_app_dbentry_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_app_dbentry_proto_goTypes = []any{ + (*DbEntry)(nil), // 0: app.DbEntry + (*timestamppb.Timestamp)(nil), // 1: google.protobuf.Timestamp +} +var file_app_dbentry_proto_depIdxs = []int32{ + 1, // 0: app.DbEntry.Created:type_name -> google.protobuf.Timestamp + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_app_dbentry_proto_init() } +func file_app_dbentry_proto_init() { + if File_app_dbentry_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_app_dbentry_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_app_dbentry_proto_goTypes, + DependencyIndexes: file_app_dbentry_proto_depIdxs, + MessageInfos: file_app_dbentry_proto_msgTypes, + }.Build() + File_app_dbentry_proto = out.File + file_app_dbentry_proto_rawDesc = nil + file_app_dbentry_proto_goTypes = nil + file_app_dbentry_proto_depIdxs = nil +} diff --git a/app/dbentry.proto b/app/dbentry.proto new file mode 100644 index 0000000..8904334 --- /dev/null +++ b/app/dbentry.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; +package app; + +import "google/protobuf/timestamp.proto"; + +option go_package = "github.com/tlinden/anydb/app"; + +message DbEntry { + string Id = 1; + string Key = 2; + string Value = 3; + bytes Bin = 4; + repeated string Tags = 5; + google.protobuf.Timestamp Created = 6; + uint64 Size = 7; + bool Encrypted = 8; +} diff --git a/go.mod b/go.mod index 676bba0..4e4d98a 100644 --- a/go.mod +++ b/go.mod @@ -29,4 +29,5 @@ require ( golang.org/x/sys v0.28.0 // indirect golang.org/x/term v0.27.0 // indirect golang.org/x/tools v0.22.0 // indirect + google.golang.org/protobuf v1.36.1 // indirect ) diff --git a/go.sum b/go.sum index 0b195c4..29263e6 100644 --- a/go.sum +++ b/go.sum @@ -89,6 +89,8 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= +google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/output/list.go b/output/list.go index eeaadf4..1d29efe 100644 --- a/output/list.go +++ b/output/list.go @@ -102,8 +102,8 @@ func ListTable(writer io.Writer, conf *cfg.Config, entries app.DbEntries) error table.Append([]string{ row.Key, strings.Join(row.Tags, ","), - strconv.Itoa(row.Size), - row.Created.Format("02.01.2006T03:04.05"), + strconv.FormatUint(row.Size, 10), + row.Created.AsTime().Format("02.01.2006T03:04.05"), row.Value, }) default: @@ -112,7 +112,7 @@ func ListTable(writer io.Writer, conf *cfg.Config, entries app.DbEntries) error strings.Join(row.Tags, ","), humanize.Bytes(uint64(row.Size)), //row.Created.Format("02.01.2006T03:04.05"), - humanize.Time(row.Created), + humanize.Time(row.Created.AsTime()), row.Value, }) }