switch to use protobuf for internal data structure in DB

Makefile

@@ -29,7 +29,7 @@ BUILD     = $(shell date +%Y.%m.%d.%H%M%S)
 VERSION  := $(if $(filter $(BRANCH), development),$(version)-$(BRANCH)-$(COMMIT)-$(BUILD),$(version))
 HAVE_POD := $(shell pod2text -h 2>/dev/null)
 
-all: $(tool).1 cmd/$(tool).go buildlocal
+all: $(tool).1 cmd/$(tool).go app/dbentry.pb.go buildlocal
 
 %.1: %.pod
 ifdef HAVE_POD
@@ -49,6 +49,11 @@ endif
 # awk '/SYNOPS/{f=1;next} /DESCR/{f=0} f' $*.pod  | sed 's/^    //' >> cmd/$*.go
 # echo "\`" >> cmd/$*.go
 
+app/dbentry.pb.go: app/dbentry.proto
+	protoc -I=. --go_out=app app/dbentry.proto
+	mv app/github.com/tlinden/anydb/app/dbentry.pb.go app/dbentry.pb.go
+	rm -rf app/github.com
+
 buildlocal:
 	go build -ldflags "-X 'github.com/tlinden/anydb/cfg.VERSION=$(VERSION)'"
 

TODO.md

@@ -1,5 +1,35 @@
+## Features
+
 - repl
 - mime-type => exec app + value
 - add waitgroup to db.go funcs
 - RestList does not support any params?
-- lc() incoming tags
+- lc() incoming tags+keys
+
+## DB Structure
+
+- put tags into sub bucket see #1
+- change structure to:
+
+data bucket
+key => {key,value[0:60],isbin:bool}
+
+value bucket
+key => value (maybe always use []byte here)
+
+tags bucket
+key/tag => tag/key
+tag/key => tag
+
+So, list just uses the data bucket, no large contents.
+A tag search only looksup matching tags, see #1.
+Only a full text search and get would need to dig into the value bucket.
+
+A delete would just delete all keys from all values and then:
+lookup in tags bucket for all key/*, then iterate over the values and
+remove all tag/key's. Then deleting a key would not leave any residue
+behind.
+
+However, maybe change the list command to just list everything and add
+an extra find command for fulltext or tag search. Maybe still provide
+filter options in list command but only filter for keys.

anydb.1

@@ -1,4 +1,4 @@
-.\" Automatically generated by Pod::Man 4.14 (Pod::Simple 3.40)
+.\" Automatically generated by Pod::Man 4.14 (Pod::Simple 3.42)
 .\"
 .\" Standard preamble:
 .\" ========================================================================
@@ -133,7 +133,7 @@
 .\" ========================================================================
 .\"
 .IX Title "ANYDB 1"
-.TH ANYDB 1 "2024-12-25" "1" "User Commands"
+.TH ANYDB 1 "2024-12-29" "1" "User Commands"
 .\" For nroff, turn off justification.  Always turn off hyphenation; it makes
 .\" way too many mistakes in technical documents.
 .if n .ad l

app/db.go

@@ -27,6 +27,8 @@ import (
 	"time"
 
 	bolt "go.etcd.io/bbolt"
+	"google.golang.org/protobuf/proto"
+	timestamppb "google.golang.org/protobuf/types/known/timestamppb"
 )
 
 const MaxValueWidth int = 60
@@ -38,17 +40,6 @@ type DB struct {
 	DB     *bolt.DB
 }
 
-type DbEntry struct {
-	Id        string    `json:"id"`
-	Key       string    `json:"key"`
-	Value     string    `json:"value"`
-	Encrypted bool      `json:"encrypted"`
-	Bin       []byte    `json:"bin"`
-	Tags      []string  `json:"tags"`
-	Created   time.Time `json:"created"`
-	Size      int
-}
-
 type BucketInfo struct {
 	Name     string
 	Keys     int
@@ -65,7 +56,7 @@ type DbInfo struct {
 // Post process an entry for list output.
 // Do NOT call it during write processing!
 func (entry *DbEntry) Normalize() {
-	entry.Size = len(entry.Value)
+	entry.Size = uint64(len(entry.Value))
 
 	if entry.Encrypted {
 		entry.Value = "<encrypted-content>"
@@ -73,7 +64,7 @@ func (entry *DbEntry) Normalize() {
 
 	if len(entry.Bin) > 0 {
 		entry.Value = "<binary-content>"
-		entry.Size = len(entry.Bin)
+		entry.Size = uint64(len(entry.Bin))
 	}
 
 	if strings.Contains(entry.Value, "\n") {
@@ -140,10 +131,10 @@ func (db *DB) List(attr *DbAttr) (DbEntries, error) {
 			return nil
 		}
 
-		err := bucket.ForEach(func(key, jsonentry []byte) error {
+		err := bucket.ForEach(func(key, pbentry []byte) error {
 			var entry DbEntry
-			if err := json.Unmarshal(jsonentry, &entry); err != nil {
+			if err := proto.Unmarshal(pbentry, &entry); err != nil {
-				return fmt.Errorf("failed to unmarshal from json: %w", err)
+				return fmt.Errorf("failed to unmarshal from protobuf: %w", err)
 			}
 
 			var include bool
@@ -196,7 +187,7 @@ func (db *DB) Set(attr *DbAttr) error {
 		Bin:       attr.Bin,
 		Tags:      attr.Tags,
 		Encrypted: attr.Encrypted,
-		Created:   time.Now(),
+		Created:   timestamppb.Now(),
 	}
 
 	// check if the  entry already exists and if yes,  check if it has
@@ -208,14 +199,14 @@ func (db *DB) Set(attr *DbAttr) error {
 			return nil
 		}
 
-		jsonentry := bucket.Get([]byte(entry.Key))
+		pbentry := bucket.Get([]byte(entry.Key))
-		if jsonentry == nil {
+		if pbentry == nil {
 			return nil
 		}
 
 		var oldentry DbEntry
-		if err := json.Unmarshal(jsonentry, &oldentry); err != nil {
+		if err := proto.Unmarshal(pbentry, &oldentry); err != nil {
-			return fmt.Errorf("failed to unmarshal from json: %w", err)
+			return fmt.Errorf("failed to unmarshal from protobuf: %w", err)
 		}
 
 		if len(oldentry.Tags) > 0 && len(entry.Tags) == 0 {
@@ -237,12 +228,12 @@ func (db *DB) Set(attr *DbAttr) error {
 			return fmt.Errorf("failed to create DB bucket: %w", err)
 		}
 
-		jsonentry, err := json.Marshal(entry)
+		pbentry, err := proto.Marshal(&entry)
 		if err != nil {
-			return fmt.Errorf("failed to marshall json: %w", err)
+			return fmt.Errorf("failed to marshall protobuf: %w", err)
 		}
 
-		err = bucket.Put([]byte(entry.Key), []byte(jsonentry))
+		err = bucket.Put([]byte(entry.Key), []byte(pbentry))
 		if err != nil {
 			return fmt.Errorf("failed to insert data: %w", err)
 		}
@@ -271,14 +262,14 @@ func (db *DB) Get(attr *DbAttr) (*DbEntry, error) {
 			return nil
 		}
 
-		jsonentry := bucket.Get([]byte(attr.Key))
+		pbentry := bucket.Get([]byte(attr.Key))
-		if jsonentry == nil {
+		if pbentry == nil {
 			// FIXME: shall we return a key not found error?
 			return nil
 		}
 
-		if err := json.Unmarshal(jsonentry, &entry); err != nil {
+		if err := proto.Unmarshal(pbentry, &entry); err != nil {
-			return fmt.Errorf("failed to unmarshal from json: %w", err)
+			return fmt.Errorf("failed to unmarshal from protobuf: %w", err)
 		}
 
 		return nil
@@ -352,12 +343,12 @@ func (db *DB) Import(attr *DbAttr) (string, error) {
 		}
 
 		for _, entry := range entries {
-			jsonentry, err := json.Marshal(entry)
+			pbentry, err := proto.Marshal(&entry)
 			if err != nil {
-				return fmt.Errorf("failed to marshall json: %w", err)
+				return fmt.Errorf("failed to marshall protobuf: %w", err)
 			}
 
-			err = bucket.Put([]byte(entry.Key), []byte(jsonentry))
+			err = bucket.Put([]byte(entry.Key), []byte(pbentry))
 			if err != nil {
 				return fmt.Errorf("failed to insert data into DB: %w", err)
 			}

app/dbentry.pb.go

@@ -0,0 +1,198 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.36.1
+// 	protoc        v3.21.12
+// source: app/dbentry.proto
+
+package app
+
+import (
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	timestamppb "google.golang.org/protobuf/types/known/timestamppb"
+	reflect "reflect"
+	sync "sync"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+type DbEntry struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Id            string                 `protobuf:"bytes,1,opt,name=Id,proto3" json:"Id,omitempty"`
+	Key           string                 `protobuf:"bytes,2,opt,name=Key,proto3" json:"Key,omitempty"`
+	Value         string                 `protobuf:"bytes,3,opt,name=Value,proto3" json:"Value,omitempty"`
+	Bin           []byte                 `protobuf:"bytes,4,opt,name=Bin,proto3" json:"Bin,omitempty"`
+	Tags          []string               `protobuf:"bytes,5,rep,name=Tags,proto3" json:"Tags,omitempty"`
+	Created       *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=Created,proto3" json:"Created,omitempty"`
+	Size          uint64                 `protobuf:"varint,7,opt,name=Size,proto3" json:"Size,omitempty"`
+	Encrypted     bool                   `protobuf:"varint,8,opt,name=Encrypted,proto3" json:"Encrypted,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *DbEntry) Reset() {
+	*x = DbEntry{}
+	mi := &file_app_dbentry_proto_msgTypes[0]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *DbEntry) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*DbEntry) ProtoMessage() {}
+
+func (x *DbEntry) ProtoReflect() protoreflect.Message {
+	mi := &file_app_dbentry_proto_msgTypes[0]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use DbEntry.ProtoReflect.Descriptor instead.
+func (*DbEntry) Descriptor() ([]byte, []int) {
+	return file_app_dbentry_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *DbEntry) GetId() string {
+	if x != nil {
+		return x.Id
+	}
+	return ""
+}
+
+func (x *DbEntry) GetKey() string {
+	if x != nil {
+		return x.Key
+	}
+	return ""
+}
+
+func (x *DbEntry) GetValue() string {
+	if x != nil {
+		return x.Value
+	}
+	return ""
+}
+
+func (x *DbEntry) GetBin() []byte {
+	if x != nil {
+		return x.Bin
+	}
+	return nil
+}
+
+func (x *DbEntry) GetTags() []string {
+	if x != nil {
+		return x.Tags
+	}
+	return nil
+}
+
+func (x *DbEntry) GetCreated() *timestamppb.Timestamp {
+	if x != nil {
+		return x.Created
+	}
+	return nil
+}
+
+func (x *DbEntry) GetSize() uint64 {
+	if x != nil {
+		return x.Size
+	}
+	return 0
+}
+
+func (x *DbEntry) GetEncrypted() bool {
+	if x != nil {
+		return x.Encrypted
+	}
+	return false
+}
+
+var File_app_dbentry_proto protoreflect.FileDescriptor
+
+var file_app_dbentry_proto_rawDesc = []byte{
+	0x0a, 0x11, 0x61, 0x70, 0x70, 0x2f, 0x64, 0x62, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x12, 0x03, 0x61, 0x70, 0x70, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
+	0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74,
+	0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xcf, 0x01, 0x0a, 0x07, 0x44, 0x62,
+	0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x02, 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01,
+	0x28, 0x09, 0x52, 0x03, 0x4b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65,
+	0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x10, 0x0a,
+	0x03, 0x42, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x42, 0x69, 0x6e, 0x12,
+	0x12, 0x0a, 0x04, 0x54, 0x61, 0x67, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x54,
+	0x61, 0x67, 0x73, 0x12, 0x34, 0x0a, 0x07, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x18, 0x06,
+	0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70,
+	0x52, 0x07, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x53, 0x69, 0x7a,
+	0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1c, 0x0a,
+	0x09, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08,
+	0x52, 0x09, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x42, 0x1e, 0x5a, 0x1c, 0x67,
+	0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x6c, 0x69, 0x6e, 0x64, 0x65,
+	0x6e, 0x2f, 0x61, 0x6e, 0x79, 0x64, 0x62, 0x2f, 0x61, 0x70, 0x70, 0x62, 0x06, 0x70, 0x72, 0x6f,
+	0x74, 0x6f, 0x33,
+}
+
+var (
+	file_app_dbentry_proto_rawDescOnce sync.Once
+	file_app_dbentry_proto_rawDescData = file_app_dbentry_proto_rawDesc
+)
+
+func file_app_dbentry_proto_rawDescGZIP() []byte {
+	file_app_dbentry_proto_rawDescOnce.Do(func() {
+		file_app_dbentry_proto_rawDescData = protoimpl.X.CompressGZIP(file_app_dbentry_proto_rawDescData)
+	})
+	return file_app_dbentry_proto_rawDescData
+}
+
+var file_app_dbentry_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
+var file_app_dbentry_proto_goTypes = []any{
+	(*DbEntry)(nil),               // 0: app.DbEntry
+	(*timestamppb.Timestamp)(nil), // 1: google.protobuf.Timestamp
+}
+var file_app_dbentry_proto_depIdxs = []int32{
+	1, // 0: app.DbEntry.Created:type_name -> google.protobuf.Timestamp
+	1, // [1:1] is the sub-list for method output_type
+	1, // [1:1] is the sub-list for method input_type
+	1, // [1:1] is the sub-list for extension type_name
+	1, // [1:1] is the sub-list for extension extendee
+	0, // [0:1] is the sub-list for field type_name
+}
+
+func init() { file_app_dbentry_proto_init() }
+func file_app_dbentry_proto_init() {
+	if File_app_dbentry_proto != nil {
+		return
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_app_dbentry_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   1,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_app_dbentry_proto_goTypes,
+		DependencyIndexes: file_app_dbentry_proto_depIdxs,
+		MessageInfos:      file_app_dbentry_proto_msgTypes,
+	}.Build()
+	File_app_dbentry_proto = out.File
+	file_app_dbentry_proto_rawDesc = nil
+	file_app_dbentry_proto_goTypes = nil
+	file_app_dbentry_proto_depIdxs = nil
+}

app/dbentry.proto

@@ -0,0 +1,17 @@
+syntax = "proto3";
+package app;
+
+import "google/protobuf/timestamp.proto";
+
+option go_package = "github.com/tlinden/anydb/app";
+
+message DbEntry {
+  string Id = 1;
+  string Key = 2;
+  string Value = 3;
+  bytes Bin = 4;
+  repeated string Tags = 5;
+  google.protobuf.Timestamp Created = 6;
+  uint64 Size = 7;
+  bool Encrypted = 8;
+}

go.mod

@@ -29,4 +29,5 @@ require (
 	golang.org/x/sys v0.28.0 // indirect
 	golang.org/x/term v0.27.0 // indirect
 	golang.org/x/tools v0.22.0 // indirect
+	google.golang.org/protobuf v1.36.1 // indirect
 )

go.sum

@@ -89,6 +89,8 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA=
 golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk=
+google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

output/list.go

@@ -102,8 +102,8 @@ func ListTable(writer io.Writer, conf *cfg.Config, entries app.DbEntries) error
 				table.Append([]string{
 					row.Key,
 					strings.Join(row.Tags, ","),
-					strconv.Itoa(row.Size),
+					strconv.FormatUint(row.Size, 10),
-					row.Created.Format("02.01.2006T03:04.05"),
+					row.Created.AsTime().Format("02.01.2006T03:04.05"),
 					row.Value,
 				})
 			default:
@@ -112,7 +112,7 @@ func ListTable(writer io.Writer, conf *cfg.Config, entries app.DbEntries) error
 					strings.Join(row.Tags, ","),
 					humanize.Bytes(uint64(row.Size)),
 					//row.Created.Format("02.01.2006T03:04.05"),
-					humanize.Time(row.Created),
+					humanize.Time(row.Created.AsTime()),
 					row.Value,
 				})
 			}