mirror of
https://codeberg.org/scip/anydb.git
synced 2025-12-17 04:20:59 +01:00
switch to use protobuf for internal data structure in DB
This commit is contained in:
53
app/db.go
53
app/db.go
@@ -27,6 +27,8 @@ import (
|
||||
"time"
|
||||
|
||||
bolt "go.etcd.io/bbolt"
|
||||
"google.golang.org/protobuf/proto"
|
||||
timestamppb "google.golang.org/protobuf/types/known/timestamppb"
|
||||
)
|
||||
|
||||
const MaxValueWidth int = 60
|
||||
@@ -38,17 +40,6 @@ type DB struct {
|
||||
DB *bolt.DB
|
||||
}
|
||||
|
||||
type DbEntry struct {
|
||||
Id string `json:"id"`
|
||||
Key string `json:"key"`
|
||||
Value string `json:"value"`
|
||||
Encrypted bool `json:"encrypted"`
|
||||
Bin []byte `json:"bin"`
|
||||
Tags []string `json:"tags"`
|
||||
Created time.Time `json:"created"`
|
||||
Size int
|
||||
}
|
||||
|
||||
type BucketInfo struct {
|
||||
Name string
|
||||
Keys int
|
||||
@@ -65,7 +56,7 @@ type DbInfo struct {
|
||||
// Post process an entry for list output.
|
||||
// Do NOT call it during write processing!
|
||||
func (entry *DbEntry) Normalize() {
|
||||
entry.Size = len(entry.Value)
|
||||
entry.Size = uint64(len(entry.Value))
|
||||
|
||||
if entry.Encrypted {
|
||||
entry.Value = "<encrypted-content>"
|
||||
@@ -73,7 +64,7 @@ func (entry *DbEntry) Normalize() {
|
||||
|
||||
if len(entry.Bin) > 0 {
|
||||
entry.Value = "<binary-content>"
|
||||
entry.Size = len(entry.Bin)
|
||||
entry.Size = uint64(len(entry.Bin))
|
||||
}
|
||||
|
||||
if strings.Contains(entry.Value, "\n") {
|
||||
@@ -140,10 +131,10 @@ func (db *DB) List(attr *DbAttr) (DbEntries, error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
err := bucket.ForEach(func(key, jsonentry []byte) error {
|
||||
err := bucket.ForEach(func(key, pbentry []byte) error {
|
||||
var entry DbEntry
|
||||
if err := json.Unmarshal(jsonentry, &entry); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal from json: %w", err)
|
||||
if err := proto.Unmarshal(pbentry, &entry); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal from protobuf: %w", err)
|
||||
}
|
||||
|
||||
var include bool
|
||||
@@ -196,7 +187,7 @@ func (db *DB) Set(attr *DbAttr) error {
|
||||
Bin: attr.Bin,
|
||||
Tags: attr.Tags,
|
||||
Encrypted: attr.Encrypted,
|
||||
Created: time.Now(),
|
||||
Created: timestamppb.Now(),
|
||||
}
|
||||
|
||||
// check if the entry already exists and if yes, check if it has
|
||||
@@ -208,14 +199,14 @@ func (db *DB) Set(attr *DbAttr) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
jsonentry := bucket.Get([]byte(entry.Key))
|
||||
if jsonentry == nil {
|
||||
pbentry := bucket.Get([]byte(entry.Key))
|
||||
if pbentry == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var oldentry DbEntry
|
||||
if err := json.Unmarshal(jsonentry, &oldentry); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal from json: %w", err)
|
||||
if err := proto.Unmarshal(pbentry, &oldentry); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal from protobuf: %w", err)
|
||||
}
|
||||
|
||||
if len(oldentry.Tags) > 0 && len(entry.Tags) == 0 {
|
||||
@@ -237,12 +228,12 @@ func (db *DB) Set(attr *DbAttr) error {
|
||||
return fmt.Errorf("failed to create DB bucket: %w", err)
|
||||
}
|
||||
|
||||
jsonentry, err := json.Marshal(entry)
|
||||
pbentry, err := proto.Marshal(&entry)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshall json: %w", err)
|
||||
return fmt.Errorf("failed to marshall protobuf: %w", err)
|
||||
}
|
||||
|
||||
err = bucket.Put([]byte(entry.Key), []byte(jsonentry))
|
||||
err = bucket.Put([]byte(entry.Key), []byte(pbentry))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to insert data: %w", err)
|
||||
}
|
||||
@@ -271,14 +262,14 @@ func (db *DB) Get(attr *DbAttr) (*DbEntry, error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
jsonentry := bucket.Get([]byte(attr.Key))
|
||||
if jsonentry == nil {
|
||||
pbentry := bucket.Get([]byte(attr.Key))
|
||||
if pbentry == nil {
|
||||
// FIXME: shall we return a key not found error?
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(jsonentry, &entry); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal from json: %w", err)
|
||||
if err := proto.Unmarshal(pbentry, &entry); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal from protobuf: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -352,12 +343,12 @@ func (db *DB) Import(attr *DbAttr) (string, error) {
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
jsonentry, err := json.Marshal(entry)
|
||||
pbentry, err := proto.Marshal(&entry)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshall json: %w", err)
|
||||
return fmt.Errorf("failed to marshall protobuf: %w", err)
|
||||
}
|
||||
|
||||
err = bucket.Put([]byte(entry.Key), []byte(jsonentry))
|
||||
err = bucket.Put([]byte(entry.Key), []byte(pbentry))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to insert data into DB: %w", err)
|
||||
}
|
||||
|
||||
198
app/dbentry.pb.go
Normal file
198
app/dbentry.pb.go
Normal file
@@ -0,0 +1,198 @@
|
||||
// Code generated by protoc-gen-go. DO NOT EDIT.
|
||||
// versions:
|
||||
// protoc-gen-go v1.36.1
|
||||
// protoc v3.21.12
|
||||
// source: app/dbentry.proto
|
||||
|
||||
package app
|
||||
|
||||
import (
|
||||
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
|
||||
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
|
||||
timestamppb "google.golang.org/protobuf/types/known/timestamppb"
|
||||
reflect "reflect"
|
||||
sync "sync"
|
||||
)
|
||||
|
||||
const (
|
||||
// Verify that this generated code is sufficiently up-to-date.
|
||||
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
|
||||
// Verify that runtime/protoimpl is sufficiently up-to-date.
|
||||
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
|
||||
)
|
||||
|
||||
type DbEntry struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
Id string `protobuf:"bytes,1,opt,name=Id,proto3" json:"Id,omitempty"`
|
||||
Key string `protobuf:"bytes,2,opt,name=Key,proto3" json:"Key,omitempty"`
|
||||
Value string `protobuf:"bytes,3,opt,name=Value,proto3" json:"Value,omitempty"`
|
||||
Bin []byte `protobuf:"bytes,4,opt,name=Bin,proto3" json:"Bin,omitempty"`
|
||||
Tags []string `protobuf:"bytes,5,rep,name=Tags,proto3" json:"Tags,omitempty"`
|
||||
Created *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=Created,proto3" json:"Created,omitempty"`
|
||||
Size uint64 `protobuf:"varint,7,opt,name=Size,proto3" json:"Size,omitempty"`
|
||||
Encrypted bool `protobuf:"varint,8,opt,name=Encrypted,proto3" json:"Encrypted,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *DbEntry) Reset() {
|
||||
*x = DbEntry{}
|
||||
mi := &file_app_dbentry_proto_msgTypes[0]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *DbEntry) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*DbEntry) ProtoMessage() {}
|
||||
|
||||
func (x *DbEntry) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_app_dbentry_proto_msgTypes[0]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use DbEntry.ProtoReflect.Descriptor instead.
|
||||
func (*DbEntry) Descriptor() ([]byte, []int) {
|
||||
return file_app_dbentry_proto_rawDescGZIP(), []int{0}
|
||||
}
|
||||
|
||||
func (x *DbEntry) GetId() string {
|
||||
if x != nil {
|
||||
return x.Id
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *DbEntry) GetKey() string {
|
||||
if x != nil {
|
||||
return x.Key
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *DbEntry) GetValue() string {
|
||||
if x != nil {
|
||||
return x.Value
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *DbEntry) GetBin() []byte {
|
||||
if x != nil {
|
||||
return x.Bin
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *DbEntry) GetTags() []string {
|
||||
if x != nil {
|
||||
return x.Tags
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *DbEntry) GetCreated() *timestamppb.Timestamp {
|
||||
if x != nil {
|
||||
return x.Created
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *DbEntry) GetSize() uint64 {
|
||||
if x != nil {
|
||||
return x.Size
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *DbEntry) GetEncrypted() bool {
|
||||
if x != nil {
|
||||
return x.Encrypted
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
var File_app_dbentry_proto protoreflect.FileDescriptor
|
||||
|
||||
var file_app_dbentry_proto_rawDesc = []byte{
|
||||
0x0a, 0x11, 0x61, 0x70, 0x70, 0x2f, 0x64, 0x62, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72,
|
||||
0x6f, 0x74, 0x6f, 0x12, 0x03, 0x61, 0x70, 0x70, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
|
||||
0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74,
|
||||
0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xcf, 0x01, 0x0a, 0x07, 0x44, 0x62,
|
||||
0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
|
||||
0x09, 0x52, 0x02, 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01,
|
||||
0x28, 0x09, 0x52, 0x03, 0x4b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65,
|
||||
0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x10, 0x0a,
|
||||
0x03, 0x42, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x42, 0x69, 0x6e, 0x12,
|
||||
0x12, 0x0a, 0x04, 0x54, 0x61, 0x67, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x54,
|
||||
0x61, 0x67, 0x73, 0x12, 0x34, 0x0a, 0x07, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x18, 0x06,
|
||||
0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72,
|
||||
0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70,
|
||||
0x52, 0x07, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x53, 0x69, 0x7a,
|
||||
0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1c, 0x0a,
|
||||
0x09, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08,
|
||||
0x52, 0x09, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x42, 0x1e, 0x5a, 0x1c, 0x67,
|
||||
0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x6c, 0x69, 0x6e, 0x64, 0x65,
|
||||
0x6e, 0x2f, 0x61, 0x6e, 0x79, 0x64, 0x62, 0x2f, 0x61, 0x70, 0x70, 0x62, 0x06, 0x70, 0x72, 0x6f,
|
||||
0x74, 0x6f, 0x33,
|
||||
}
|
||||
|
||||
var (
|
||||
file_app_dbentry_proto_rawDescOnce sync.Once
|
||||
file_app_dbentry_proto_rawDescData = file_app_dbentry_proto_rawDesc
|
||||
)
|
||||
|
||||
func file_app_dbentry_proto_rawDescGZIP() []byte {
|
||||
file_app_dbentry_proto_rawDescOnce.Do(func() {
|
||||
file_app_dbentry_proto_rawDescData = protoimpl.X.CompressGZIP(file_app_dbentry_proto_rawDescData)
|
||||
})
|
||||
return file_app_dbentry_proto_rawDescData
|
||||
}
|
||||
|
||||
var file_app_dbentry_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
|
||||
var file_app_dbentry_proto_goTypes = []any{
|
||||
(*DbEntry)(nil), // 0: app.DbEntry
|
||||
(*timestamppb.Timestamp)(nil), // 1: google.protobuf.Timestamp
|
||||
}
|
||||
var file_app_dbentry_proto_depIdxs = []int32{
|
||||
1, // 0: app.DbEntry.Created:type_name -> google.protobuf.Timestamp
|
||||
1, // [1:1] is the sub-list for method output_type
|
||||
1, // [1:1] is the sub-list for method input_type
|
||||
1, // [1:1] is the sub-list for extension type_name
|
||||
1, // [1:1] is the sub-list for extension extendee
|
||||
0, // [0:1] is the sub-list for field type_name
|
||||
}
|
||||
|
||||
func init() { file_app_dbentry_proto_init() }
|
||||
func file_app_dbentry_proto_init() {
|
||||
if File_app_dbentry_proto != nil {
|
||||
return
|
||||
}
|
||||
type x struct{}
|
||||
out := protoimpl.TypeBuilder{
|
||||
File: protoimpl.DescBuilder{
|
||||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
||||
RawDescriptor: file_app_dbentry_proto_rawDesc,
|
||||
NumEnums: 0,
|
||||
NumMessages: 1,
|
||||
NumExtensions: 0,
|
||||
NumServices: 0,
|
||||
},
|
||||
GoTypes: file_app_dbentry_proto_goTypes,
|
||||
DependencyIndexes: file_app_dbentry_proto_depIdxs,
|
||||
MessageInfos: file_app_dbentry_proto_msgTypes,
|
||||
}.Build()
|
||||
File_app_dbentry_proto = out.File
|
||||
file_app_dbentry_proto_rawDesc = nil
|
||||
file_app_dbentry_proto_goTypes = nil
|
||||
file_app_dbentry_proto_depIdxs = nil
|
||||
}
|
||||
17
app/dbentry.proto
Normal file
17
app/dbentry.proto
Normal file
@@ -0,0 +1,17 @@
|
||||
syntax = "proto3";
|
||||
package app;
|
||||
|
||||
import "google/protobuf/timestamp.proto";
|
||||
|
||||
option go_package = "github.com/tlinden/anydb/app";
|
||||
|
||||
message DbEntry {
|
||||
string Id = 1;
|
||||
string Key = 2;
|
||||
string Value = 3;
|
||||
bytes Bin = 4;
|
||||
repeated string Tags = 5;
|
||||
google.protobuf.Timestamp Created = 6;
|
||||
uint64 Size = 7;
|
||||
bool Encrypted = 8;
|
||||
}
|
||||
Reference in New Issue
Block a user