restructured data storage, values now have their own sub bucket

This commit is contained in:
2024-12-29 18:29:43 +01:00
parent c144e99b41
commit a4b6a3cfdf
12 changed files with 330 additions and 151 deletions

View File

@@ -20,17 +20,19 @@ import (
"fmt"
"io"
"os"
"strings"
"unicode/utf8"
)
type DbAttr struct {
Key string
Val string
Bin []byte
Preview string
Val []byte
Args []string
Tags []string
File string
Encrypted bool
Binary bool
}
func (attr *DbAttr) ParseKV() error {
@@ -43,7 +45,7 @@ func (attr *DbAttr) ParseKV() error {
}
case 2:
attr.Key = attr.Args[0]
attr.Val = attr.Args[1]
attr.Val = []byte(attr.Args[1])
if attr.Args[1] == "-" {
attr.File = "-"
@@ -51,7 +53,29 @@ func (attr *DbAttr) ParseKV() error {
}
if attr.File != "" {
return attr.GetFileValue()
if err := attr.GetFileValue(); err != nil {
return err
}
}
if attr.Binary {
attr.Preview = "<encrypted-content>"
} else {
if len(attr.Val) > MaxValueWidth {
attr.Preview = string(attr.Val)[0:MaxValueWidth] + "..."
if strings.Contains(attr.Preview, "\n") {
parts := strings.Split(attr.Preview, "\n")
if len(parts) > 0 {
attr.Preview = parts[0]
}
}
} else {
attr.Preview = string(attr.Val)
}
}
if attr.Encrypted {
attr.Preview = "<encrypted-content>"
}
return nil
@@ -82,11 +106,12 @@ func (attr *DbAttr) GetFileValue() error {
}
// poor man's text file test
sdata := string(data)
if utf8.ValidString(sdata) {
attr.Val = sdata
attr.Val = data
if utf8.ValidString(string(data)) {
attr.Binary = false
} else {
attr.Bin = data
attr.Binary = true
}
} else {
// read from console stdin
@@ -101,7 +126,7 @@ func (attr *DbAttr) GetFileValue() error {
data += input + "\n"
}
attr.Val = data
attr.Val = []byte(data)
}
return nil

View File

@@ -18,7 +18,6 @@ package app
import (
"crypto/rand"
"encoding/base64"
"errors"
"fmt"
"os"
@@ -104,7 +103,7 @@ func GetRandom(size int, capacity int) ([]byte, error) {
// modifying it.
//
// The cipher text consists of:
// base64(password-salt) + base64(12 byte nonce + ciphertext + 16 byte mac)
// password-salt) + (12 byte nonce + ciphertext + 16 byte mac)
func Encrypt(pass []byte, attr *DbAttr) error {
key, err := DeriveKey(pass, nil)
if err != nil {
@@ -116,25 +115,17 @@ func Encrypt(pass []byte, attr *DbAttr) error {
return fmt.Errorf("failed to create AEAD cipher: %w", err)
}
var plain []byte
if attr.Val != "" {
plain = []byte(attr.Val)
} else {
plain = attr.Bin
}
total := aead.NonceSize() + len(plain) + aead.Overhead()
total := aead.NonceSize() + len(attr.Val) + aead.Overhead()
nonce, err := GetRandom(aead.NonceSize(), total)
if err != nil {
return err
}
cipher := aead.Seal(nonce, nonce, plain, nil)
cipher := aead.Seal(nonce, nonce, attr.Val, nil)
attr.Bin = nil
attr.Val = base64.RawStdEncoding.EncodeToString(key.Salt) +
base64.RawStdEncoding.EncodeToString(cipher)
attr.Val = append(attr.Val, key.Salt...)
attr.Val = append(attr.Val, cipher...)
attr.Encrypted = true
@@ -142,21 +133,17 @@ func Encrypt(pass []byte, attr *DbAttr) error {
}
// Do the reverse
func Decrypt(pass []byte, cipherb64 string) ([]byte, error) {
salt, err := base64.RawStdEncoding.Strict().DecodeString(cipherb64[0:B64SaltLen])
if err != nil {
return nil, fmt.Errorf("failed to encode to base64: %w", err)
func Decrypt(pass []byte, cipherb []byte) ([]byte, error) {
if len(cipherb) < B64SaltLen {
return nil, fmt.Errorf("encrypted cipher block too small")
}
key, err := DeriveKey(pass, salt)
key, err := DeriveKey(pass, cipherb[0:B64SaltLen])
if err != nil {
return nil, err
}
cipher, err := base64.RawStdEncoding.Strict().DecodeString(cipherb64[B64SaltLen:])
if err != nil {
return nil, fmt.Errorf("failed to encode to base64: %w", err)
}
cipher := cipherb[B64SaltLen:]
aead, err := chacha20poly1305.New(key.Key)
if err != nil {

204
app/db.go
View File

@@ -53,32 +53,6 @@ type DbInfo struct {
Path string
}
// Post process an entry for list output.
// Do NOT call it during write processing!
func (entry *DbEntry) Normalize() {
entry.Size = uint64(len(entry.Value))
if entry.Encrypted {
entry.Value = "<encrypted-content>"
}
if len(entry.Bin) > 0 {
entry.Value = "<binary-content>"
entry.Size = uint64(len(entry.Bin))
}
if strings.Contains(entry.Value, "\n") {
parts := strings.Split(entry.Value, "\n")
if len(parts) > 0 {
entry.Value = parts[0]
}
}
if len(entry.Value) > MaxValueWidth {
entry.Value = entry.Value[0:MaxValueWidth] + "..."
}
}
type DbEntries []DbEntry
type DbTag struct {
@@ -126,7 +100,12 @@ func (db *DB) List(attr *DbAttr) (DbEntries, error) {
err := db.DB.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(db.Bucket))
root := tx.Bucket([]byte(db.Bucket))
if root == nil {
return nil
}
bucket := root.Bucket([]byte("meta"))
if bucket == nil {
return nil
}
@@ -141,8 +120,7 @@ func (db *DB) List(attr *DbAttr) (DbEntries, error) {
switch {
case filter != nil:
if filter.MatchString(entry.Value) ||
filter.MatchString(entry.Key) ||
if filter.MatchString(entry.Key) ||
filter.MatchString(strings.Join(entry.Tags, " ")) {
include = true
}
@@ -183,18 +161,24 @@ func (db *DB) Set(attr *DbAttr) error {
entry := DbEntry{
Key: attr.Key,
Value: attr.Val,
Bin: attr.Bin,
Binary: attr.Binary,
Tags: attr.Tags,
Encrypted: attr.Encrypted,
Created: timestamppb.Now(),
Size: uint64(len(attr.Val)),
Preview: attr.Preview,
}
// check if the entry already exists and if yes, check if it has
// any tags. if so, we initialize our update struct with these
// tags unless it has new tags configured.
err := db.DB.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(db.Bucket))
root := tx.Bucket([]byte(db.Bucket))
if root == nil {
return nil
}
bucket := root.Bucket([]byte("meta"))
if bucket == nil {
return nil
}
@@ -221,23 +205,43 @@ func (db *DB) Set(attr *DbAttr) error {
return err
}
// marshall our data
pbentry, err := proto.Marshal(&entry)
if err != nil {
return fmt.Errorf("failed to marshall protobuf: %w", err)
}
err = db.DB.Update(func(tx *bolt.Tx) error {
// insert data
bucket, err := tx.CreateBucketIfNotExists([]byte(db.Bucket))
// create root bucket
root, err := tx.CreateBucketIfNotExists([]byte(db.Bucket))
if err != nil {
return fmt.Errorf("failed to create DB bucket: %w", err)
}
pbentry, err := proto.Marshal(&entry)
// create meta bucket
bucket, err := root.CreateBucketIfNotExists([]byte("meta"))
if err != nil {
return fmt.Errorf("failed to marshall protobuf: %w", err)
return fmt.Errorf("failed to create DB meta sub bucket: %w", err)
}
// write meta data
err = bucket.Put([]byte(entry.Key), []byte(pbentry))
if err != nil {
return fmt.Errorf("failed to insert data: %w", err)
}
// create data bucket
databucket, err := root.CreateBucketIfNotExists([]byte("data"))
if err != nil {
return fmt.Errorf("failed to create DB data sub bucket: %w", err)
}
// write value
err = databucket.Put([]byte(entry.Key), attr.Val)
if err != nil {
return fmt.Errorf("failed to insert data: %w", err)
}
return nil
})
@@ -257,21 +261,35 @@ func (db *DB) Get(attr *DbAttr) (*DbEntry, error) {
entry := DbEntry{}
err := db.DB.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(db.Bucket))
root := tx.Bucket([]byte(db.Bucket))
if root == nil {
return nil
}
bucket := root.Bucket([]byte("meta"))
if bucket == nil {
return nil
}
pbentry := bucket.Get([]byte(attr.Key))
if pbentry == nil {
// FIXME: shall we return a key not found error?
return nil
return fmt.Errorf("no such key: %s", attr.Key)
}
if err := proto.Unmarshal(pbentry, &entry); err != nil {
return fmt.Errorf("failed to unmarshal from protobuf: %w", err)
}
databucket := root.Bucket([]byte("data"))
if databucket == nil {
return fmt.Errorf("failed to retrieve data sub bucket")
}
entry.Value = databucket.Get([]byte(attr.Key))
if len(entry.Value) == 0 {
return fmt.Errorf("no such key: %s", attr.Key)
}
return nil
})
@@ -308,7 +326,7 @@ func (db *DB) Import(attr *DbAttr) (string, error) {
return "", err
}
if attr.Val == "" {
if len(attr.Val) == 0 {
return "", errors.New("empty json file")
}
@@ -336,10 +354,16 @@ func (db *DB) Import(attr *DbAttr) (string, error) {
defer db.Close()
err := db.DB.Update(func(tx *bolt.Tx) error {
// insert data
bucket, err := tx.CreateBucketIfNotExists([]byte(db.Bucket))
// create root bucket
root, err := tx.CreateBucketIfNotExists([]byte(db.Bucket))
if err != nil {
return fmt.Errorf("failed to create bucket: %w", err)
return fmt.Errorf("failed to create DB bucket: %w", err)
}
// create meta bucket
bucket, err := root.CreateBucketIfNotExists([]byte("meta"))
if err != nil {
return fmt.Errorf("failed to create DB meta sub bucket: %w", err)
}
for _, entry := range entries {
@@ -348,10 +372,23 @@ func (db *DB) Import(attr *DbAttr) (string, error) {
return fmt.Errorf("failed to marshall protobuf: %w", err)
}
// write meta data
err = bucket.Put([]byte(entry.Key), []byte(pbentry))
if err != nil {
return fmt.Errorf("failed to insert data into DB: %w", err)
}
// create data bucket
databucket, err := root.CreateBucketIfNotExists([]byte("data"))
if err != nil {
return fmt.Errorf("failed to create DB data sub bucket: %w", err)
}
// write value
err = databucket.Put([]byte(entry.Key), entry.Value)
if err != nil {
return fmt.Errorf("failed to insert data: %w", err)
}
}
return nil
@@ -408,3 +445,84 @@ func (db *DB) Info() (*DbInfo, error) {
return info, err
}
func (db *DB) Find(attr *DbAttr) (DbEntries, error) {
if err := db.Open(); err != nil {
return nil, err
}
defer db.Close()
var entries DbEntries
var filter *regexp.Regexp
if len(attr.Args) > 0 {
filter = regexp.MustCompile(attr.Args[0])
}
err := db.DB.View(func(tx *bolt.Tx) error {
root := tx.Bucket([]byte(db.Bucket))
if root == nil {
return nil
}
bucket := root.Bucket([]byte("meta"))
if bucket == nil {
return nil
}
databucket := root.Bucket([]byte("data"))
if databucket == nil {
return fmt.Errorf("failed to retrieve data sub bucket")
}
err := bucket.ForEach(func(key, pbentry []byte) error {
var entry DbEntry
if err := proto.Unmarshal(pbentry, &entry); err != nil {
return fmt.Errorf("failed to unmarshal from protobuf: %w", err)
}
entry.Value = databucket.Get([]byte(entry.Key))
var include bool
switch {
case filter != nil:
if filter.MatchString(entry.Key) ||
filter.MatchString(strings.Join(entry.Tags, " ")) {
include = true
}
if !entry.Binary && !include {
if filter.MatchString(string(entry.Value)) {
include = true
}
}
case len(attr.Tags) > 0:
for _, search := range attr.Tags {
for _, tag := range entry.Tags {
if tag == search {
include = true
break
}
}
if include {
break
}
}
default:
include = true
}
if include {
entries = append(entries, entry)
}
return nil
})
return err
})
return entries, err
}

View File

@@ -1,3 +1,5 @@
// -*-c++-*-
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.36.1
@@ -25,12 +27,13 @@ type DbEntry struct {
state protoimpl.MessageState `protogen:"open.v1"`
Id string `protobuf:"bytes,1,opt,name=Id,proto3" json:"Id,omitempty"`
Key string `protobuf:"bytes,2,opt,name=Key,proto3" json:"Key,omitempty"`
Value string `protobuf:"bytes,3,opt,name=Value,proto3" json:"Value,omitempty"`
Bin []byte `protobuf:"bytes,4,opt,name=Bin,proto3" json:"Bin,omitempty"`
Tags []string `protobuf:"bytes,5,rep,name=Tags,proto3" json:"Tags,omitempty"`
Created *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=Created,proto3" json:"Created,omitempty"`
Size uint64 `protobuf:"varint,7,opt,name=Size,proto3" json:"Size,omitempty"`
Encrypted bool `protobuf:"varint,8,opt,name=Encrypted,proto3" json:"Encrypted,omitempty"`
Preview string `protobuf:"bytes,3,opt,name=Preview,proto3" json:"Preview,omitempty"`
Tags []string `protobuf:"bytes,4,rep,name=Tags,proto3" json:"Tags,omitempty"`
Created *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=Created,proto3" json:"Created,omitempty"`
Size uint64 `protobuf:"varint,6,opt,name=Size,proto3" json:"Size,omitempty"`
Encrypted bool `protobuf:"varint,7,opt,name=Encrypted,proto3" json:"Encrypted,omitempty"`
Binary bool `protobuf:"varint,8,opt,name=Binary,proto3" json:"Binary,omitempty"`
Value []byte `protobuf:"bytes,9,opt,name=Value,proto3" json:"Value,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -79,20 +82,13 @@ func (x *DbEntry) GetKey() string {
return ""
}
func (x *DbEntry) GetValue() string {
func (x *DbEntry) GetPreview() string {
if x != nil {
return x.Value
return x.Preview
}
return ""
}
func (x *DbEntry) GetBin() []byte {
if x != nil {
return x.Bin
}
return nil
}
func (x *DbEntry) GetTags() []string {
if x != nil {
return x.Tags
@@ -121,26 +117,42 @@ func (x *DbEntry) GetEncrypted() bool {
return false
}
func (x *DbEntry) GetBinary() bool {
if x != nil {
return x.Binary
}
return false
}
func (x *DbEntry) GetValue() []byte {
if x != nil {
return x.Value
}
return nil
}
var File_app_dbentry_proto protoreflect.FileDescriptor
var file_app_dbentry_proto_rawDesc = []byte{
0x0a, 0x11, 0x61, 0x70, 0x70, 0x2f, 0x64, 0x62, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x12, 0x03, 0x61, 0x70, 0x70, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74,
0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xcf, 0x01, 0x0a, 0x07, 0x44, 0x62,
0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xef, 0x01, 0x0a, 0x07, 0x44, 0x62,
0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
0x09, 0x52, 0x02, 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01,
0x28, 0x09, 0x52, 0x03, 0x4b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65,
0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x10, 0x0a,
0x03, 0x42, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x42, 0x69, 0x6e, 0x12,
0x12, 0x0a, 0x04, 0x54, 0x61, 0x67, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x54,
0x61, 0x67, 0x73, 0x12, 0x34, 0x0a, 0x07, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x18, 0x06,
0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70,
0x52, 0x07, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x53, 0x69, 0x7a,
0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1c, 0x0a,
0x09, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08,
0x52, 0x09, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x42, 0x1e, 0x5a, 0x1c, 0x67,
0x28, 0x09, 0x52, 0x03, 0x4b, 0x65, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x50, 0x72, 0x65, 0x76, 0x69,
0x65, 0x77, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x50, 0x72, 0x65, 0x76, 0x69, 0x65,
0x77, 0x12, 0x12, 0x0a, 0x04, 0x54, 0x61, 0x67, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52,
0x04, 0x54, 0x61, 0x67, 0x73, 0x12, 0x34, 0x0a, 0x07, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64,
0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61,
0x6d, 0x70, 0x52, 0x07, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x53,
0x69, 0x7a, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x12,
0x1c, 0x0a, 0x09, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x18, 0x07, 0x20, 0x01,
0x28, 0x08, 0x52, 0x09, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x12, 0x16, 0x0a,
0x06, 0x42, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x42,
0x69, 0x6e, 0x61, 0x72, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x09,
0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x1e, 0x5a, 0x1c, 0x67,
0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x6c, 0x69, 0x6e, 0x64, 0x65,
0x6e, 0x2f, 0x61, 0x6e, 0x79, 0x64, 0x62, 0x2f, 0x61, 0x70, 0x70, 0x62, 0x06, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x33,

View File

@@ -1,3 +1,5 @@
// -*-c++-*-
syntax = "proto3";
package app;
@@ -8,10 +10,11 @@ option go_package = "github.com/tlinden/anydb/app";
message DbEntry {
string Id = 1;
string Key = 2;
string Value = 3;
bytes Bin = 4;
repeated string Tags = 5;
google.protobuf.Timestamp Created = 6;
uint64 Size = 7;
bool Encrypted = 8;
string Preview = 3;
repeated string Tags = 4;
google.protobuf.Timestamp Created = 5;
uint64 Size = 6;
bool Encrypted = 7;
bool Binary = 8;
bytes Value = 9;
}