restructured data storage, values now have their own sub bucket

TODO.md

@@ -33,3 +33,5 @@ behind.
 However, maybe change the list command to just list everything and add
 an extra find command for fulltext or tag search. Maybe still provide
 filter options in list command but only filter for keys.
+
+DONE: most of the above, except the tag stuff. manpage needs update and tests.

app/attr.go

@@ -20,17 +20,19 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"strings"
 	"unicode/utf8"
 )
 
 type DbAttr struct {
 	Key       string
-	Val       string
+	Preview   string
-	Bin       []byte
+	Val       []byte
 	Args      []string
 	Tags      []string
 	File      string
 	Encrypted bool
+	Binary    bool
 }
 
 func (attr *DbAttr) ParseKV() error {
@@ -43,7 +45,7 @@ func (attr *DbAttr) ParseKV() error {
 		}
 	case 2:
 		attr.Key = attr.Args[0]
-		attr.Val = attr.Args[1]
+		attr.Val = []byte(attr.Args[1])
 
 		if attr.Args[1] == "-" {
 			attr.File = "-"
@@ -51,7 +53,29 @@ func (attr *DbAttr) ParseKV() error {
 	}
 
 	if attr.File != "" {
-		return attr.GetFileValue()
+		if err := attr.GetFileValue(); err != nil {
+			return err
+		}
+	}
+
+	if attr.Binary {
+		attr.Preview = "<encrypted-content>"
+	} else {
+		if len(attr.Val) > MaxValueWidth {
+			attr.Preview = string(attr.Val)[0:MaxValueWidth] + "..."
+
+			if strings.Contains(attr.Preview, "\n") {
+				parts := strings.Split(attr.Preview, "\n")
+				if len(parts) > 0 {
+					attr.Preview = parts[0]
+				}
+			}
+		} else {
+			attr.Preview = string(attr.Val)
+		}
+	}
+	if attr.Encrypted {
+		attr.Preview = "<encrypted-content>"
 	}
 
 	return nil
@@ -82,11 +106,12 @@ func (attr *DbAttr) GetFileValue() error {
 		}
 
 		// poor man's text file test
-		sdata := string(data)
+		attr.Val = data
-		if utf8.ValidString(sdata) {
+
-			attr.Val = sdata
+		if utf8.ValidString(string(data)) {
+			attr.Binary = false
 		} else {
-			attr.Bin = data
+			attr.Binary = true
 		}
 	} else {
 		// read from console stdin
@@ -101,7 +126,7 @@ func (attr *DbAttr) GetFileValue() error {
 			data += input + "\n"
 		}
 
-		attr.Val = data
+		attr.Val = []byte(data)
 	}
 
 	return nil

app/crypto.go

@@ -18,7 +18,6 @@ package app
 
 import (
 	"crypto/rand"
-	"encoding/base64"
 	"errors"
 	"fmt"
 	"os"
@@ -104,7 +103,7 @@ func GetRandom(size int, capacity int) ([]byte, error) {
 // modifying it.
 //
 // The cipher text consists of:
-// base64(password-salt) + base64(12 byte nonce + ciphertext + 16 byte mac)
+// password-salt) + (12 byte nonce + ciphertext + 16 byte mac)
 func Encrypt(pass []byte, attr *DbAttr) error {
 	key, err := DeriveKey(pass, nil)
 	if err != nil {
@@ -116,25 +115,17 @@ func Encrypt(pass []byte, attr *DbAttr) error {
 		return fmt.Errorf("failed to create AEAD cipher: %w", err)
 	}
 
-	var plain []byte
+	total := aead.NonceSize() + len(attr.Val) + aead.Overhead()
-	if attr.Val != "" {
-		plain = []byte(attr.Val)
-	} else {
-		plain = attr.Bin
-	}
-
-	total := aead.NonceSize() + len(plain) + aead.Overhead()
 
 	nonce, err := GetRandom(aead.NonceSize(), total)
 	if err != nil {
 		return err
 	}
 
-	cipher := aead.Seal(nonce, nonce, plain, nil)
+	cipher := aead.Seal(nonce, nonce, attr.Val, nil)
 
-	attr.Bin = nil
+	attr.Val = append(attr.Val, key.Salt...)
-	attr.Val = base64.RawStdEncoding.EncodeToString(key.Salt) +
+	attr.Val = append(attr.Val, cipher...)
-		base64.RawStdEncoding.EncodeToString(cipher)
 
 	attr.Encrypted = true
 
@@ -142,21 +133,17 @@ func Encrypt(pass []byte, attr *DbAttr) error {
 }
 
 // Do the reverse
-func Decrypt(pass []byte, cipherb64 string) ([]byte, error) {
+func Decrypt(pass []byte, cipherb []byte) ([]byte, error) {
-	salt, err := base64.RawStdEncoding.Strict().DecodeString(cipherb64[0:B64SaltLen])
+	if len(cipherb) < B64SaltLen {
-	if err != nil {
+		return nil, fmt.Errorf("encrypted cipher block too small")
-		return nil, fmt.Errorf("failed to encode to base64: %w", err)
 	}
 
-	key, err := DeriveKey(pass, salt)
+	key, err := DeriveKey(pass, cipherb[0:B64SaltLen])
 	if err != nil {
 		return nil, err
 	}
 
-	cipher, err := base64.RawStdEncoding.Strict().DecodeString(cipherb64[B64SaltLen:])
+	cipher := cipherb[B64SaltLen:]
-	if err != nil {
-		return nil, fmt.Errorf("failed to encode to base64: %w", err)
-	}
 
 	aead, err := chacha20poly1305.New(key.Key)
 	if err != nil {

app/db.go

@@ -53,32 +53,6 @@ type DbInfo struct {
 	Path    string
 }
 
-// Post process an entry for list output.
-// Do NOT call it during write processing!
-func (entry *DbEntry) Normalize() {
-	entry.Size = uint64(len(entry.Value))
-
-	if entry.Encrypted {
-		entry.Value = "<encrypted-content>"
-	}
-
-	if len(entry.Bin) > 0 {
-		entry.Value = "<binary-content>"
-		entry.Size = uint64(len(entry.Bin))
-	}
-
-	if strings.Contains(entry.Value, "\n") {
-		parts := strings.Split(entry.Value, "\n")
-		if len(parts) > 0 {
-			entry.Value = parts[0]
-		}
-	}
-
-	if len(entry.Value) > MaxValueWidth {
-		entry.Value = entry.Value[0:MaxValueWidth] + "..."
-	}
-}
-
 type DbEntries []DbEntry
 
 type DbTag struct {
@@ -126,7 +100,12 @@ func (db *DB) List(attr *DbAttr) (DbEntries, error) {
 
 	err := db.DB.View(func(tx *bolt.Tx) error {
 
-		bucket := tx.Bucket([]byte(db.Bucket))
+		root := tx.Bucket([]byte(db.Bucket))
+		if root == nil {
+			return nil
+		}
+
+		bucket := root.Bucket([]byte("meta"))
 		if bucket == nil {
 			return nil
 		}
@@ -141,8 +120,7 @@ func (db *DB) List(attr *DbAttr) (DbEntries, error) {
 
 			switch {
 			case filter != nil:
-				if filter.MatchString(entry.Value) ||
+				if filter.MatchString(entry.Key) ||
-					filter.MatchString(entry.Key) ||
 					filter.MatchString(strings.Join(entry.Tags, " ")) {
 					include = true
 				}
@@ -183,18 +161,24 @@ func (db *DB) Set(attr *DbAttr) error {
 
 	entry := DbEntry{
 		Key:       attr.Key,
-		Value:     attr.Val,
+		Binary:    attr.Binary,
-		Bin:       attr.Bin,
 		Tags:      attr.Tags,
 		Encrypted: attr.Encrypted,
 		Created:   timestamppb.Now(),
+		Size:      uint64(len(attr.Val)),
+		Preview:   attr.Preview,
 	}
 
 	// check if the  entry already exists and if yes,  check if it has
 	// any  tags. if so,  we initialize  our update struct  with these
 	// tags unless it has new tags configured.
 	err := db.DB.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(db.Bucket))
+		root := tx.Bucket([]byte(db.Bucket))
+		if root == nil {
+			return nil
+		}
+
+		bucket := root.Bucket([]byte("meta"))
 		if bucket == nil {
 			return nil
 		}
@@ -221,23 +205,43 @@ func (db *DB) Set(attr *DbAttr) error {
 		return err
 	}
 
+	// marshall our data
+	pbentry, err := proto.Marshal(&entry)
+	if err != nil {
+		return fmt.Errorf("failed to marshall protobuf: %w", err)
+	}
+
 	err = db.DB.Update(func(tx *bolt.Tx) error {
-		// insert data
+		// create root bucket
-		bucket, err := tx.CreateBucketIfNotExists([]byte(db.Bucket))
+		root, err := tx.CreateBucketIfNotExists([]byte(db.Bucket))
 		if err != nil {
 			return fmt.Errorf("failed to create DB bucket: %w", err)
 		}
 
-		pbentry, err := proto.Marshal(&entry)
+		// create meta bucket
+		bucket, err := root.CreateBucketIfNotExists([]byte("meta"))
 		if err != nil {
-			return fmt.Errorf("failed to marshall protobuf: %w", err)
+			return fmt.Errorf("failed to create DB meta sub bucket: %w", err)
 		}
 
+		// write meta data
 		err = bucket.Put([]byte(entry.Key), []byte(pbentry))
 		if err != nil {
 			return fmt.Errorf("failed to insert data: %w", err)
 		}
 
+		// create data bucket
+		databucket, err := root.CreateBucketIfNotExists([]byte("data"))
+		if err != nil {
+			return fmt.Errorf("failed to create DB data sub bucket: %w", err)
+		}
+
+		// write value
+		err = databucket.Put([]byte(entry.Key), attr.Val)
+		if err != nil {
+			return fmt.Errorf("failed to insert data: %w", err)
+		}
+
 		return nil
 	})
 
@@ -257,21 +261,35 @@ func (db *DB) Get(attr *DbAttr) (*DbEntry, error) {
 	entry := DbEntry{}
 
 	err := db.DB.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(db.Bucket))
+		root := tx.Bucket([]byte(db.Bucket))
+		if root == nil {
+			return nil
+		}
+
+		bucket := root.Bucket([]byte("meta"))
 		if bucket == nil {
 			return nil
 		}
 
 		pbentry := bucket.Get([]byte(attr.Key))
 		if pbentry == nil {
-			// FIXME: shall we return a key not found error?
+			return fmt.Errorf("no such key: %s", attr.Key)
-			return nil
 		}
 
 		if err := proto.Unmarshal(pbentry, &entry); err != nil {
 			return fmt.Errorf("failed to unmarshal from protobuf: %w", err)
 		}
 
+		databucket := root.Bucket([]byte("data"))
+		if databucket == nil {
+			return fmt.Errorf("failed to retrieve data sub bucket")
+		}
+
+		entry.Value = databucket.Get([]byte(attr.Key))
+		if len(entry.Value) == 0 {
+			return fmt.Errorf("no such key: %s", attr.Key)
+		}
+
 		return nil
 	})
 
@@ -308,7 +326,7 @@ func (db *DB) Import(attr *DbAttr) (string, error) {
 		return "", err
 	}
 
-	if attr.Val == "" {
+	if len(attr.Val) == 0 {
 		return "", errors.New("empty json file")
 	}
 
@@ -336,10 +354,16 @@ func (db *DB) Import(attr *DbAttr) (string, error) {
 	defer db.Close()
 
 	err := db.DB.Update(func(tx *bolt.Tx) error {
-		// insert data
+		// create root bucket
-		bucket, err := tx.CreateBucketIfNotExists([]byte(db.Bucket))
+		root, err := tx.CreateBucketIfNotExists([]byte(db.Bucket))
 		if err != nil {
-			return fmt.Errorf("failed to create bucket: %w", err)
+			return fmt.Errorf("failed to create DB bucket: %w", err)
+		}
+
+		// create meta bucket
+		bucket, err := root.CreateBucketIfNotExists([]byte("meta"))
+		if err != nil {
+			return fmt.Errorf("failed to create DB meta sub bucket: %w", err)
 		}
 
 		for _, entry := range entries {
@@ -348,10 +372,23 @@ func (db *DB) Import(attr *DbAttr) (string, error) {
 				return fmt.Errorf("failed to marshall protobuf: %w", err)
 			}
 
+			// write meta data
 			err = bucket.Put([]byte(entry.Key), []byte(pbentry))
 			if err != nil {
 				return fmt.Errorf("failed to insert data into DB: %w", err)
 			}
+
+			// create data bucket
+			databucket, err := root.CreateBucketIfNotExists([]byte("data"))
+			if err != nil {
+				return fmt.Errorf("failed to create DB data sub bucket: %w", err)
+			}
+
+			// write value
+			err = databucket.Put([]byte(entry.Key), entry.Value)
+			if err != nil {
+				return fmt.Errorf("failed to insert data: %w", err)
+			}
 		}
 
 		return nil
@@ -408,3 +445,84 @@ func (db *DB) Info() (*DbInfo, error) {
 
 	return info, err
 }
+
+func (db *DB) Find(attr *DbAttr) (DbEntries, error) {
+	if err := db.Open(); err != nil {
+		return nil, err
+	}
+	defer db.Close()
+
+	var entries DbEntries
+	var filter *regexp.Regexp
+
+	if len(attr.Args) > 0 {
+		filter = regexp.MustCompile(attr.Args[0])
+	}
+
+	err := db.DB.View(func(tx *bolt.Tx) error {
+
+		root := tx.Bucket([]byte(db.Bucket))
+		if root == nil {
+			return nil
+		}
+
+		bucket := root.Bucket([]byte("meta"))
+		if bucket == nil {
+			return nil
+		}
+
+		databucket := root.Bucket([]byte("data"))
+		if databucket == nil {
+			return fmt.Errorf("failed to retrieve data sub bucket")
+		}
+
+		err := bucket.ForEach(func(key, pbentry []byte) error {
+			var entry DbEntry
+			if err := proto.Unmarshal(pbentry, &entry); err != nil {
+				return fmt.Errorf("failed to unmarshal from protobuf: %w", err)
+			}
+
+			entry.Value = databucket.Get([]byte(entry.Key))
+
+			var include bool
+
+			switch {
+			case filter != nil:
+				if filter.MatchString(entry.Key) ||
+					filter.MatchString(strings.Join(entry.Tags, " ")) {
+					include = true
+				}
+
+				if !entry.Binary && !include {
+					if filter.MatchString(string(entry.Value)) {
+						include = true
+					}
+				}
+			case len(attr.Tags) > 0:
+				for _, search := range attr.Tags {
+					for _, tag := range entry.Tags {
+						if tag == search {
+							include = true
+							break
+						}
+					}
+
+					if include {
+						break
+					}
+				}
+			default:
+				include = true
+			}
+
+			if include {
+				entries = append(entries, entry)
+			}
+
+			return nil
+		})
+
+		return err
+	})
+	return entries, err
+}

app/dbentry.pb.go

@@ -1,3 +1,5 @@
+// -*-c++-*-
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.36.1
@@ -25,12 +27,13 @@ type DbEntry struct {
 	state         protoimpl.MessageState `protogen:"open.v1"`
 	Id            string                 `protobuf:"bytes,1,opt,name=Id,proto3" json:"Id,omitempty"`
 	Key           string                 `protobuf:"bytes,2,opt,name=Key,proto3" json:"Key,omitempty"`
-	Value         string                 `protobuf:"bytes,3,opt,name=Value,proto3" json:"Value,omitempty"`
+	Preview       string                 `protobuf:"bytes,3,opt,name=Preview,proto3" json:"Preview,omitempty"`
-	Bin           []byte                 `protobuf:"bytes,4,opt,name=Bin,proto3" json:"Bin,omitempty"`
+	Tags          []string               `protobuf:"bytes,4,rep,name=Tags,proto3" json:"Tags,omitempty"`
-	Tags          []string               `protobuf:"bytes,5,rep,name=Tags,proto3" json:"Tags,omitempty"`
+	Created       *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=Created,proto3" json:"Created,omitempty"`
-	Created       *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=Created,proto3" json:"Created,omitempty"`
+	Size          uint64                 `protobuf:"varint,6,opt,name=Size,proto3" json:"Size,omitempty"`
-	Size          uint64                 `protobuf:"varint,7,opt,name=Size,proto3" json:"Size,omitempty"`
+	Encrypted     bool                   `protobuf:"varint,7,opt,name=Encrypted,proto3" json:"Encrypted,omitempty"`
-	Encrypted     bool                   `protobuf:"varint,8,opt,name=Encrypted,proto3" json:"Encrypted,omitempty"`
+	Binary        bool                   `protobuf:"varint,8,opt,name=Binary,proto3" json:"Binary,omitempty"`
+	Value         []byte                 `protobuf:"bytes,9,opt,name=Value,proto3" json:"Value,omitempty"`
 	unknownFields protoimpl.UnknownFields
 	sizeCache     protoimpl.SizeCache
 }
@@ -79,20 +82,13 @@ func (x *DbEntry) GetKey() string {
 	return ""
 }
 
-func (x *DbEntry) GetValue() string {
+func (x *DbEntry) GetPreview() string {
 	if x != nil {
-		return x.Value
+		return x.Preview
 	}
 	return ""
 }
 
-func (x *DbEntry) GetBin() []byte {
-	if x != nil {
-		return x.Bin
-	}
-	return nil
-}
-
 func (x *DbEntry) GetTags() []string {
 	if x != nil {
 		return x.Tags
@@ -121,26 +117,42 @@ func (x *DbEntry) GetEncrypted() bool {
 	return false
 }
 
+func (x *DbEntry) GetBinary() bool {
+	if x != nil {
+		return x.Binary
+	}
+	return false
+}
+
+func (x *DbEntry) GetValue() []byte {
+	if x != nil {
+		return x.Value
+	}
+	return nil
+}
+
 var File_app_dbentry_proto protoreflect.FileDescriptor
 
 var file_app_dbentry_proto_rawDesc = []byte{
 	0x0a, 0x11, 0x61, 0x70, 0x70, 0x2f, 0x64, 0x62, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72,
 	0x6f, 0x74, 0x6f, 0x12, 0x03, 0x61, 0x70, 0x70, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
 	0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74,
-	0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xcf, 0x01, 0x0a, 0x07, 0x44, 0x62,
+	0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xef, 0x01, 0x0a, 0x07, 0x44, 0x62,
 	0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
 	0x09, 0x52, 0x02, 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01,
-	0x28, 0x09, 0x52, 0x03, 0x4b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65,
+	0x28, 0x09, 0x52, 0x03, 0x4b, 0x65, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x50, 0x72, 0x65, 0x76, 0x69,
-	0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x10, 0x0a,
+	0x65, 0x77, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x50, 0x72, 0x65, 0x76, 0x69, 0x65,
-	0x03, 0x42, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x42, 0x69, 0x6e, 0x12,
+	0x77, 0x12, 0x12, 0x0a, 0x04, 0x54, 0x61, 0x67, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52,
-	0x12, 0x0a, 0x04, 0x54, 0x61, 0x67, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x54,
+	0x04, 0x54, 0x61, 0x67, 0x73, 0x12, 0x34, 0x0a, 0x07, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64,
-	0x61, 0x67, 0x73, 0x12, 0x34, 0x0a, 0x07, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x18, 0x06,
+	0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e,
-	0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72,
+	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61,
-	0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70,
+	0x6d, 0x70, 0x52, 0x07, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x53,
-	0x52, 0x07, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x53, 0x69, 0x7a,
+	0x69, 0x7a, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x12,
-	0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1c, 0x0a,
+	0x1c, 0x0a, 0x09, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x18, 0x07, 0x20, 0x01,
-	0x09, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08,
+	0x28, 0x08, 0x52, 0x09, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x12, 0x16, 0x0a,
-	0x52, 0x09, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x42, 0x1e, 0x5a, 0x1c, 0x67,
+	0x06, 0x42, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x42,
+	0x69, 0x6e, 0x61, 0x72, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x09,
+	0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x1e, 0x5a, 0x1c, 0x67,
 	0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x6c, 0x69, 0x6e, 0x64, 0x65,
 	0x6e, 0x2f, 0x61, 0x6e, 0x79, 0x64, 0x62, 0x2f, 0x61, 0x70, 0x70, 0x62, 0x06, 0x70, 0x72, 0x6f,
 	0x74, 0x6f, 0x33,

app/dbentry.proto

@@ -1,3 +1,5 @@
+// -*-c++-*-
+
 syntax = "proto3";
 package app;
 
@@ -8,10 +10,11 @@ option go_package = "github.com/tlinden/anydb/app";
 message DbEntry {
   string Id = 1;
   string Key = 2;
-  string Value = 3;
+  string Preview = 3;
-  bytes Bin = 4;
+  repeated string Tags = 4;
-  repeated string Tags = 5;
+  google.protobuf.Timestamp Created = 5;
-  google.protobuf.Timestamp Created = 6;
+  uint64 Size = 6;
-  uint64 Size = 7;
+  bool Encrypted = 7;
-  bool Encrypted = 8;
+  bool Binary = 8;
+  bytes Value = 9;
 }

cfg/config.go

@@ -26,7 +26,7 @@ import (
 	"github.com/tlinden/anydb/common"
 )
 
-var Version string = "v0.0.7"
+var Version string = "v0.1.0"
 
 type BucketConfig struct {
 	Encrypt bool

cmd/crud.go

@@ -20,7 +20,6 @@ import (
 	"errors"
 	"os"
 	"strings"
-	"unicode/utf8"
 
 	"github.com/spf13/cobra"
 	"github.com/tlinden/anydb/app"
@@ -124,12 +123,7 @@ func Get(conf *cfg.Config) *cobra.Command {
 					return err
 				}
 
-				if utf8.ValidString(string(clear)) {
+				entry.Value = clear
-					entry.Value = string(clear)
-				} else {
-					entry.Bin = clear
-				}
-
 				entry.Encrypted = false
 			}
 
@@ -188,7 +182,7 @@ func List(conf *cfg.Config) *cobra.Command {
 	)
 
 	var cmd = &cobra.Command{
-		Use:   "list  [<filter-regex>] [-t <tag>] [-m <mode>] [-n -N] [-T <tpl>] [-i]",
+		Use:   "list  [<filter-regex>] [-m <mode>] [-n -N] [-T <tpl>] [-i]",
 		Short: "List database contents",
 		Long:  `List database contents`,
 		RunE: func(cmd *cobra.Command, args []string) error {
@@ -221,6 +215,58 @@ func List(conf *cfg.Config) *cobra.Command {
 		},
 	}
 
+	cmd.PersistentFlags().StringVarP(&conf.Mode, "mode", "m", "", "output format (table|wide|json|template), wide is a verbose table. (default 'table')")
+	cmd.PersistentFlags().StringVarP(&conf.Template, "template", "T", "", "go template for '-m template'")
+	cmd.PersistentFlags().BoolVarP(&wide, "wide-output", "l", false, "output mode: wide")
+	cmd.PersistentFlags().BoolVarP(&conf.NoHeaders, "no-headers", "n", false, "omit headers in tables")
+	cmd.PersistentFlags().BoolVarP(&conf.NoHumanize, "no-human", "N", false, "do not translate to human readable values")
+	cmd.PersistentFlags().BoolVarP(&conf.CaseInsensitive, "case-insensitive", "i", false, "filter case insensitive")
+
+	cmd.Aliases = append(cmd.Aliases, "ls")
+
+	return cmd
+}
+
+func Find(conf *cfg.Config) *cobra.Command {
+	var (
+		attr app.DbAttr
+		wide bool
+	)
+
+	var cmd = &cobra.Command{
+		Use:   "find <filter-regex> | -t <tag> [-m <mode>] [-n -N] [-T <tpl>] [-i]",
+		Short: "Find database contents",
+		Long:  `Find database contents`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			// errors at this stage do not cause the usage to be shown
+			cmd.SilenceUsage = true
+
+			if len(args) > 0 {
+				if conf.CaseInsensitive {
+					attr.Args = []string{"(?i)" + args[0]}
+				} else {
+					attr.Args = args
+				}
+			}
+
+			// turn comma list into slice, if needed
+			if len(attr.Tags) == 1 && strings.Contains(attr.Tags[0], ",") {
+				attr.Tags = strings.Split(attr.Tags[0], ",")
+			}
+
+			if wide {
+				conf.Mode = "wide"
+			}
+
+			entries, err := conf.DB.Find(&attr)
+			if err != nil {
+				return err
+			}
+
+			return output.List(os.Stdout, conf, entries)
+		},
+	}
+
 	cmd.PersistentFlags().StringVarP(&conf.Mode, "mode", "m", "", "output format (table|wide|json|template), wide is a verbose table. (default 'table')")
 	cmd.PersistentFlags().StringVarP(&conf.Template, "template", "T", "", "go template for '-m template'")
 	cmd.PersistentFlags().BoolVarP(&wide, "wide-output", "l", false, "output mode: wide")
@@ -230,7 +276,8 @@ func List(conf *cfg.Config) *cobra.Command {
 	cmd.PersistentFlags().StringArrayVarP(&attr.Tags, "tags", "t", nil, "tags, multiple allowed")
 
 	cmd.Aliases = append(cmd.Aliases, "/")
-	cmd.Aliases = append(cmd.Aliases, "ls")
+	cmd.Aliases = append(cmd.Aliases, "f")
+	cmd.Aliases = append(cmd.Aliases, "search")
 
 	return cmd
 }

cmd/extra.go

@@ -23,7 +23,6 @@ import (
 	"io"
 	"os"
 	"os/exec"
-	"unicode/utf8"
 
 	"github.com/spf13/cobra"
 	"github.com/tlinden/anydb/app"
@@ -199,7 +198,7 @@ func Edit(conf *cfg.Config) *cobra.Command {
 				return err
 			}
 
-			if len(entry.Value) == 0 && len(entry.Bin) > 0 {
+			if len(entry.Value) == 0 && entry.Binary {
 				return errors.New("key contains binary uneditable content")
 			}
 
@@ -216,12 +215,7 @@ func Edit(conf *cfg.Config) *cobra.Command {
 					return err
 				}
 
-				if utf8.ValidString(string(clear)) {
+				entry.Value = clear
-					entry.Value = string(clear)
-				} else {
-					entry.Bin = clear
-				}
-
 				entry.Encrypted = false
 			}
 
@@ -231,7 +225,7 @@ func Edit(conf *cfg.Config) *cobra.Command {
 			// save file to a temp file, call the editor with it, read
 			// it  back in and  compare the content with  the original
 			// one
-			newcontent, err := editContent(editor, entry.Value)
+			newcontent, err := editContent(editor, string(entry.Value))
 			if err != nil {
 				return err
 			}
@@ -241,7 +235,7 @@ func Edit(conf *cfg.Config) *cobra.Command {
 				Key:       attr.Key,
 				Tags:      attr.Tags,
 				Encrypted: attr.Encrypted,
-				Val:       newcontent,
+				Val:       []byte(newcontent),
 			}
 
 			// encrypt if needed

cmd/root.go

@@ -122,6 +122,7 @@ func Execute() {
 	// CRUD
 	rootCmd.AddCommand(Set(&conf))
 	rootCmd.AddCommand(List(&conf))
+	rootCmd.AddCommand(Find(&conf))
 	rootCmd.AddCommand(Get(&conf))
 	rootCmd.AddCommand(Del(&conf))
 

output/list.go

@@ -65,8 +65,6 @@ func ListTemplate(writer io.Writer, conf *cfg.Config, entries app.DbEntries) err
 	buf := bytes.Buffer{}
 
 	for _, row := range entries {
-		row.Normalize()
-
 		buf.Reset()
 		err = tmpl.Execute(&buf, row)
 		if err != nil {
@@ -94,8 +92,6 @@ func ListTable(writer io.Writer, conf *cfg.Config, entries app.DbEntries) error
 	}
 
 	for _, row := range entries {
-		row.Normalize()
-
 		if conf.Mode == "wide" {
 			switch conf.NoHumanize {
 			case true:
@@ -104,21 +100,20 @@ func ListTable(writer io.Writer, conf *cfg.Config, entries app.DbEntries) error
 					strings.Join(row.Tags, ","),
 					strconv.FormatUint(row.Size, 10),
 					row.Created.AsTime().Format("02.01.2006T03:04.05"),
-					row.Value,
+					row.Preview,
 				})
 			default:
 				table.Append([]string{
 					row.Key,
 					strings.Join(row.Tags, ","),
 					humanize.Bytes(uint64(row.Size)),
-					//row.Created.Format("02.01.2006T03:04.05"),
 					humanize.Time(row.Created.AsTime()),
-					row.Value,
+					row.Preview,
 				})
 			}
 
 		} else {
-			table.Append([]string{row.Key, row.Value})
+			table.Append([]string{row.Key, row.Preview})
 		}
 	}
 

output/single.go

@@ -22,7 +22,6 @@ import (
 	"io"
 	"os"
 	"reflect"
-	"strings"
 
 	"github.com/dustin/go-humanize"
 	"github.com/tlinden/anydb/app"
@@ -40,16 +39,15 @@ func Print(writer io.Writer, conf *cfg.Config, attr *app.DbAttr, entry *app.DbEn
 
 	switch conf.Mode {
 	case "simple", "":
-		if len(entry.Bin) > 0 {
+		if entry.Binary {
 			if isatty {
 				fmt.Println("binary data omitted")
 			} else {
-				os.Stdout.Write(entry.Bin)
+				os.Stdout.Write(entry.Value)
 			}
 		} else {
-			fmt.Print(entry.Value)
+			fmt.Print(string(entry.Value))
-
+			if entry.Value[entry.Size-1] != '\n' {
-			if !strings.HasSuffix(entry.Value, "\n") {
 				// always add a terminal newline
 				fmt.Println()
 			}
@@ -87,17 +85,14 @@ func WriteFile(writer io.Writer, conf *cfg.Config, attr *app.DbAttr, entry *app.
 		fileHandle = fd
 	}
 
-	if len(entry.Bin) > 0 {
+	if entry.Binary {
 		// binary file content
-		_, err = fileHandle.Write(entry.Bin)
+		_, err = fileHandle.Write(entry.Value)
-	} else {
-		val := entry.Value
-		if !strings.HasSuffix(val, "\n") {
-			// always add a terminal newline
-			val += "\n"
-		}
 
-		_, err = fileHandle.Write([]byte(val))
+		if entry.Value[entry.Size-1] != '\n' {
+			// always add a terminal newline
+			_, err = fileHandle.Write([]byte{'\n'})
+		}
 	}
 
 	if err != nil {