added bolt db support, +TODO list

README.md

@@ -1,2 +1,17 @@
 # up
 Simple standalone file upload server with api and cli
+
+## TODO
+
+- decouple db and http code in Runserver()
+- store ts
+- implement goroutine to expire after 1d, 10m etc
+- use bolt db to retrieve list of items to expire
+- return a meaningful  message if a file has expired,  not just a 404,
+  that is: do  remove the file when it expires  but not the associated
+  db entry.
+- also serve a html upload page
+- add api version in path
+- add auth options (access key, users, roles, oauth2)
+- add metrics
+- add upctl command to remove a file

upctl/upctl.hcl

@@ -1 +1 @@
-endpoint = "http://daemon.de/api"
+endpoint = "http://localhost:8080/api"

upd/cfg/config.go

@@ -31,6 +31,7 @@ type Config struct {
 	Listen     string
 	StorageDir string
 	Url        string
+	DbFile     string
 }
 
 func Getversion() string {

upd/cmd/root.go

@@ -86,6 +86,7 @@ func Execute() {
 	rootCmd.PersistentFlags().StringVarP(&conf.StorageDir, "storagedir", "s", "/tmp", "storage directory for uploaded files")
 	rootCmd.PersistentFlags().StringVarP(&conf.ApiPrefix, "apiprefix", "a", "/api", "API endpoint path")
 	rootCmd.PersistentFlags().StringVarP(&conf.Url, "url", "u", "", "HTTP endpoint w/o path")
+	rootCmd.PersistentFlags().StringVarP(&conf.DbFile, "dbfile", "D", "/tmp/uploads.db", "Bold database file to use")
 
 	err := rootCmd.Execute()
 	if err != nil {

upd/go.mod

@@ -32,9 +32,10 @@ require (
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/subosito/gotenv v1.4.2 // indirect
 	github.com/ugorji/go/codec v1.2.7 // indirect
+	go.etcd.io/bbolt v1.3.7 // indirect
 	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
 	golang.org/x/net v0.4.0 // indirect
-	golang.org/x/sys v0.3.0 // indirect
+	golang.org/x/sys v0.4.0 // indirect
 	golang.org/x/text v0.5.0 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect

upd/go.sum

@@ -219,6 +219,8 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
+go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -361,6 +363,8 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
+golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

upd/lib/server.go

@@ -22,8 +22,10 @@ import (
 	"fmt"
 	"github.com/gin-gonic/gin"
 	//"github.com/gin-gonic/gin/binding"
+	"encoding/json"
 	"github.com/google/uuid"
 	"github.com/tlinden/up/upd/cfg"
+	bolt "go.etcd.io/bbolt"
 	"io"
 	"net/http"
 	"os"
@@ -54,17 +56,30 @@ func NormalizeFilename(file string) string {
 	return Ts() + r.ReplaceAllString(file, "")
 }
 
-// Binding from JSON
+// Binding from JSON, data coming from user, not tainted
 type Meta struct {
 	Expire string `form:"expire"`
 }
 
+type DbEntry struct {
+	Id     string `json:"id"`
+	Expire string `json:"expire"`
+	File   string `json:"file"`
+}
+
 func Runserver(cfg *cfg.Config, args []string) error {
 	dst := cfg.StorageDir
 	router := gin.Default()
 	router.SetTrustedProxies(nil)
 	api := router.Group(cfg.ApiPrefix)
 
+	db, err := bolt.Open(cfg.DbFile, 0600, nil)
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+
+	// FIXME: put these beast into their own funcs!!!!!!!
 	{
 		api.POST("/putfile", func(c *gin.Context) {
 			// supports upload of multiple files with:
@@ -114,8 +129,11 @@ func Runserver(cfg *cfg.Config, args []string) error {
 			}
 
 			var returnUrl string
+			entry := &DbEntry{Id: id, Expire: formdata.Expire}
+
 			if len(uploaded) == 1 {
 				returnUrl = cfg.Url + cfg.ApiPrefix + "/getfile/" + id + "/" + uploaded[0]
+				entry.File = uploaded[0]
 			} else {
 				zipfile := Ts() + "data.zip"
 
@@ -130,6 +148,7 @@ func Runserver(cfg *cfg.Config, args []string) error {
 				}
 
 				returnUrl = strings.Join([]string{cfg.Url + cfg.ApiPrefix, "getfile", id, zipfile}, "/")
+				entry.File = zipfile
 
 				// clean up after us
 				go func() {
@@ -150,6 +169,33 @@ func Runserver(cfg *cfg.Config, args []string) error {
 
 			Log("Now serving %s from %s/%s", returnUrl, dst, id)
 			Log("Expire set to: %s", formdata.Expire)
+
+			go func() {
+				err := db.Update(func(tx *bolt.Tx) error {
+					b, err := tx.CreateBucketIfNotExists([]byte("uploads"))
+					if err != nil {
+						return fmt.Errorf("create bucket: %s", err)
+					}
+
+					jsonentry, err := json.Marshal(entry)
+					if err != nil {
+						return fmt.Errorf("json marshalling failure: %s", err)
+					}
+
+					err = b.Put([]byte(id), []byte(jsonentry))
+					if err != nil {
+						return fmt.Errorf("insert data: %s", err)
+					}
+
+					// results in:
+					// bbolt get /tmp/uploads.db uploads fb242922-86cb-43a8-92bc-b027c35f0586
+					// {"id":"fb242922-86cb-43a8-92bc-b027c35f0586","expire":"1d","file":"2023-02-17-13-09-data.zip"}
+					return nil
+				})
+				if err != nil {
+					Log("DB error: %s", err.Error())
+				}
+			}()
 		})
 
 		api.GET("/getfile/:id/:file", func(c *gin.Context) {