diff --git a/README.md b/README.md index 3f8d365..626abe1 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,17 @@ # up Simple standalone file upload server with api and cli + +## TODO + +- decouple db and http code in Runserver() +- store ts +- implement goroutine to expire after 1d, 10m etc +- use bolt db to retrieve list of items to expire +- return a meaningful message if a file has expired, not just a 404, + that is: do remove the file when it expires but not the associated + db entry. +- also serve a html upload page +- add api version in path +- add auth options (access key, users, roles, oauth2) +- add metrics +- add upctl command to remove a file diff --git a/upctl/upctl.hcl b/upctl/upctl.hcl index 838eb6a..7d177f8 100644 --- a/upctl/upctl.hcl +++ b/upctl/upctl.hcl @@ -1 +1 @@ -endpoint = "http://daemon.de/api" +endpoint = "http://localhost:8080/api" diff --git a/upd/cfg/config.go b/upd/cfg/config.go index e8db56c..dd2b69f 100644 --- a/upd/cfg/config.go +++ b/upd/cfg/config.go @@ -31,6 +31,7 @@ type Config struct { Listen string StorageDir string Url string + DbFile string } func Getversion() string { diff --git a/upd/cmd/root.go b/upd/cmd/root.go index 1599a22..033a13a 100644 --- a/upd/cmd/root.go +++ b/upd/cmd/root.go @@ -86,6 +86,7 @@ func Execute() { rootCmd.PersistentFlags().StringVarP(&conf.StorageDir, "storagedir", "s", "/tmp", "storage directory for uploaded files") rootCmd.PersistentFlags().StringVarP(&conf.ApiPrefix, "apiprefix", "a", "/api", "API endpoint path") rootCmd.PersistentFlags().StringVarP(&conf.Url, "url", "u", "", "HTTP endpoint w/o path") + rootCmd.PersistentFlags().StringVarP(&conf.DbFile, "dbfile", "D", "/tmp/uploads.db", "Bold database file to use") err := rootCmd.Execute() if err != nil { diff --git a/upd/go.mod b/upd/go.mod index 8a24a82..5b05b91 100644 --- a/upd/go.mod +++ b/upd/go.mod @@ -32,9 +32,10 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/subosito/gotenv v1.4.2 // indirect github.com/ugorji/go/codec v1.2.7 // indirect + go.etcd.io/bbolt v1.3.7 // indirect golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect golang.org/x/net v0.4.0 // indirect - golang.org/x/sys v0.3.0 // indirect + golang.org/x/sys v0.4.0 // indirect golang.org/x/text v0.5.0 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/upd/go.sum b/upd/go.sum index cf6e2f4..8af145c 100644 --- a/upd/go.sum +++ b/upd/go.sum @@ -219,6 +219,8 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= +go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -361,6 +363,8 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/upd/lib/server.go b/upd/lib/server.go index f98db1c..2bb9886 100644 --- a/upd/lib/server.go +++ b/upd/lib/server.go @@ -22,8 +22,10 @@ import ( "fmt" "github.com/gin-gonic/gin" //"github.com/gin-gonic/gin/binding" + "encoding/json" "github.com/google/uuid" "github.com/tlinden/up/upd/cfg" + bolt "go.etcd.io/bbolt" "io" "net/http" "os" @@ -54,17 +56,30 @@ func NormalizeFilename(file string) string { return Ts() + r.ReplaceAllString(file, "") } -// Binding from JSON +// Binding from JSON, data coming from user, not tainted type Meta struct { Expire string `form:"expire"` } +type DbEntry struct { + Id string `json:"id"` + Expire string `json:"expire"` + File string `json:"file"` +} + func Runserver(cfg *cfg.Config, args []string) error { dst := cfg.StorageDir router := gin.Default() router.SetTrustedProxies(nil) api := router.Group(cfg.ApiPrefix) + db, err := bolt.Open(cfg.DbFile, 0600, nil) + if err != nil { + return err + } + defer db.Close() + + // FIXME: put these beast into their own funcs!!!!!!! { api.POST("/putfile", func(c *gin.Context) { // supports upload of multiple files with: @@ -114,8 +129,11 @@ func Runserver(cfg *cfg.Config, args []string) error { } var returnUrl string + entry := &DbEntry{Id: id, Expire: formdata.Expire} + if len(uploaded) == 1 { returnUrl = cfg.Url + cfg.ApiPrefix + "/getfile/" + id + "/" + uploaded[0] + entry.File = uploaded[0] } else { zipfile := Ts() + "data.zip" @@ -130,6 +148,7 @@ func Runserver(cfg *cfg.Config, args []string) error { } returnUrl = strings.Join([]string{cfg.Url + cfg.ApiPrefix, "getfile", id, zipfile}, "/") + entry.File = zipfile // clean up after us go func() { @@ -150,6 +169,33 @@ func Runserver(cfg *cfg.Config, args []string) error { Log("Now serving %s from %s/%s", returnUrl, dst, id) Log("Expire set to: %s", formdata.Expire) + + go func() { + err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucketIfNotExists([]byte("uploads")) + if err != nil { + return fmt.Errorf("create bucket: %s", err) + } + + jsonentry, err := json.Marshal(entry) + if err != nil { + return fmt.Errorf("json marshalling failure: %s", err) + } + + err = b.Put([]byte(id), []byte(jsonentry)) + if err != nil { + return fmt.Errorf("insert data: %s", err) + } + + // results in: + // bbolt get /tmp/uploads.db uploads fb242922-86cb-43a8-92bc-b027c35f0586 + // {"id":"fb242922-86cb-43a8-92bc-b027c35f0586","expire":"1d","file":"2023-02-17-13-09-data.zip"} + return nil + }) + if err != nil { + Log("DB error: %s", err.Error()) + } + }() }) api.GET("/getfile/:id/:file", func(c *gin.Context) {