diff --git a/README.md b/README.md index 9a9023c..558fff0 100644 --- a/README.md +++ b/README.md @@ -7,14 +7,31 @@ Simple standalone file upload server with api and cli - store ts - implement goroutine to expire after 1d, 10m etc - use bolt db to retrieve list of items to expire -- return a meaningful message if a file has expired, not just a 404, - that is: do remove the file when it expires but not the associated - db entry. - also serve a html upload page - add auth options (access key, users, roles, oauth2) - add metrics - add upctl command to remove a file -- upd: add short uuid to files, in case multiple files with the same - name are being uploaded - use global map of api endpoints like /file/get/ etc -- use separate group for /file/ +- create cobra client commands (upload, list, delete, edit) + + + +## curl commands + +### upload + +``` +curl -X POST localhost:8080/api/putfile -F "upload[]=@xxx" -F "upload[]=@yyy" -H "Content-Type: multipart/form-data" +``` + +### download +``` +curl -O http://localhost:8080/api/v1/file/388f41f4-3f0d-41e1-a022-9132c0e9e16f/2023-02-28-18-33-xxx +``` + +### delete +``` +curl -X DELETE http://localhost:8080/api/v1/file/388f41f4-3f0d-41e1-a022-9132c0e9e16f/ +curl -X DELETE http://localhost:8080/api/v1/file/?id=388f41f4-3f0d-41e1-a022-9132c0e9e16f/ +curl -X DELETE -H "Accept: application/json" -d '{"id":"$id"}' http://localhost:8080/api/v1/file/ +``` diff --git a/upctl/cfg/config.go b/upctl/cfg/config.go index 4808b2e..6f58638 100644 --- a/upctl/cfg/config.go +++ b/upctl/cfg/config.go @@ -29,6 +29,7 @@ type Config struct { Endpoint string Debug bool Retries int + Expire string } func Getversion() string { diff --git a/upctl/cmd/root.go b/upctl/cmd/root.go index f3e0d6e..ca76778 100644 --- a/upctl/cmd/root.go +++ b/upctl/cmd/root.go @@ -87,7 +87,8 @@ func Execute() { rootCmd.PersistentFlags().BoolVarP(&conf.Debug, "debug", "d", false, "Enable debugging") rootCmd.PersistentFlags().StringVarP(&cfgFile, "config", "c", "", "custom config file") rootCmd.PersistentFlags().IntVarP(&conf.Retries, "retries", "r", 3, "How often shall we retry to access our endpoint") - rootCmd.PersistentFlags().StringVarP(&conf.Endpoint, "endpoint", "e", "http://localhost:8080/api", "upload api endpoint url") + rootCmd.PersistentFlags().StringVarP(&conf.Endpoint, "endpoint", "p", "http://localhost:8080/api", "upload api endpoint url") + rootCmd.PersistentFlags().StringVarP(&conf.Expire, "expire", "e", "asap", "Expire setting: asap or duration (accepted shortcuts: dmh)") err := rootCmd.Execute() if err != nil { diff --git a/upctl/lib/client.go b/upctl/lib/client.go index b0bf3df..c86766b 100644 --- a/upctl/lib/client.go +++ b/upctl/lib/client.go @@ -44,7 +44,7 @@ func Runclient(c *cfg.Config, args []string) error { client.SetUserAgent("upctl-" + cfg.VERSION) - url := c.Endpoint + ApiVersion + "/file/put" + url := c.Endpoint + ApiVersion + "/file/" rq := client.R() for _, file := range args { @@ -92,7 +92,7 @@ func Runclient(c *cfg.Config, args []string) error { resp, err := rq. SetFormData(map[string]string{ - "expire": "1d", + "expire": c.Expire, }). SetUploadCallbackWithInterval(func(info req.UploadInfo) { fmt.Printf("\r%q uploaded %.2f%%", info.FileName, float64(info.UploadedSize)/float64(info.FileSize)*100.0) diff --git a/upd/api/api.go b/upd/api/common.go similarity index 100% rename from upd/api/api.go rename to upd/api/common.go diff --git a/upd/api/db.go b/upd/api/db.go new file mode 100644 index 0000000..f25c498 --- /dev/null +++ b/upd/api/db.go @@ -0,0 +1,101 @@ +/* +Copyright © 2023 Thomas von Dein + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package api + +import ( + "encoding/json" + "fmt" + + bolt "go.etcd.io/bbolt" +) + +const Bucket string = "uploads" + +func DbInsert(db *bolt.DB, id string, entry *Upload) { + err := db.Update(func(tx *bolt.Tx) error { + bucket, err := tx.CreateBucketIfNotExists([]byte(Bucket)) + if err != nil { + return fmt.Errorf("create bucket: %s", err) + } + + jsonentry, err := json.Marshal(entry) + if err != nil { + return fmt.Errorf("json marshalling failure: %s", err) + } + + err = bucket.Put([]byte(id), []byte(jsonentry)) + if err != nil { + return fmt.Errorf("insert data: %s", err) + } + + // results in: + // bbolt get /tmp/uploads.db uploads fb242922-86cb-43a8-92bc-b027c35f0586 + // {"id":"fb242922-86cb-43a8-92bc-b027c35f0586","expire":"1d","file":"2023-02-17-13-09-data.zip"} + return nil + }) + if err != nil { + Log("DB error: %s", err.Error()) + } +} + +func DbLookupId(db *bolt.DB, id string) (Upload, error) { + var upload Upload + + err := db.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(Bucket)) + j := bucket.Get([]byte(id)) + + if len(j) == 0 { + return fmt.Errorf("id %s not found", id) + } + + if err := json.Unmarshal(j, &upload); err != nil { + return fmt.Errorf("unable to unmarshal json: %s", err) + } + + return nil + }) + + if err != nil { + Log("DB error: %s", err.Error()) + return upload, err + } + + return upload, nil +} + +func DbDeleteId(db *bolt.DB, id string) error { + err := db.Update(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(Bucket)) + + j := bucket.Get([]byte(id)) + + if len(j) == 0 { + return fmt.Errorf("id %s not found", id) + } + + err := bucket.Delete([]byte(id)) + return err + }) + + if err != nil { + Log("DB error: %s", err.Error()) + } + + return err +} diff --git a/upd/api/fileio.go b/upd/api/fileio.go index b3eb1a5..d2068e7 100644 --- a/upd/api/fileio.go +++ b/upd/api/fileio.go @@ -19,6 +19,7 @@ package api import ( "archive/zip" + "errors" "io" "os" "path/filepath" @@ -26,15 +27,12 @@ import ( ) // cleanup an upload directory, either because we got an error in the -// middle of an upload or something else went wrong. we fork off a go -// routine because this may block. +// middle of an upload or something else went wrong. func cleanup(dir string) { - go func() { - err := os.RemoveAll(dir) - if err != nil { - Log("Failed to remove dir %s: %s", dir, err.Error()) - } - }() + err := os.RemoveAll(dir) + if err != nil { + Log("Failed to remove dir %s: %s", dir, err.Error()) + } } func ZipSource(source, target string) error { @@ -60,7 +58,10 @@ func ZipSource(source, target string) error { newDir, err := os.Getwd() if err != nil { } - //Log("Current Working Direcotry: %s\n, source: %s", newDir, source) + if newDir != source { + err = errors.New("Failed to changedir!") + return + } err = filepath.Walk(".", func(path string, info os.FileInfo, err error) error { // 2. Go through all the files of the source diff --git a/upd/api/handlers.go b/upd/api/handlers.go index ebbf08e..47af5a1 100644 --- a/upd/api/handlers.go +++ b/upd/api/handlers.go @@ -18,25 +18,18 @@ along with this program. If not, see . package api import ( - //"archive/zip" - "fmt" - //"github.com/gin-gonic/gin" - //"github.com/gin-gonic/gin/binding" - "encoding/json" "github.com/gofiber/fiber/v2" "github.com/google/uuid" "github.com/tlinden/up/upd/cfg" bolt "go.etcd.io/bbolt" - //"io" - // "net/http" + "os" "path/filepath" - //"regexp" "strings" "time" ) -func Putfile(c *fiber.Ctx, cfg *cfg.Config, db *bolt.DB) (string, error) { +func FilePut(c *fiber.Ctx, cfg *cfg.Config, db *bolt.DB) (string, error) { // supports upload of multiple files with: // // curl -X POST localhost:8080/putfile \ @@ -94,9 +87,10 @@ func Putfile(c *fiber.Ctx, cfg *cfg.Config, db *bolt.DB) (string, error) { } if len(entry.Members) == 1 { - returnUrl = cfg.Url + cfg.ApiPrefix + "/getfile/" + id + "/" + entry.Members[0] + returnUrl = strings.Join([]string{cfg.Url + cfg.ApiPrefix + ApiVersion, "file", id, entry.Members[0]}, "/") entry.File = entry.Members[0] } else { + // FIXME => func! zipfile := Ts() + "data.zip" tmpzip := filepath.Join(cfg.StorageDir, zipfile) finalzip := filepath.Join(cfg.StorageDir, id, zipfile) @@ -113,7 +107,7 @@ func Putfile(c *fiber.Ctx, cfg *cfg.Config, db *bolt.DB) (string, error) { return "", err } - returnUrl = strings.Join([]string{cfg.Url + cfg.ApiPrefix + ApiVersion, "file/get", id, zipfile}, "/") + returnUrl = strings.Join([]string{cfg.Url + cfg.ApiPrefix + ApiVersion, "file", id, zipfile}, "/") entry.File = zipfile // clean up after us @@ -130,33 +124,80 @@ func Putfile(c *fiber.Ctx, cfg *cfg.Config, db *bolt.DB) (string, error) { Log("Now serving %s from %s/%s", returnUrl, cfg.StorageDir, id) Log("Expire set to: %s", entry.Expire) - go func() { - // => db.go ! - err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucketIfNotExists([]byte("uploads")) - if err != nil { - return fmt.Errorf("create bucket: %s", err) - } - - jsonentry, err := json.Marshal(entry) - if err != nil { - return fmt.Errorf("json marshalling failure: %s", err) - } - - err = b.Put([]byte(id), []byte(jsonentry)) - if err != nil { - return fmt.Errorf("insert data: %s", err) - } - - // results in: - // bbolt get /tmp/uploads.db uploads fb242922-86cb-43a8-92bc-b027c35f0586 - // {"id":"fb242922-86cb-43a8-92bc-b027c35f0586","expire":"1d","file":"2023-02-17-13-09-data.zip"} - return nil - }) - if err != nil { - Log("DB error: %s", err.Error()) - } - }() + // we do this in the background to not thwart the server + go DbInsert(db, id, entry) return returnUrl, nil } + +func FileGet(c *fiber.Ctx, cfg *cfg.Config, db *bolt.DB) error { + // deliver a file and delete it after a (configurable?) delay + + id := c.Params("id") + file := c.Params("file") + + upload, err := DbLookupId(db, id) + if err != nil { + // non existent db entry with that id, or other db error, see logs + return fiber.NewError(404, "No download with that id could be found!") + } + + if len(file) == 0 { + // actual file name is optional + file = upload.File + } + + filename := filepath.Join(cfg.StorageDir, id, file) + + if _, err := os.Stat(filename); err != nil { + // db entry is there, but file isn't (anymore?) + go DbDeleteId(db, id) + } + + // finally put the file to the client + err = c.Download(filename, file) + + go func() { + // check if we need to delete the file now + if upload.Expire == "asap" { + cleanup(filepath.Join(cfg.StorageDir, id)) + go DbDeleteId(db, id) + } + }() + + return err +} + +type Id struct { + Id string `json:"name" xml:"name" form:"name"` +} + +func FileDelete(c *fiber.Ctx, cfg *cfg.Config, db *bolt.DB) error { + // delete file, id dir and db entry + + id := c.Params("id") + + // try: path, body(json), query param + if len(id) == 0 { + p := new(Id) + if err := c.BodyParser(p); err != nil { + if len(p.Id) == 0 { + id = c.Query("id") + if len(p.Id) == 0 { + return fiber.NewError(403, "No id given!") + } + } + id = p.Id + } + } + + cleanup(filepath.Join(cfg.StorageDir, id)) + + err := DbDeleteId(db, id) + if err != nil { + // non existent db entry with that id, or other db error, see logs + return fiber.NewError(404, "No upload with that id could be found!") + } + + return nil +} diff --git a/upd/api/server.go b/upd/api/server.go index c0d4799..d8b247b 100644 --- a/upd/api/server.go +++ b/upd/api/server.go @@ -23,20 +23,24 @@ import ( "github.com/gofiber/fiber/v2/middleware/requestid" "github.com/tlinden/up/upd/cfg" bolt "go.etcd.io/bbolt" - "os" - "path/filepath" - "time" ) func Runserver(cfg *cfg.Config, args []string) error { - dst := cfg.StorageDir - router := fiber.New() + router := fiber.New(fiber.Config{ + CaseSensitive: true, + StrictRouting: true, + Immutable: true, + Prefork: cfg.Prefork, + ServerHeader: "upd", + AppName: cfg.AppName, + BodyLimit: cfg.BodyLimit, + Network: cfg.Network, + }) + router.Use(requestid.New()) router.Use(logger.New(logger.Config{ - // For more options, see the Config section Format: "${pid} ${locals:requestid} ${status} - ${method} ${path}​\n", })) - api := router.Group(cfg.ApiPrefix + ApiVersion) db, err := bolt.Open(cfg.DbFile, 0600, nil) if err != nil { @@ -44,42 +48,27 @@ func Runserver(cfg *cfg.Config, args []string) error { } defer db.Close() + api := router.Group(cfg.ApiPrefix + ApiVersion) { - api.Post("/file/put", func(c *fiber.Ctx) error { - uri, err := Putfile(c, cfg, db) - - if err != nil { - return c.Status(fiber.StatusBadRequest).JSON(Result{ - Code: fiber.StatusBadRequest, - Message: err.Error(), - Success: false, - }) - } else { - return c.Status(fiber.StatusOK).JSON(Result{ - Code: fiber.StatusOK, - Message: uri, - Success: true, - }) - } + api.Post("/file/", func(c *fiber.Ctx) error { + msg, err := FilePut(c, cfg, db) + return SendResponse(c, msg, err) }) - api.Get("/file/get/:id/:file", func(c *fiber.Ctx) error { - // deliver a file and delete it after a delay (FIXME: check - // when gin has done delivering it?). Redirect to the static - // handler for actual delivery. - id := c.Params("id") - file := c.Params("file") + api.Get("/file/:id/:file", func(c *fiber.Ctx) error { + return FileGet(c, cfg, db) + }) - filename := filepath.Join(dst, id, file) + api.Get("/file/:id/", func(c *fiber.Ctx) error { + return FileGet(c, cfg, db) + }) - if _, err := os.Stat(filename); err == nil { - go func() { - time.Sleep(500 * time.Millisecond) - cleanup(filepath.Join(dst, id)) - }() - } + api.Delete("/file/:id/", func(c *fiber.Ctx) error { + return FileDelete(c, cfg, db) + }) - return c.Download(filename, file) + api.Delete("/file/", func(c *fiber.Ctx) error { + return FileDelete(c, cfg, db) }) } @@ -87,7 +76,22 @@ func Runserver(cfg *cfg.Config, args []string) error { return c.Send([]byte("welcome to upload api, use /api enpoint!")) }) - router.Listen(cfg.Listen) + return router.Listen(cfg.Listen) - return nil +} + +func SendResponse(c *fiber.Ctx, msg string, err error) error { + if err != nil { + return c.Status(fiber.StatusBadRequest).JSON(Result{ + Code: fiber.StatusBadRequest, + Message: err.Error(), + Success: false, + }) + } + + return c.Status(fiber.StatusOK).JSON(Result{ + Code: fiber.StatusOK, + Message: msg, + Success: true, + }) } diff --git a/upd/cfg/config.go b/upd/cfg/config.go index dd2b69f..41c9b9f 100644 --- a/upd/cfg/config.go +++ b/upd/cfg/config.go @@ -32,6 +32,15 @@ type Config struct { StorageDir string Url string DbFile string + + // fiber settings, see: + // https://docs.gofiber.io/api/fiber/#config + Prefork bool + AppName string + BodyLimit int + V4only bool + V6only bool + Network string } func Getversion() string { @@ -44,6 +53,10 @@ func Getversion() string { return fmt.Sprintf("This is upd version %s", VERSION) } +func (c *Config) GetVersion() string { + return VERSION +} + // post processing of options, if any func (c *Config) ApplyDefaults() { if len(c.Url) == 0 { @@ -53,4 +66,17 @@ func (c *Config) ApplyDefaults() { c.Url = "http://" + c.Listen } } + + switch { + case c.V4only: + c.Network = "tcp4" + case c.V6only: + c.Network = "tcp6" + default: + if c.Prefork { + c.Network = "tcp4" + } else { + c.Network = "tcp" // dual stack + } + } } diff --git a/upd/cmd/root.go b/upd/cmd/root.go index 2913e88..6a05b2c 100644 --- a/upd/cmd/root.go +++ b/upd/cmd/root.go @@ -82,12 +82,21 @@ func Execute() { rootCmd.PersistentFlags().BoolVarP(&ShowVersion, "version", "v", false, "Print program version") rootCmd.PersistentFlags().StringVarP(&cfgFile, "config", "c", "", "custom config file") rootCmd.PersistentFlags().BoolVarP(&conf.Debug, "debug", "d", false, "Enable debugging") - rootCmd.PersistentFlags().StringVarP(&conf.Listen, "listen", "l", ":8080", "listen to custom ip:port") + rootCmd.PersistentFlags().StringVarP(&conf.Listen, "listen", "l", ":8080", "listen to custom ip:port (use [ip]:port for ipv6)") rootCmd.PersistentFlags().StringVarP(&conf.StorageDir, "storagedir", "s", "/tmp", "storage directory for uploaded files") rootCmd.PersistentFlags().StringVarP(&conf.ApiPrefix, "apiprefix", "a", "/api", "API endpoint path") rootCmd.PersistentFlags().StringVarP(&conf.Url, "url", "u", "", "HTTP endpoint w/o path") rootCmd.PersistentFlags().StringVarP(&conf.DbFile, "dbfile", "D", "/tmp/uploads.db", "Bold database file to use") + // server settings + rootCmd.PersistentFlags().BoolVarP(&conf.V4only, "ipv4", "4", false, "Only listen on ipv4") + rootCmd.PersistentFlags().BoolVarP(&conf.V6only, "ipv6", "6", false, "Only listen on ipv6") + rootCmd.MarkFlagsMutuallyExclusive("ipv4", "ipv6") + + rootCmd.PersistentFlags().BoolVarP(&conf.Prefork, "prefork", "p", false, "Prefork server threads") + rootCmd.PersistentFlags().StringVarP(&conf.AppName, "appname", "n", "upd "+conf.GetVersion(), "App name to say hi as") + rootCmd.PersistentFlags().IntVarP(&conf.BodyLimit, "bodylimit", "b", 10250000000, "Max allowed upload size in bytes") + err := rootCmd.Execute() if err != nil { os.Exit(1) diff --git a/upd/testzip.go b/upd/testzip.go deleted file mode 100644 index d7c61d4..0000000 --- a/upd/testzip.go +++ /dev/null @@ -1,18 +0,0 @@ -package main - -import ( - "github.com/tlinden/up/upd/api" - "log" - "os" -) - -func main() { - if len(os.Args) > 2 { - dir, err := os.Getwd() - if err != nil { - } - if err := api.ZipSource(dir+"/"+os.Args[1], os.Args[2]); err != nil { - log.Fatal(err) - } - } -}