diff --git a/upctl/cfg/config.go b/upctl/cfg/config.go index 6f58638..6b5a0df 100644 --- a/upctl/cfg/config.go +++ b/upctl/cfg/config.go @@ -23,6 +23,7 @@ import ( const Version string = "v0.0.1" +var ApiVersion string = "/v1" var VERSION string // maintained by -x type Config struct { diff --git a/upctl/cmd/root.go b/upctl/cmd/root.go index ca76778..5baa20d 100644 --- a/upctl/cmd/root.go +++ b/upctl/cmd/root.go @@ -23,7 +23,6 @@ import ( "github.com/spf13/pflag" "github.com/spf13/viper" "github.com/tlinden/up/upctl/cfg" - "github.com/tlinden/up/upctl/lib" "os" "strings" ) @@ -56,8 +55,8 @@ func Execute() { var rootCmd = &cobra.Command{ Use: "upctl [options]", - Short: "upload client", - Long: `Upload files to an upload api.`, + Short: "upload api client", + Long: `Manage files on an upload api server.`, RunE: func(cmd *cobra.Command, args []string) error { if ShowVersion { fmt.Println(cfg.Getversion()) @@ -69,13 +68,10 @@ func Execute() { } if len(args) == 0 { - return errors.New("No files specified to upload!") + return errors.New("No command specified!") } - // errors at this stage do not cause the usage to be shown - cmd.SilenceUsage = true - - return lib.Runclient(&conf, args) + return nil }, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { return initConfig(cmd, &conf) @@ -88,7 +84,8 @@ func Execute() { rootCmd.PersistentFlags().StringVarP(&cfgFile, "config", "c", "", "custom config file") rootCmd.PersistentFlags().IntVarP(&conf.Retries, "retries", "r", 3, "How often shall we retry to access our endpoint") rootCmd.PersistentFlags().StringVarP(&conf.Endpoint, "endpoint", "p", "http://localhost:8080/api", "upload api endpoint url") - rootCmd.PersistentFlags().StringVarP(&conf.Expire, "expire", "e", "asap", "Expire setting: asap or duration (accepted shortcuts: dmh)") + + rootCmd.AddCommand(UploadCommand(&conf)) err := rootCmd.Execute() if err != nil { diff --git a/upctl/cmd/upload.go b/upctl/cmd/upload.go new file mode 100644 index 0000000..47d59ea --- /dev/null +++ b/upctl/cmd/upload.go @@ -0,0 +1,47 @@ +/* +Copyright © 2023 Thomas von Dein + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ +package cmd + +import ( + "errors" + "github.com/spf13/cobra" + "github.com/tlinden/up/upctl/cfg" + "github.com/tlinden/up/upctl/lib" +) + +func UploadCommand(conf *cfg.Config) *cobra.Command { + var uploadCmd = &cobra.Command{ + Use: "upload [options] [file ..]", + Short: "upload files", + Long: `Upload files to an upload api.`, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return errors.New("No files specified to upload!") + } + + // errors at this stage do not cause the usage to be shown + cmd.SilenceUsage = true + + return lib.Upload(conf, args) + }, + } + + // options + uploadCmd.PersistentFlags().StringVarP(&conf.Expire, "expire", "e", "", "Expire setting: asap or duration (accepted shortcuts: dmh)") + + return uploadCmd +} diff --git a/upctl/lib/client.go b/upctl/lib/client.go index c86766b..ea33561 100644 --- a/upctl/lib/client.go +++ b/upctl/lib/client.go @@ -19,7 +19,7 @@ package lib import ( "encoding/json" - //"errors" + "errors" "fmt" "github.com/imroc/req/v3" "github.com/tlinden/up/upctl/cfg" @@ -34,9 +34,12 @@ type Response struct { Message string `json:"message"` } -const ApiVersion string = "/v1" +type Request struct { + R *req.Request + Url string +} -func Runclient(c *cfg.Config, args []string) error { +func Setup(c *cfg.Config, path string) *Request { client := req.C() if c.Debug { client.DevMode() @@ -44,9 +47,28 @@ func Runclient(c *cfg.Config, args []string) error { client.SetUserAgent("upctl-" + cfg.VERSION) - url := c.Endpoint + ApiVersion + "/file/" + R := client.R() - rq := client.R() + if c.Retries > 0 { + // Enable retry and set the maximum retry count. + R.SetRetryCount(c.Retries). + // Set the retry sleep interval with a commonly used + // algorithm: capped exponential backoff with jitter + // (https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/). + SetRetryBackoffInterval(1*time.Second, 5*time.Second). + AddRetryHook(func(resp *req.Response, err error) { + req := resp.Request.RawRequest + if c.Debug { + fmt.Println("Retrying endpoint request:", req.Method, req.URL) + } + }) + } + + return &Request{Url: c.Endpoint + cfg.ApiVersion + "/file/", R: R} + +} + +func GatherFiles(rq *Request, args []string) error { for _, file := range args { info, err := os.Stat(file) @@ -61,7 +83,7 @@ func Runclient(c *cfg.Config, args []string) error { } if !info.IsDir() { - rq.SetFile("upload[]", path) + rq.R.SetFile("upload[]", path) } return nil }) @@ -70,27 +92,24 @@ func Runclient(c *cfg.Config, args []string) error { return err } } else { - rq.SetFile("upload[]", file) + rq.R.SetFile("upload[]", file) } - } - if c.Retries > 0 { - // Enable retry and set the maximum retry count. - rq.SetRetryCount(c.Retries). - // Set the retry sleep interval with a commonly used - // algorithm: capped exponential backoff with jitter - // (https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/). - SetRetryBackoffInterval(1*time.Second, 5*time.Second). - AddRetryHook(func(resp *req.Response, err error) { - req := resp.Request.RawRequest - if c.Debug { - fmt.Println("Retrying endpoint request:", req.Method, req.URL) - } - }) + return nil +} + +func Upload(c *cfg.Config, args []string) error { + // setup url, req.Request, timeout handling etc + rq := Setup(c, "/file/") + + // collect files to upload from @argv + if err := GatherFiles(rq, args); err != nil { + return err } - resp, err := rq. + // actual post w/ settings + resp, err := rq.R. SetFormData(map[string]string{ "expire": c.Expire, }). @@ -98,7 +117,7 @@ func Runclient(c *cfg.Config, args []string) error { fmt.Printf("\r%q uploaded %.2f%%", info.FileName, float64(info.UploadedSize)/float64(info.FileSize)*100.0) fmt.Println() }, 10*time.Millisecond). - Post(url) + Post(rq.Url) fmt.Println("") @@ -106,12 +125,10 @@ func Runclient(c *cfg.Config, args []string) error { return err } + // we expect a json response r := Response{} - json.Unmarshal([]byte(resp.String()), &r) - fmt.Println(r) - if c.Debug { trace := resp.TraceInfo() // Use `resp.Request.TraceInfo()` to avoid unnecessary struct copy in production. fmt.Println(trace.Blame()) // Print out exactly where the http request is slowing down. @@ -119,5 +136,11 @@ func Runclient(c *cfg.Config, args []string) error { fmt.Println(trace) } + if !r.Success { + return errors.New(r.Message) + } + + fmt.Println(r.Message) + return nil } diff --git a/upd/api/fileio.go b/upd/api/fileio.go index d2068e7..b4e8d82 100644 --- a/upd/api/fileio.go +++ b/upd/api/fileio.go @@ -20,9 +20,13 @@ package api import ( "archive/zip" "errors" + "github.com/gofiber/fiber/v2" + "github.com/tlinden/up/upd/cfg" "io" + "mime/multipart" "os" "path/filepath" + "strings" "sync" ) @@ -35,10 +39,69 @@ func cleanup(dir string) { } } -func ZipSource(source, target string) error { - // 1. Create a ZIP file and zip.Writer - // source must be an absolute path, target a zip file - f, err := os.Create(target) +// Extract form file[s] and store them on disk, returns a list of files +func SaveFormFiles(c *fiber.Ctx, cfg *cfg.Config, files []*multipart.FileHeader, id string) ([]string, error) { + members := []string{} + for _, file := range files { + filename := NormalizeFilename(filepath.Base(file.Filename)) + path := filepath.Join(cfg.StorageDir, id, filename) + members = append(members, filename) + Log("Received: %s => %s/%s", file.Filename, id, filename) + + if err := c.SaveFile(file, path); err != nil { + cleanup(filepath.Join(cfg.StorageDir, id)) + return nil, err + } + } + + return members, nil +} + +// generate return url. in case of multiple files, zip and remove them +func ProcessFormFiles(cfg *cfg.Config, members []string, id string) (string, string, error) { + returnUrl := "" + Filename := "" + + if len(members) == 1 { + returnUrl = strings.Join([]string{cfg.Url + cfg.ApiPrefix + ApiVersion, "file", id, members[0]}, "/") + Filename = members[0] + } else { + zipfile := Ts() + "data.zip" + tmpzip := filepath.Join(cfg.StorageDir, zipfile) + finalzip := filepath.Join(cfg.StorageDir, id, zipfile) + iddir := filepath.Join(cfg.StorageDir, id) + + if err := ZipDir(iddir, tmpzip); err != nil { + cleanup(iddir) + Log("zip error") + return "", "", err + } + + if err := os.Rename(tmpzip, finalzip); err != nil { + cleanup(iddir) + return "", "", err + } + + returnUrl = strings.Join([]string{cfg.Url + cfg.ApiPrefix + ApiVersion, "file", id, zipfile}, "/") + Filename = zipfile + + // clean up after us + go func() { + for _, file := range members { + if err := os.Remove(filepath.Join(cfg.StorageDir, id, file)); err != nil { + Log("ERROR: unable to delete %s: %s", file, err) + } + } + }() + } + + return returnUrl, Filename, nil +} + +// Create a zip archive from a directory +// FIXME: -e option, if any, goes here +func ZipDir(directory, zipfilename string) error { + f, err := os.Create(zipfilename) if err != nil { return err } @@ -54,17 +117,17 @@ func ZipSource(source, target string) error { go func() { defer wg.Done() - os.Chdir(source) + os.Chdir(directory) newDir, err := os.Getwd() if err != nil { } - if newDir != source { + if newDir != directory { err = errors.New("Failed to changedir!") return } err = filepath.Walk(".", func(path string, info os.FileInfo, err error) error { - // 2. Go through all the files of the source + // 2. Go through all the files of the directory if err != nil { return err } @@ -80,7 +143,7 @@ func ZipSource(source, target string) error { // 4. Set relative path of a file as the header name header.Name = path - //Log("a: <%s>, b: <%s>, rel: <%s>", filepath.Dir(source), path, header.Name) + //Log("a: <%s>, b: <%s>, rel: <%s>", filepath.Dir(directory), path, header.Name) if err != nil { return err } diff --git a/upd/api/handlers.go b/upd/api/handlers.go index 7443468..e8a8ff5 100644 --- a/upd/api/handlers.go +++ b/upd/api/handlers.go @@ -24,7 +24,6 @@ import ( "os" "path/filepath" - "strings" "time" ) @@ -61,67 +60,35 @@ func FilePut(c *fiber.Ctx, cfg *cfg.Config, db *Db) (string, error) { // retrieve files, if any files := form.File["upload[]"] - for _, file := range files { - filename := NormalizeFilename(filepath.Base(file.Filename)) - path := filepath.Join(cfg.StorageDir, id, filename) - entry.Members = append(entry.Members, filename) - Log("Received: %s => %s/%s", file.Filename, id, filename) - - if err := c.SaveFile(file, path); err != nil { - cleanup(filepath.Join(cfg.StorageDir, id)) - return "", err - } + members, err := SaveFormFiles(c, cfg, files, id) + if err != nil { + return "", fiber.NewError(500, "Could not store uploaded file[s]!") } + entry.Members = members + // extract auxilliary form data (expire field et al) if err := c.BodyParser(&formdata); err != nil { Log("bodyparser error %s", err.Error()) return "", err } + // post process expire if len(formdata.Expire) == 0 { entry.Expire = "asap" } else { - ex, err := Untaint(formdata.Expire, `[dhms0-9]`) // duration or asap allowed + ex, err := Untaint(formdata.Expire, `[^dhms0-9]`) // duration or asap allowed if err != nil { return "", err } entry.Expire = ex } - if len(entry.Members) == 1 { - returnUrl = strings.Join([]string{cfg.Url + cfg.ApiPrefix + ApiVersion, "file", id, entry.Members[0]}, "/") - entry.File = entry.Members[0] - } else { - // FIXME => func! - zipfile := Ts() + "data.zip" - tmpzip := filepath.Join(cfg.StorageDir, zipfile) - finalzip := filepath.Join(cfg.StorageDir, id, zipfile) - iddir := filepath.Join(cfg.StorageDir, id) - - if err := ZipSource(iddir, tmpzip); err != nil { - cleanup(iddir) - Log("zip error") - return "", err - } - - if err := os.Rename(tmpzip, finalzip); err != nil { - cleanup(iddir) - return "", err - } - - returnUrl = strings.Join([]string{cfg.Url + cfg.ApiPrefix + ApiVersion, "file", id, zipfile}, "/") - entry.File = zipfile - - // clean up after us - go func() { - for _, file := range entry.Members { - if err := os.Remove(filepath.Join(cfg.StorageDir, id, file)); err != nil { - Log("ERROR: unable to delete %s: %s", file, err) - } - } - }() - + // get url [and zip if there are multiple files] + returnUrl, Newfilename, err := ProcessFormFiles(cfg, entry.Members, id) + if err != nil { + return "", fiber.NewError(500, "Could not process uploaded file[s]!") } + entry.File = Newfilename Log("Now serving %s from %s/%s", returnUrl, cfg.StorageDir, id) Log("Expire set to: %s", entry.Expire) @@ -149,12 +116,12 @@ func FileGet(c *fiber.Ctx, cfg *cfg.Config, db *Db) error { } file := upload.File - filename := filepath.Join(cfg.StorageDir, id, file) if _, err := os.Stat(filename); err != nil { // db entry is there, but file isn't (anymore?) go db.Delete(id) + return fiber.NewError(404, "No download with that id could be found!") } // finally put the file to the client @@ -183,18 +150,8 @@ func FileDelete(c *fiber.Ctx, cfg *cfg.Config, db *Db) error { return fiber.NewError(403, "Invalid id provided!") } - // try: path, body(json), query param if len(id) == 0 { - p := new(Id) - if err := c.BodyParser(p); err != nil { - if len(p.Id) == 0 { - id = c.Query("id") - if len(p.Id) == 0 { - return fiber.NewError(403, "No id given!") - } - } - id = p.Id - } + return fiber.NewError(403, "No id given!") } cleanup(filepath.Join(cfg.StorageDir, id)) diff --git a/upd/api/server.go b/upd/api/server.go index af6aad1..7f5bdf2 100644 --- a/upd/api/server.go +++ b/upd/api/server.go @@ -65,10 +65,6 @@ func Runserver(cfg *cfg.Config, args []string) error { api.Delete("/file/:id/", func(c *fiber.Ctx) error { return FileDelete(c, cfg, db) }) - - api.Delete("/file/", func(c *fiber.Ctx) error { - return FileDelete(c, cfg, db) - }) } router.Get("/", func(c *fiber.Ctx) error { @@ -80,6 +76,8 @@ func Runserver(cfg *cfg.Config, args []string) error { } func SendResponse(c *fiber.Ctx, msg string, err error) error { + // FIXME: + // respect fiber.NewError(500, "Could not process uploaded file[s]!") if err != nil { return c.Status(fiber.StatusBadRequest).JSON(Result{ Code: fiber.StatusBadRequest,