scip/ephemerup
1
0
Fork 0
mirror of https://codeberg.org/scip/ephemerup.git synced 2025-12-17 20:50:56 +01:00
Code Issues Packages Projects Releases Wiki Activity

some changes:

Browse Source 
- reorganized client code, using cobra subcommands
- more reorg in upd code
This commit is contained in:
Thomas von Dein
2023-03-02 18:31:07 +01:00
parent 632f7d6a2a
commit 2a2e41126d
7 changed files with 190 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
upctl/cfg/config.go
View File

@@ -23,6 +23,7 @@ import (
const Version string = "v0.0.1"
var ApiVersion string = "/v1"
var VERSION string // maintained by -x
type Config struct {

15
upctl/cmd/root.go
View File

@@ -23,7 +23,6 @@ import (
"github.com/spf13/pflag"
"github.com/spf13/viper"
"github.com/tlinden/up/upctl/cfg"
"github.com/tlinden/up/upctl/lib"
"os"
"strings"
)
@@ -56,8 +55,8 @@ func Execute() {
var rootCmd = &cobra.Command{
Use: "upctl [options]",
Short: "upload client",
Long: `Upload files to an upload api.`,
Short: "upload api client",
Long: `Manage files on an upload api server.`,
RunE: func(cmd *cobra.Command, args []string) error {
if ShowVersion {
fmt.Println(cfg.Getversion())
@@ -69,13 +68,10 @@ func Execute() {
}
if len(args) == 0 {
return errors.New("No files specified to upload!")
return errors.New("No command specified!")
}
// errors at this stage do not cause the usage to be shown
cmd.SilenceUsage = true
return lib.Runclient(&conf, args)
return nil
},
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
return initConfig(cmd, &conf)
@@ -88,7 +84,8 @@ func Execute() {
rootCmd.PersistentFlags().StringVarP(&cfgFile, "config", "c", "", "custom config file")
rootCmd.PersistentFlags().IntVarP(&conf.Retries, "retries", "r", 3, "How often shall we retry to access our endpoint")
rootCmd.PersistentFlags().StringVarP(&conf.Endpoint, "endpoint", "p", "http://localhost:8080/api", "upload api endpoint url")
rootCmd.PersistentFlags().StringVarP(&conf.Expire, "expire", "e", "asap", "Expire setting: asap or duration (accepted shortcuts: dmh)")
rootCmd.AddCommand(UploadCommand(&conf))
err := rootCmd.Execute()
if err != nil {

47
upctl/cmd/upload.go Normal file
View File

@@ -0,0 +1,47 @@
/*
Copyright © 2023 Thomas von Dein
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package cmd
import (
"errors"
"github.com/spf13/cobra"
"github.com/tlinden/up/upctl/cfg"
"github.com/tlinden/up/upctl/lib"
)
func UploadCommand(conf *cfg.Config) *cobra.Command {
var uploadCmd = &cobra.Command{
Use: "upload [options] [file ..]",
Short: "upload files",
Long: `Upload files to an upload api.`,
RunE: func(cmd *cobra.Command, args []string) error {
if len(args) == 0 {
return errors.New("No files specified to upload!")
}
// errors at this stage do not cause the usage to be shown
cmd.SilenceUsage = true
return lib.Upload(conf, args)
},
}
// options
uploadCmd.PersistentFlags().StringVarP(&conf.Expire, "expire", "e", "", "Expire setting: asap or duration (accepted shortcuts: dmh)")
return uploadCmd
}

75
upctl/lib/client.go
View File

@@ -19,7 +19,7 @@ package lib
import (
"encoding/json"
//"errors"
"errors"
"fmt"
"github.com/imroc/req/v3"
"github.com/tlinden/up/upctl/cfg"
@@ -34,9 +34,12 @@ type Response struct {
Message string `json:"message"`
}
const ApiVersion string = "/v1"
type Request struct {
R *req.Request
Url string
}
func Runclient(c *cfg.Config, args []string) error {
func Setup(c *cfg.Config, path string) *Request {
client := req.C()
if c.Debug {
client.DevMode()
@@ -44,9 +47,28 @@ func Runclient(c *cfg.Config, args []string) error {
client.SetUserAgent("upctl-" + cfg.VERSION)
url := c.Endpoint + ApiVersion + "/file/"
R := client.R()
rq := client.R()
if c.Retries > 0 {
// Enable retry and set the maximum retry count.
R.SetRetryCount(c.Retries).
// Set the retry sleep interval with a commonly used
// algorithm: capped exponential backoff with jitter
// (https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/).
SetRetryBackoffInterval(1*time.Second, 5*time.Second).
AddRetryHook(func(resp *req.Response, err error) {
req := resp.Request.RawRequest
if c.Debug {
fmt.Println("Retrying endpoint request:", req.Method, req.URL)
}
})
}
return &Request{Url: c.Endpoint + cfg.ApiVersion + "/file/", R: R}
}
func GatherFiles(rq *Request, args []string) error {
for _, file := range args {
info, err := os.Stat(file)
@@ -61,7 +83,7 @@ func Runclient(c *cfg.Config, args []string) error {
}
if !info.IsDir() {
rq.SetFile("upload[]", path)
rq.R.SetFile("upload[]", path)
}
return nil
})
@@ -70,27 +92,24 @@ func Runclient(c *cfg.Config, args []string) error {
return err
}
} else {
rq.SetFile("upload[]", file)
rq.R.SetFile("upload[]", file)
}
}
if c.Retries > 0 {
// Enable retry and set the maximum retry count.
rq.SetRetryCount(c.Retries).
// Set the retry sleep interval with a commonly used
// algorithm: capped exponential backoff with jitter
// (https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/).
SetRetryBackoffInterval(1*time.Second, 5*time.Second).
AddRetryHook(func(resp *req.Response, err error) {
req := resp.Request.RawRequest
if c.Debug {
fmt.Println("Retrying endpoint request:", req.Method, req.URL)
}
})
return nil
}
func Upload(c *cfg.Config, args []string) error {
// setup url, req.Request, timeout handling etc
rq := Setup(c, "/file/")
// collect files to upload from @argv
if err := GatherFiles(rq, args); err != nil {
return err
}
resp, err := rq.
// actual post w/ settings
resp, err := rq.R.
SetFormData(map[string]string{
"expire": c.Expire,
}).
@@ -98,7 +117,7 @@ func Runclient(c *cfg.Config, args []string) error {
fmt.Printf("\r%q uploaded %.2f%%", info.FileName, float64(info.UploadedSize)/float64(info.FileSize)*100.0)
fmt.Println()
}, 10*time.Millisecond).
Post(url)
Post(rq.Url)
fmt.Println("")
@@ -106,12 +125,10 @@ func Runclient(c *cfg.Config, args []string) error {
return err
}
// we expect a json response
r := Response{}
json.Unmarshal([]byte(resp.String()), &r)
fmt.Println(r)
if c.Debug {
trace := resp.TraceInfo() // Use `resp.Request.TraceInfo()` to avoid unnecessary struct copy in production.
fmt.Println(trace.Blame()) // Print out exactly where the http request is slowing down.
@@ -119,5 +136,11 @@ func Runclient(c *cfg.Config, args []string) error {
fmt.Println(trace)
}
if !r.Success {
return errors.New(r.Message)
}
fmt.Println(r.Message)
return nil
}

79
upd/api/fileio.go
View File

@@ -20,9 +20,13 @@ package api
import (
"archive/zip"
"errors"
"github.com/gofiber/fiber/v2"
"github.com/tlinden/up/upd/cfg"
"io"
"mime/multipart"
"os"
"path/filepath"
"strings"
"sync"
)
@@ -35,10 +39,69 @@ func cleanup(dir string) {
}
}
func ZipSource(source, target string) error {
// 1. Create a ZIP file and zip.Writer
// source must be an absolute path, target a zip file
f, err := os.Create(target)
// Extract form file[s] and store them on disk, returns a list of files
func SaveFormFiles(c *fiber.Ctx, cfg *cfg.Config, files []*multipart.FileHeader, id string) ([]string, error) {
members := []string{}
for _, file := range files {
filename := NormalizeFilename(filepath.Base(file.Filename))
path := filepath.Join(cfg.StorageDir, id, filename)
members = append(members, filename)
Log("Received: %s => %s/%s", file.Filename, id, filename)
if err := c.SaveFile(file, path); err != nil {
cleanup(filepath.Join(cfg.StorageDir, id))
return nil, err
}
}
return members, nil
}
// generate return url. in case of multiple files, zip and remove them
func ProcessFormFiles(cfg *cfg.Config, members []string, id string) (string, string, error) {
returnUrl := ""
Filename := ""
if len(members) == 1 {
returnUrl = strings.Join([]string{cfg.Url + cfg.ApiPrefix + ApiVersion, "file", id, members[0]}, "/")
Filename = members[0]
} else {
zipfile := Ts() + "data.zip"
tmpzip := filepath.Join(cfg.StorageDir, zipfile)
finalzip := filepath.Join(cfg.StorageDir, id, zipfile)
iddir := filepath.Join(cfg.StorageDir, id)
if err := ZipDir(iddir, tmpzip); err != nil {
cleanup(iddir)
Log("zip error")
return "", "", err
}
if err := os.Rename(tmpzip, finalzip); err != nil {
cleanup(iddir)
return "", "", err
}
returnUrl = strings.Join([]string{cfg.Url + cfg.ApiPrefix + ApiVersion, "file", id, zipfile}, "/")
Filename = zipfile
// clean up after us
go func() {
for _, file := range members {
if err := os.Remove(filepath.Join(cfg.StorageDir, id, file)); err != nil {
Log("ERROR: unable to delete %s: %s", file, err)
}
}
}()
}
return returnUrl, Filename, nil
}
// Create a zip archive from a directory
// FIXME: -e option, if any, goes here
func ZipDir(directory, zipfilename string) error {
f, err := os.Create(zipfilename)
if err != nil {
return err
}
@@ -54,17 +117,17 @@ func ZipSource(source, target string) error {
go func() {
defer wg.Done()
os.Chdir(source)
os.Chdir(directory)
newDir, err := os.Getwd()
if err != nil {
}
if newDir != source {
if newDir != directory {
err = errors.New("Failed to changedir!")
return
}
err = filepath.Walk(".", func(path string, info os.FileInfo, err error) error {
// 2. Go through all the files of the source
// 2. Go through all the files of the directory
if err != nil {
return err
}
@@ -80,7 +143,7 @@ func ZipSource(source, target string) error {
// 4. Set relative path of a file as the header name
header.Name = path
//Log("a: <%s>, b: <%s>, rel: <%s>", filepath.Dir(source), path, header.Name)
//Log("a: <%s>, b: <%s>, rel: <%s>", filepath.Dir(directory), path, header.Name)
if err != nil {
return err
}

71
upd/api/handlers.go
View File

@@ -24,7 +24,6 @@ import (
"os"
"path/filepath"
"strings"
"time"
)
@@ -61,67 +60,35 @@ func FilePut(c *fiber.Ctx, cfg *cfg.Config, db *Db) (string, error) {
// retrieve files, if any
files := form.File["upload[]"]
for _, file := range files {
filename := NormalizeFilename(filepath.Base(file.Filename))
path := filepath.Join(cfg.StorageDir, id, filename)
entry.Members = append(entry.Members, filename)
Log("Received: %s => %s/%s", file.Filename, id, filename)
if err := c.SaveFile(file, path); err != nil {
cleanup(filepath.Join(cfg.StorageDir, id))
return "", err
}
members, err := SaveFormFiles(c, cfg, files, id)
if err != nil {
return "", fiber.NewError(500, "Could not store uploaded file[s]!")
}
entry.Members = members
// extract auxilliary form data (expire field et al)
if err := c.BodyParser(&formdata); err != nil {
Log("bodyparser error %s", err.Error())
return "", err
}
// post process expire
if len(formdata.Expire) == 0 {
entry.Expire = "asap"
} else {
ex, err := Untaint(formdata.Expire, `[dhms0-9]`) // duration or asap allowed
ex, err := Untaint(formdata.Expire, `[^dhms0-9]`) // duration or asap allowed
if err != nil {
return "", err
}
entry.Expire = ex
}
if len(entry.Members) == 1 {
returnUrl = strings.Join([]string{cfg.Url + cfg.ApiPrefix + ApiVersion, "file", id, entry.Members[0]}, "/")
entry.File = entry.Members[0]
} else {
// FIXME => func!
zipfile := Ts() + "data.zip"
tmpzip := filepath.Join(cfg.StorageDir, zipfile)
finalzip := filepath.Join(cfg.StorageDir, id, zipfile)
iddir := filepath.Join(cfg.StorageDir, id)
if err := ZipSource(iddir, tmpzip); err != nil {
cleanup(iddir)
Log("zip error")
return "", err
}
if err := os.Rename(tmpzip, finalzip); err != nil {
cleanup(iddir)
return "", err
}
returnUrl = strings.Join([]string{cfg.Url + cfg.ApiPrefix + ApiVersion, "file", id, zipfile}, "/")
entry.File = zipfile
// clean up after us
go func() {
for _, file := range entry.Members {
if err := os.Remove(filepath.Join(cfg.StorageDir, id, file)); err != nil {
Log("ERROR: unable to delete %s: %s", file, err)
}
}
}()
// get url [and zip if there are multiple files]
returnUrl, Newfilename, err := ProcessFormFiles(cfg, entry.Members, id)
if err != nil {
return "", fiber.NewError(500, "Could not process uploaded file[s]!")
}
entry.File = Newfilename
Log("Now serving %s from %s/%s", returnUrl, cfg.StorageDir, id)
Log("Expire set to: %s", entry.Expire)
@@ -149,12 +116,12 @@ func FileGet(c *fiber.Ctx, cfg *cfg.Config, db *Db) error {
}
file := upload.File
filename := filepath.Join(cfg.StorageDir, id, file)
if _, err := os.Stat(filename); err != nil {
// db entry is there, but file isn't (anymore?)
go db.Delete(id)
return fiber.NewError(404, "No download with that id could be found!")
}
// finally put the file to the client
@@ -183,18 +150,8 @@ func FileDelete(c *fiber.Ctx, cfg *cfg.Config, db *Db) error {
return fiber.NewError(403, "Invalid id provided!")
}
// try: path, body(json), query param
if len(id) == 0 {
p := new(Id)
if err := c.BodyParser(p); err != nil {
if len(p.Id) == 0 {
id = c.Query("id")
if len(p.Id) == 0 {
return fiber.NewError(403, "No id given!")
}
}
id = p.Id
}
return fiber.NewError(403, "No id given!")
}
cleanup(filepath.Join(cfg.StorageDir, id))

6
upd/api/server.go
View File

@@ -65,10 +65,6 @@ func Runserver(cfg *cfg.Config, args []string) error {
api.Delete("/file/:id/", func(c *fiber.Ctx) error {
return FileDelete(c, cfg, db)
})
api.Delete("/file/", func(c *fiber.Ctx) error {
return FileDelete(c, cfg, db)
})
}
router.Get("/", func(c *fiber.Ctx) error {
@@ -80,6 +76,8 @@ func Runserver(cfg *cfg.Config, args []string) error {
}
func SendResponse(c *fiber.Ctx, msg string, err error) error {
// FIXME:
// respect fiber.NewError(500, "Could not process uploaded file[s]!")
if err != nil {
return c.Status(fiber.StatusBadRequest).JSON(Result{
Code: fiber.StatusBadRequest,