implemented various things

README.md

@@ -7,14 +7,31 @@ Simple standalone file upload server with api and cli
 - store ts
 - implement goroutine to expire after 1d, 10m etc
 - use bolt db to retrieve list of items to expire
-- return a meaningful  message if a file has expired,  not just a 404,
-  that is: do  remove the file when it expires  but not the associated
-  db entry.
 - also serve a html upload page
 - add auth options (access key, users, roles, oauth2)
 - add metrics
 - add upctl command to remove a file
-- upd: add short  uuid to files, in case multiple  files with the same
-  name are being uploaded
 - use global map of api endpoints like /file/get/ etc
-- use separate group for /file/
+- create cobra client commands (upload, list, delete, edit)
+
+
+
+## curl commands
+
+### upload
+
+```
+curl -X POST localhost:8080/api/putfile -F "upload[]=@xxx" -F "upload[]=@yyy" -H "Content-Type: multipart/form-data"
+```
+
+### download
+```
+curl -O http://localhost:8080/api/v1/file/388f41f4-3f0d-41e1-a022-9132c0e9e16f/2023-02-28-18-33-xxx
+```
+
+### delete
+```
+curl -X DELETE http://localhost:8080/api/v1/file/388f41f4-3f0d-41e1-a022-9132c0e9e16f/
+curl -X DELETE http://localhost:8080/api/v1/file/?id=388f41f4-3f0d-41e1-a022-9132c0e9e16f/
+curl -X DELETE -H "Accept: application/json"  -d '{"id":"$id"}' http://localhost:8080/api/v1/file/
+```

upctl/cfg/config.go

@@ -29,6 +29,7 @@ type Config struct {
 	Endpoint string
 	Debug    bool
 	Retries  int
+	Expire   string
 }
 
 func Getversion() string {

upctl/cmd/root.go

@@ -87,7 +87,8 @@ func Execute() {
 	rootCmd.PersistentFlags().BoolVarP(&conf.Debug, "debug", "d", false, "Enable debugging")
 	rootCmd.PersistentFlags().StringVarP(&cfgFile, "config", "c", "", "custom config file")
 	rootCmd.PersistentFlags().IntVarP(&conf.Retries, "retries", "r", 3, "How often shall we retry to access our endpoint")
-	rootCmd.PersistentFlags().StringVarP(&conf.Endpoint, "endpoint", "e", "http://localhost:8080/api", "upload api endpoint url")
+	rootCmd.PersistentFlags().StringVarP(&conf.Endpoint, "endpoint", "p", "http://localhost:8080/api", "upload api endpoint url")
+	rootCmd.PersistentFlags().StringVarP(&conf.Expire, "expire", "e", "asap", "Expire setting: asap or duration (accepted shortcuts: dmh)")
 
 	err := rootCmd.Execute()
 	if err != nil {

upctl/lib/client.go

@@ -44,7 +44,7 @@ func Runclient(c *cfg.Config, args []string) error {
 
 	client.SetUserAgent("upctl-" + cfg.VERSION)
 
-	url := c.Endpoint + ApiVersion + "/file/put"
+	url := c.Endpoint + ApiVersion + "/file/"
 
 	rq := client.R()
 	for _, file := range args {
@@ -92,7 +92,7 @@ func Runclient(c *cfg.Config, args []string) error {
 
 	resp, err := rq.
 		SetFormData(map[string]string{
-			"expire": "1d",
+			"expire": c.Expire,
 		}).
 		SetUploadCallbackWithInterval(func(info req.UploadInfo) {
 			fmt.Printf("\r%q uploaded %.2f%%", info.FileName, float64(info.UploadedSize)/float64(info.FileSize)*100.0)

upd/api/db.go

@@ -0,0 +1,101 @@
+/*
+Copyright © 2023 Thomas von Dein
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package api
+
+import (
+	"encoding/json"
+	"fmt"
+
+	bolt "go.etcd.io/bbolt"
+)
+
+const Bucket string = "uploads"
+
+func DbInsert(db *bolt.DB, id string, entry *Upload) {
+	err := db.Update(func(tx *bolt.Tx) error {
+		bucket, err := tx.CreateBucketIfNotExists([]byte(Bucket))
+		if err != nil {
+			return fmt.Errorf("create bucket: %s", err)
+		}
+
+		jsonentry, err := json.Marshal(entry)
+		if err != nil {
+			return fmt.Errorf("json marshalling failure: %s", err)
+		}
+
+		err = bucket.Put([]byte(id), []byte(jsonentry))
+		if err != nil {
+			return fmt.Errorf("insert data: %s", err)
+		}
+
+		// results in:
+		// bbolt get /tmp/uploads.db uploads fb242922-86cb-43a8-92bc-b027c35f0586
+		// {"id":"fb242922-86cb-43a8-92bc-b027c35f0586","expire":"1d","file":"2023-02-17-13-09-data.zip"}
+		return nil
+	})
+	if err != nil {
+		Log("DB error: %s", err.Error())
+	}
+}
+
+func DbLookupId(db *bolt.DB, id string) (Upload, error) {
+	var upload Upload
+
+	err := db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(Bucket))
+		j := bucket.Get([]byte(id))
+
+		if len(j) == 0 {
+			return fmt.Errorf("id %s not found", id)
+		}
+
+		if err := json.Unmarshal(j, &upload); err != nil {
+			return fmt.Errorf("unable to unmarshal json: %s", err)
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		Log("DB error: %s", err.Error())
+		return upload, err
+	}
+
+	return upload, nil
+}
+
+func DbDeleteId(db *bolt.DB, id string) error {
+	err := db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(Bucket))
+
+		j := bucket.Get([]byte(id))
+
+		if len(j) == 0 {
+			return fmt.Errorf("id %s not found", id)
+		}
+
+		err := bucket.Delete([]byte(id))
+		return err
+	})
+
+	if err != nil {
+		Log("DB error: %s", err.Error())
+	}
+
+	return err
+}

upd/api/fileio.go

@@ -19,6 +19,7 @@ package api
 
 import (
 	"archive/zip"
+	"errors"
 	"io"
 	"os"
 	"path/filepath"
@@ -26,15 +27,12 @@ import (
 )
 
 // cleanup an upload directory, either because  we got an error in the
-// middle of an upload or something else  went wrong. we fork off a go
-// routine because this may block.
+// middle of an upload or something else  went wrong.
 func cleanup(dir string) {
-	go func() {
-		err := os.RemoveAll(dir)
-		if err != nil {
-			Log("Failed to remove dir %s: %s", dir, err.Error())
-		}
-	}()
+	err := os.RemoveAll(dir)
+	if err != nil {
+		Log("Failed to remove dir %s: %s", dir, err.Error())
+	}
 }
 
 func ZipSource(source, target string) error {
@@ -60,7 +58,10 @@ func ZipSource(source, target string) error {
 		newDir, err := os.Getwd()
 		if err != nil {
 		}
-		//Log("Current Working Direcotry: %s\n, source: %s", newDir, source)
+		if newDir != source {
+			err = errors.New("Failed to changedir!")
+			return
+		}
 
 		err = filepath.Walk(".", func(path string, info os.FileInfo, err error) error {
 			// 2. Go through all the files of the source

upd/api/handlers.go

@@ -18,25 +18,18 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
 package api
 
 import (
-	//"archive/zip"
-	"fmt"
-	//"github.com/gin-gonic/gin"
-	//"github.com/gin-gonic/gin/binding"
-	"encoding/json"
 	"github.com/gofiber/fiber/v2"
 	"github.com/google/uuid"
 	"github.com/tlinden/up/upd/cfg"
 	bolt "go.etcd.io/bbolt"
-	//"io"
-	// "net/http"
+
 	"os"
 	"path/filepath"
-	//"regexp"
 	"strings"
 	"time"
 )
 
-func Putfile(c *fiber.Ctx, cfg *cfg.Config, db *bolt.DB) (string, error) {
+func FilePut(c *fiber.Ctx, cfg *cfg.Config, db *bolt.DB) (string, error) {
 	// supports upload of multiple files with:
 	//
 	// curl -X POST localhost:8080/putfile \
@@ -94,9 +87,10 @@ func Putfile(c *fiber.Ctx, cfg *cfg.Config, db *bolt.DB) (string, error) {
 	}
 
 	if len(entry.Members) == 1 {
-		returnUrl = cfg.Url + cfg.ApiPrefix + "/getfile/" + id + "/" + entry.Members[0]
+		returnUrl = strings.Join([]string{cfg.Url + cfg.ApiPrefix + ApiVersion, "file", id, entry.Members[0]}, "/")
 		entry.File = entry.Members[0]
 	} else {
+		// FIXME => func!
 		zipfile := Ts() + "data.zip"
 		tmpzip := filepath.Join(cfg.StorageDir, zipfile)
 		finalzip := filepath.Join(cfg.StorageDir, id, zipfile)
@@ -113,7 +107,7 @@ func Putfile(c *fiber.Ctx, cfg *cfg.Config, db *bolt.DB) (string, error) {
 			return "", err
 		}
 
-		returnUrl = strings.Join([]string{cfg.Url + cfg.ApiPrefix + ApiVersion, "file/get", id, zipfile}, "/")
+		returnUrl = strings.Join([]string{cfg.Url + cfg.ApiPrefix + ApiVersion, "file", id, zipfile}, "/")
 		entry.File = zipfile
 
 		// clean up after us
@@ -130,33 +124,80 @@ func Putfile(c *fiber.Ctx, cfg *cfg.Config, db *bolt.DB) (string, error) {
 	Log("Now serving %s from %s/%s", returnUrl, cfg.StorageDir, id)
 	Log("Expire set to: %s", entry.Expire)
 
-	go func() {
-		// => db.go !
-		err := db.Update(func(tx *bolt.Tx) error {
-			b, err := tx.CreateBucketIfNotExists([]byte("uploads"))
-			if err != nil {
-				return fmt.Errorf("create bucket: %s", err)
-			}
-
-			jsonentry, err := json.Marshal(entry)
-			if err != nil {
-				return fmt.Errorf("json marshalling failure: %s", err)
-			}
-
-			err = b.Put([]byte(id), []byte(jsonentry))
-			if err != nil {
-				return fmt.Errorf("insert data: %s", err)
-			}
-
-			// results in:
-			// bbolt get /tmp/uploads.db uploads fb242922-86cb-43a8-92bc-b027c35f0586
-			// {"id":"fb242922-86cb-43a8-92bc-b027c35f0586","expire":"1d","file":"2023-02-17-13-09-data.zip"}
-			return nil
-		})
-		if err != nil {
-			Log("DB error: %s", err.Error())
-		}
-	}()
+	// we do this in the background to not thwart the server
+	go DbInsert(db, id, entry)
 
 	return returnUrl, nil
 }
+
+func FileGet(c *fiber.Ctx, cfg *cfg.Config, db *bolt.DB) error {
+	// deliver  a file and delete  it after a (configurable?) delay
+
+	id := c.Params("id")
+	file := c.Params("file")
+
+	upload, err := DbLookupId(db, id)
+	if err != nil {
+		// non existent db entry with that id, or other db error, see logs
+		return fiber.NewError(404, "No download with that id could be found!")
+	}
+
+	if len(file) == 0 {
+		// actual file name is optional
+		file = upload.File
+	}
+
+	filename := filepath.Join(cfg.StorageDir, id, file)
+
+	if _, err := os.Stat(filename); err != nil {
+		// db entry is there, but file isn't (anymore?)
+		go DbDeleteId(db, id)
+	}
+
+	// finally put the file to the client
+	err = c.Download(filename, file)
+
+	go func() {
+		// check if we need to delete the file now
+		if upload.Expire == "asap" {
+			cleanup(filepath.Join(cfg.StorageDir, id))
+			go DbDeleteId(db, id)
+		}
+	}()
+
+	return err
+}
+
+type Id struct {
+	Id string `json:"name" xml:"name" form:"name"`
+}
+
+func FileDelete(c *fiber.Ctx, cfg *cfg.Config, db *bolt.DB) error {
+	// delete file, id dir and db entry
+
+	id := c.Params("id")
+
+	// try: path, body(json), query param
+	if len(id) == 0 {
+		p := new(Id)
+		if err := c.BodyParser(p); err != nil {
+			if len(p.Id) == 0 {
+				id = c.Query("id")
+				if len(p.Id) == 0 {
+					return fiber.NewError(403, "No id given!")
+				}
+			}
+			id = p.Id
+		}
+	}
+
+	cleanup(filepath.Join(cfg.StorageDir, id))
+
+	err := DbDeleteId(db, id)
+	if err != nil {
+		// non existent db entry with that id, or other db error, see logs
+		return fiber.NewError(404, "No upload with that id could be found!")
+	}
+
+	return nil
+}

upd/api/server.go

@@ -23,20 +23,24 @@ import (
 	"github.com/gofiber/fiber/v2/middleware/requestid"
 	"github.com/tlinden/up/upd/cfg"
 	bolt "go.etcd.io/bbolt"
-	"os"
-	"path/filepath"
-	"time"
 )
 
 func Runserver(cfg *cfg.Config, args []string) error {
-	dst := cfg.StorageDir
-	router := fiber.New()
+	router := fiber.New(fiber.Config{
+		CaseSensitive: true,
+		StrictRouting: true,
+		Immutable:     true,
+		Prefork:       cfg.Prefork,
+		ServerHeader:  "upd",
+		AppName:       cfg.AppName,
+		BodyLimit:     cfg.BodyLimit,
+		Network:       cfg.Network,
+	})
+
 	router.Use(requestid.New())
 	router.Use(logger.New(logger.Config{
-		// For more options, see the Config section
 		Format: "${pid} ${locals:requestid} ${status} - ${method} ${path}​\n",
 	}))
-	api := router.Group(cfg.ApiPrefix + ApiVersion)
 
 	db, err := bolt.Open(cfg.DbFile, 0600, nil)
 	if err != nil {
@@ -44,42 +48,27 @@ func Runserver(cfg *cfg.Config, args []string) error {
 	}
 	defer db.Close()
 
+	api := router.Group(cfg.ApiPrefix + ApiVersion)
 	{
-		api.Post("/file/put", func(c *fiber.Ctx) error {
-			uri, err := Putfile(c, cfg, db)
-
-			if err != nil {
-				return c.Status(fiber.StatusBadRequest).JSON(Result{
-					Code:    fiber.StatusBadRequest,
-					Message: err.Error(),
-					Success: false,
-				})
-			} else {
-				return c.Status(fiber.StatusOK).JSON(Result{
-					Code:    fiber.StatusOK,
-					Message: uri,
-					Success: true,
-				})
-			}
+		api.Post("/file/", func(c *fiber.Ctx) error {
+			msg, err := FilePut(c, cfg, db)
+			return SendResponse(c, msg, err)
 		})
 
-		api.Get("/file/get/:id/:file", func(c *fiber.Ctx) error {
-			// deliver  a file and delete  it after a delay  (FIXME: check
-			// when gin  has done delivering it?). Redirect  to the static
-			// handler for actual delivery.
-			id := c.Params("id")
-			file := c.Params("file")
+		api.Get("/file/:id/:file", func(c *fiber.Ctx) error {
+			return FileGet(c, cfg, db)
+		})
 
-			filename := filepath.Join(dst, id, file)
+		api.Get("/file/:id/", func(c *fiber.Ctx) error {
+			return FileGet(c, cfg, db)
+		})
 
-			if _, err := os.Stat(filename); err == nil {
-				go func() {
-					time.Sleep(500 * time.Millisecond)
-					cleanup(filepath.Join(dst, id))
-				}()
-			}
+		api.Delete("/file/:id/", func(c *fiber.Ctx) error {
+			return FileDelete(c, cfg, db)
+		})
 
-			return c.Download(filename, file)
+		api.Delete("/file/", func(c *fiber.Ctx) error {
+			return FileDelete(c, cfg, db)
 		})
 	}
 
@@ -87,7 +76,22 @@ func Runserver(cfg *cfg.Config, args []string) error {
 		return c.Send([]byte("welcome to upload api, use /api enpoint!"))
 	})
 
-	router.Listen(cfg.Listen)
+	return router.Listen(cfg.Listen)
 
-	return nil
+}
+
+func SendResponse(c *fiber.Ctx, msg string, err error) error {
+	if err != nil {
+		return c.Status(fiber.StatusBadRequest).JSON(Result{
+			Code:    fiber.StatusBadRequest,
+			Message: err.Error(),
+			Success: false,
+		})
+	}
+
+	return c.Status(fiber.StatusOK).JSON(Result{
+		Code:    fiber.StatusOK,
+		Message: msg,
+		Success: true,
+	})
 }

upd/cfg/config.go

@@ -32,6 +32,15 @@ type Config struct {
 	StorageDir string
 	Url        string
 	DbFile     string
+
+	// fiber settings, see:
+	// https://docs.gofiber.io/api/fiber/#config
+	Prefork   bool
+	AppName   string
+	BodyLimit int
+	V4only    bool
+	V6only    bool
+	Network   string
 }
 
 func Getversion() string {
@@ -44,6 +53,10 @@ func Getversion() string {
 	return fmt.Sprintf("This is upd version %s", VERSION)
 }
 
+func (c *Config) GetVersion() string {
+	return VERSION
+}
+
 // post processing of options, if any
 func (c *Config) ApplyDefaults() {
 	if len(c.Url) == 0 {
@@ -53,4 +66,17 @@ func (c *Config) ApplyDefaults() {
 			c.Url = "http://" + c.Listen
 		}
 	}
+
+	switch {
+	case c.V4only:
+		c.Network = "tcp4"
+	case c.V6only:
+		c.Network = "tcp6"
+	default:
+		if c.Prefork {
+			c.Network = "tcp4"
+		} else {
+			c.Network = "tcp" // dual stack
+		}
+	}
 }

upd/cmd/root.go

@@ -82,12 +82,21 @@ func Execute() {
 	rootCmd.PersistentFlags().BoolVarP(&ShowVersion, "version", "v", false, "Print program version")
 	rootCmd.PersistentFlags().StringVarP(&cfgFile, "config", "c", "", "custom config file")
 	rootCmd.PersistentFlags().BoolVarP(&conf.Debug, "debug", "d", false, "Enable debugging")
-	rootCmd.PersistentFlags().StringVarP(&conf.Listen, "listen", "l", ":8080", "listen to custom ip:port")
+	rootCmd.PersistentFlags().StringVarP(&conf.Listen, "listen", "l", ":8080", "listen to custom ip:port (use [ip]:port for ipv6)")
 	rootCmd.PersistentFlags().StringVarP(&conf.StorageDir, "storagedir", "s", "/tmp", "storage directory for uploaded files")
 	rootCmd.PersistentFlags().StringVarP(&conf.ApiPrefix, "apiprefix", "a", "/api", "API endpoint path")
 	rootCmd.PersistentFlags().StringVarP(&conf.Url, "url", "u", "", "HTTP endpoint w/o path")
 	rootCmd.PersistentFlags().StringVarP(&conf.DbFile, "dbfile", "D", "/tmp/uploads.db", "Bold database file to use")
 
+	// server settings
+	rootCmd.PersistentFlags().BoolVarP(&conf.V4only, "ipv4", "4", false, "Only listen on ipv4")
+	rootCmd.PersistentFlags().BoolVarP(&conf.V6only, "ipv6", "6", false, "Only listen on ipv6")
+	rootCmd.MarkFlagsMutuallyExclusive("ipv4", "ipv6")
+
+	rootCmd.PersistentFlags().BoolVarP(&conf.Prefork, "prefork", "p", false, "Prefork server threads")
+	rootCmd.PersistentFlags().StringVarP(&conf.AppName, "appname", "n", "upd "+conf.GetVersion(), "App name to say hi as")
+	rootCmd.PersistentFlags().IntVarP(&conf.BodyLimit, "bodylimit", "b", 10250000000, "Max allowed upload size in bytes")
+
 	err := rootCmd.Execute()
 	if err != nil {
 		os.Exit(1)

upd/testzip.go

@@ -1,18 +0,0 @@
-package main
-
-import (
-	"github.com/tlinden/up/upd/api"
-	"log"
-	"os"
-)
-
-func main() {
-	if len(os.Args) > 2 {
-		dir, err := os.Getwd()
-		if err != nil {
-		}
-		if err := api.ZipSource(dir+"/"+os.Args[1], os.Args[2]); err != nil {
-			log.Fatal(err)
-		}
-	}
-}