couple of enhancements, reorg etc

upd/api/api.go

@@ -0,0 +1,63 @@
+/*
+Copyright © 2023 Thomas von Dein
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package api
+
+import (
+	"fmt"
+	"regexp"
+	"time"
+)
+
+const ApiVersion string = "/v1"
+
+// used to return to the api client
+type Result struct {
+	Success bool
+	Message string
+	Code    int
+}
+
+// Binding from JSON, data coming from user, not tainted
+type Meta struct {
+	Expire string `json:"expire" form:"expire"`
+}
+
+// stores 1 upload object, gets into db
+type Upload struct {
+	Id       string    `json:"id"`
+	Expire   string    `json:"expire"`
+	File     string    `json:"file"`    // final filename (visible to the downloader)
+	Members  []string  `json:"members"` // contains multiple files, so File is an archive
+	Uploaded time.Time `json:"uploaded"`
+}
+
+// vaious helbers
+func Log(format string, values ...any) {
+	fmt.Printf("[DEBUG] "+format+"\n", values...)
+}
+
+func Ts() string {
+	t := time.Now()
+	return t.Format("2006-01-02-15-04-")
+}
+
+func NormalizeFilename(file string) string {
+	r := regexp.MustCompile(`[^\w\d\-_\\.]`)
+
+	return Ts() + r.ReplaceAllString(file, "")
+}

upd/api/fileio.go

@@ -0,0 +1,114 @@
+/*
+Copyright © 2023 Thomas von Dein
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package api
+
+import (
+	"archive/zip"
+	"io"
+	"os"
+	"path/filepath"
+	"sync"
+)
+
+// cleanup an upload directory, either because  we got an error in the
+// middle of an upload or something else  went wrong. we fork off a go
+// routine because this may block.
+func cleanup(dir string) {
+	go func() {
+		err := os.RemoveAll(dir)
+		if err != nil {
+			Log("Failed to remove dir %s: %s", dir, err.Error())
+		}
+	}()
+}
+
+func ZipSource(source, target string) error {
+	// 1. Create a ZIP file and zip.Writer
+	// source must be an absolute path, target a zip file
+	f, err := os.Create(target)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	writer := zip.NewWriter(f)
+	defer writer.Close()
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+
+	// don't chdir the server itself
+	go func() {
+		defer wg.Done()
+
+		os.Chdir(source)
+		newDir, err := os.Getwd()
+		if err != nil {
+		}
+		//Log("Current Working Direcotry: %s\n, source: %s", newDir, source)
+
+		err = filepath.Walk(".", func(path string, info os.FileInfo, err error) error {
+			// 2. Go through all the files of the source
+			if err != nil {
+				return err
+			}
+
+			// 3. Create a local file header
+			header, err := zip.FileInfoHeader(info)
+			if err != nil {
+				return err
+			}
+
+			// set compression
+			header.Method = zip.Deflate
+
+			// 4. Set relative path of a file as the header name
+			header.Name = path
+			//Log("a: <%s>, b: <%s>, rel: <%s>", filepath.Dir(source), path, header.Name)
+			if err != nil {
+				return err
+			}
+			if info.IsDir() {
+				header.Name += "/"
+			}
+
+			// 5. Create writer for the file header and save content of the file
+			headerWriter, err := writer.CreateHeader(header)
+			if err != nil {
+				return err
+			}
+
+			if info.IsDir() {
+				return nil
+			}
+
+			f, err := os.Open(path)
+			if err != nil {
+				return err
+			}
+			defer f.Close()
+
+			_, err = io.Copy(headerWriter, f)
+			return err
+		})
+	}()
+
+	wg.Wait()
+
+	return err
+}

upd/api/handlers.go

@@ -0,0 +1,162 @@
+/*
+Copyright © 2023 Thomas von Dein
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package api
+
+import (
+	//"archive/zip"
+	"fmt"
+	//"github.com/gin-gonic/gin"
+	//"github.com/gin-gonic/gin/binding"
+	"encoding/json"
+	"github.com/gofiber/fiber/v2"
+	"github.com/google/uuid"
+	"github.com/tlinden/up/upd/cfg"
+	bolt "go.etcd.io/bbolt"
+	//"io"
+	// "net/http"
+	"os"
+	"path/filepath"
+	//"regexp"
+	"strings"
+	"time"
+)
+
+func Putfile(c *fiber.Ctx, cfg *cfg.Config, db *bolt.DB) (string, error) {
+	// supports upload of multiple files with:
+	//
+	// curl -X POST localhost:8080/putfile \
+	//   -F "upload[]=@/home/scip/2023-02-06_10-51.png" \
+	//   -F "upload[]=@/home/scip/pgstat.png" \
+	//   -H "Content-Type: multipart/form-data"
+	//
+	// If multiple files are  uploaded, they are zipped, otherwise
+	// the  file is being stored  as is.
+	//
+	// Returns the  name of the uploaded file.
+	//
+	// FIXME: normalize or rename filename of single file to avoid dumb file names
+	id := uuid.NewString()
+
+	var returnUrl string
+	var formdata Meta
+
+	os.MkdirAll(filepath.Join(cfg.StorageDir, id), os.ModePerm)
+
+	// fetch auxiliary form data
+	form, err := c.MultipartForm()
+	if err != nil {
+		Log("multipart error %s", err.Error())
+		return "", err
+	}
+
+	entry := &Upload{Id: id, Uploaded: time.Now()}
+
+	// init upload obj
+
+	// retrieve files, if any
+	files := form.File["upload[]"]
+	for _, file := range files {
+		filename := NormalizeFilename(filepath.Base(file.Filename))
+		path := filepath.Join(cfg.StorageDir, id, filename)
+		entry.Members = append(entry.Members, filename)
+		Log("Received: %s => %s/%s", file.Filename, id, filename)
+
+		if err := c.SaveFile(file, path); err != nil {
+			cleanup(filepath.Join(cfg.StorageDir, id))
+			return "", err
+		}
+	}
+
+	if err := c.BodyParser(&formdata); err != nil {
+		Log("bodyparser error %s", err.Error())
+		return "", err
+	}
+
+	if len(formdata.Expire) == 0 {
+		entry.Expire = "asap"
+	} else {
+		entry.Expire = formdata.Expire // FIXME: validate
+	}
+
+	if len(entry.Members) == 1 {
+		returnUrl = cfg.Url + cfg.ApiPrefix + "/getfile/" + id + "/" + entry.Members[0]
+		entry.File = entry.Members[0]
+	} else {
+		zipfile := Ts() + "data.zip"
+		tmpzip := filepath.Join(cfg.StorageDir, zipfile)
+		finalzip := filepath.Join(cfg.StorageDir, id, zipfile)
+		iddir := filepath.Join(cfg.StorageDir, id)
+
+		if err := ZipSource(iddir, tmpzip); err != nil {
+			cleanup(iddir)
+			Log("zip error")
+			return "", err
+		}
+
+		if err := os.Rename(tmpzip, finalzip); err != nil {
+			cleanup(iddir)
+			return "", err
+		}
+
+		returnUrl = strings.Join([]string{cfg.Url + cfg.ApiPrefix + ApiVersion, "file/get", id, zipfile}, "/")
+		entry.File = zipfile
+
+		// clean up after us
+		go func() {
+			for _, file := range entry.Members {
+				if err := os.Remove(filepath.Join(cfg.StorageDir, id, file)); err != nil {
+					Log("ERROR: unable to delete %s: %s", file, err)
+				}
+			}
+		}()
+
+	}
+
+	Log("Now serving %s from %s/%s", returnUrl, cfg.StorageDir, id)
+	Log("Expire set to: %s", entry.Expire)
+
+	go func() {
+		// => db.go !
+		err := db.Update(func(tx *bolt.Tx) error {
+			b, err := tx.CreateBucketIfNotExists([]byte("uploads"))
+			if err != nil {
+				return fmt.Errorf("create bucket: %s", err)
+			}
+
+			jsonentry, err := json.Marshal(entry)
+			if err != nil {
+				return fmt.Errorf("json marshalling failure: %s", err)
+			}
+
+			err = b.Put([]byte(id), []byte(jsonentry))
+			if err != nil {
+				return fmt.Errorf("insert data: %s", err)
+			}
+
+			// results in:
+			// bbolt get /tmp/uploads.db uploads fb242922-86cb-43a8-92bc-b027c35f0586
+			// {"id":"fb242922-86cb-43a8-92bc-b027c35f0586","expire":"1d","file":"2023-02-17-13-09-data.zip"}
+			return nil
+		})
+		if err != nil {
+			Log("DB error: %s", err.Error())
+		}
+	}()
+
+	return returnUrl, nil
+}

upd/api/server.go

@@ -0,0 +1,93 @@
+/*
+Copyright © 2023 Thomas von Dein
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package api
+
+import (
+	"github.com/gofiber/fiber/v2"
+	"github.com/gofiber/fiber/v2/middleware/logger"
+	"github.com/gofiber/fiber/v2/middleware/requestid"
+	"github.com/tlinden/up/upd/cfg"
+	bolt "go.etcd.io/bbolt"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+func Runserver(cfg *cfg.Config, args []string) error {
+	dst := cfg.StorageDir
+	router := fiber.New()
+	router.Use(requestid.New())
+	router.Use(logger.New(logger.Config{
+		// For more options, see the Config section
+		Format: "${pid} ${locals:requestid} ${status} - ${method} ${path}​\n",
+	}))
+	api := router.Group(cfg.ApiPrefix + ApiVersion)
+
+	db, err := bolt.Open(cfg.DbFile, 0600, nil)
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+
+	{
+		api.Post("/file/put", func(c *fiber.Ctx) error {
+			uri, err := Putfile(c, cfg, db)
+
+			if err != nil {
+				return c.Status(fiber.StatusBadRequest).JSON(Result{
+					Code:    fiber.StatusBadRequest,
+					Message: err.Error(),
+					Success: false,
+				})
+			} else {
+				return c.Status(fiber.StatusOK).JSON(Result{
+					Code:    fiber.StatusOK,
+					Message: uri,
+					Success: true,
+				})
+			}
+		})
+
+		api.Get("/file/get/:id/:file", func(c *fiber.Ctx) error {
+			// deliver  a file and delete  it after a delay  (FIXME: check
+			// when gin  has done delivering it?). Redirect  to the static
+			// handler for actual delivery.
+			id := c.Params("id")
+			file := c.Params("file")
+
+			filename := filepath.Join(dst, id, file)
+
+			if _, err := os.Stat(filename); err == nil {
+				go func() {
+					time.Sleep(500 * time.Millisecond)
+					cleanup(filepath.Join(dst, id))
+				}()
+			}
+
+			return c.Download(filename, file)
+		})
+	}
+
+	router.Get("/", func(c *fiber.Ctx) error {
+		return c.Send([]byte("welcome to upload api, use /api enpoint!"))
+	})
+
+	router.Listen(cfg.Listen)
+
+	return nil
+}