scip/ephemerup
1
0
Fork 0
mirror of https://codeberg.org/scip/ephemerup.git synced 2025-12-17 12:40:57 +01:00
Code Issues Packages Projects Releases Wiki Activity

changes:

Browse Source 
- added cleaner goroutine
- added delete cmd
- added list cmd
- refactoring
This commit is contained in:
Thomas von Dein
2023-03-08 19:31:42 +01:00
parent 50660da26a
commit 01a0dc054d
17 changed files with 506 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
upd/api/cleaner.go Normal file
View File

@@ -0,0 +1,85 @@
/*
Copyright © 2023 Thomas von Dein
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package api
import (
"fmt"
//"github.com/alecthomas/repr"
"encoding/json"
"github.com/tlinden/up/upd/cfg"
bolt "go.etcd.io/bbolt"
"path/filepath"
"time"
)
func DeleteExpiredUploads(conf *cfg.Config, db *Db) error {
err := db.bolt.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(Bucket))
if bucket == nil {
return nil // nothin to delete so far
}
err := bucket.ForEach(func(id, j []byte) error {
upload := &Upload{}
if err := json.Unmarshal(j, &upload); err != nil {
return fmt.Errorf("unable to unmarshal json: %s", err)
}
if IsExpired(conf, upload.Uploaded.Time, upload.Expire) {
if err := bucket.Delete([]byte(id)); err != nil {
return nil
}
cleanup(filepath.Join(conf.StorageDir, upload.Id))
Log("Cleaned up upload " + upload.Id)
}
return nil
})
return err
})
if err != nil {
Log("DB error: %s", err.Error())
}
return err
}
func BackgroundCleaner(conf *cfg.Config, db *Db) chan bool {
ticker := time.NewTicker(conf.CleanInterval)
fmt.Println(conf.CleanInterval)
done := make(chan bool)
go func() {
for {
select {
case <-ticker.C:
DeleteExpiredUploads(conf, db)
case <-done:
ticker.Stop()
return
}
}
}()
return done
}

46
upd/api/common.go
View File

@@ -20,6 +20,7 @@ package api
import (
"errors"
"fmt"
"github.com/tlinden/up/upd/cfg"
"regexp"
"strconv"
"time"
@@ -39,6 +40,29 @@ type Meta struct {
Expire string `json:"expire" form:"expire"`
}
// stores 1 upload object, gets into db
type Upload struct {
Id string `json:"id"`
Expire string `json:"expire"`
File string `json:"file"` // final filename (visible to the downloader)
Members []string `json:"members"` // contains multiple files, so File is an archive
Uploaded Timestamp `json:"uploaded"`
Context string `json:"context"`
}
// this one is also used for marshalling to the client
type Uploads struct {
Entries []*Upload `json:"uploads"`
// integrate the Result struct so we can signal success
Result
}
// incoming id
type Id struct {
Id string `json:"name" xml:"name" form:"name"`
}
// vaious helbers
func Log(format string, values ...any) {
fmt.Printf("[DEBUG] "+format+"\n", values...)
@@ -49,12 +73,6 @@ func Ts() string {
return t.Format("2006-01-02-15-04-")
}
func NormalizeFilename(file string) string {
r := regexp.MustCompile(`[^\w\d\-_\\.]`)
return Ts() + r.ReplaceAllString(file, "")
}
/*
We could use time.ParseDuration(), but this doesn't support days.
@@ -96,9 +114,16 @@ func duration2int(duration string) int {
aka:
if(now - start) >= duration { time is up}
*/
func IsExpired(start time.Time, duration string) bool {
func IsExpired(conf *cfg.Config, start time.Time, duration string) bool {
var expiretime int // seconds
now := time.Now()
expiretime := duration2int(duration)
if duration == "asap" {
expiretime = conf.DefaultExpire
} else {
expiretime = duration2int(duration)
}
if now.Unix()-start.Unix() >= int64(expiretime) {
return true
@@ -120,9 +145,8 @@ func IsExpired(start time.Time, duration string) bool {
it. You may ignore the error and use the untainted string or bail
out.
*/
func Untaint(input string, wanted string) (string, error) {
re := regexp.MustCompile(wanted)
untainted := re.ReplaceAllString(input, "")
func Untaint(input string, wanted *regexp.Regexp) (string, error) {
untainted := wanted.ReplaceAllString(input, "")
if len(untainted) != len(input) {
return untainted, errors.New("Invalid input string!")

29
upd/api/db.go
View File

@@ -20,7 +20,7 @@ package api
import (
"encoding/json"
"fmt"
"github.com/alecthomas/repr"
//"github.com/alecthomas/repr"
bolt "go.etcd.io/bbolt"
)
@@ -31,20 +31,6 @@ type Db struct {
bolt *bolt.DB
}
// stores 1 upload object, gets into db
type Upload struct {
Id string `json:"id"`
Expire string `json:"expire"`
File string `json:"file"` // final filename (visible to the downloader)
Members []string `json:"members"` // contains multiple files, so File is an archive
Uploaded Timestamp `json:"uploaded"`
Context string `json:"context"`
}
type Uploads struct {
Entries []*Upload `json:"uploads"`
}
func NewDb(file string) (*Db, error) {
b, err := bolt.Open(file, 0600, nil)
db := Db{bolt: b}
@@ -89,6 +75,11 @@ func (db *Db) Lookup(id string) (Upload, error) {
err := db.bolt.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(Bucket))
if bucket == nil {
return fmt.Errorf("id %s not found", id)
}
j := bucket.Get([]byte(id))
if len(j) == 0 {
@@ -114,6 +105,10 @@ func (db *Db) Delete(id string) error {
err := db.bolt.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(Bucket))
if bucket == nil {
return fmt.Errorf("id %s not found", id)
}
j := bucket.Get([]byte(id))
if len(j) == 0 {
@@ -136,6 +131,10 @@ func (db *Db) List(apicontext string) (*Uploads, error) {
err := db.bolt.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(Bucket))
if bucket == nil {
return nil
}
err := bucket.ForEach(func(id, j []byte) error {
upload := &Upload{}
if err := json.Unmarshal(j, &upload); err != nil {

2
upd/api/fileio.go
View File

@@ -43,7 +43,7 @@ func cleanup(dir string) {
func SaveFormFiles(c *fiber.Ctx, cfg *cfg.Config, files []*multipart.FileHeader, id string) ([]string, error) {
members := []string{}
for _, file := range files {
filename := NormalizeFilename(filepath.Base(file.Filename))
filename, _ := Untaint(filepath.Base(file.Filename), cfg.RegNormalizedFilename)
path := filepath.Join(cfg.StorageDir, id, filename)
members = append(members, filename)
Log("Received: %s => %s/%s", file.Filename, id, filename)

36
upd/api/handlers.go
View File

@@ -23,7 +23,6 @@ import (
"github.com/google/uuid"
"github.com/tlinden/up/upd/cfg"
"encoding/json"
"os"
"path/filepath"
"time"
@@ -77,7 +76,7 @@ func FilePut(c *fiber.Ctx, cfg *cfg.Config, db *Db) (string, error) {
if len(formdata.Expire) == 0 {
entry.Expire = "asap"
} else {
ex, err := Untaint(formdata.Expire, `[^dhms0-9]`) // duration or asap allowed
ex, err := Untaint(formdata.Expire, cfg.RegDuration) // duration or asap allowed
if err != nil {
return "", err
}
@@ -119,7 +118,7 @@ func FileGet(c *fiber.Ctx, cfg *cfg.Config, db *Db, shallExpire ...bool) error {
// we ignore c.Params("file"), cause it may be malign. Also we've
// got it in the db anyway
id, err := Untaint(c.Params("id"), `[^a-zA-Z0-9\-]`)
id, err := Untaint(c.Params("id"), cfg.RegKey)
if err != nil {
return fiber.NewError(403, "Invalid id provided!")
}
@@ -157,14 +156,10 @@ func FileGet(c *fiber.Ctx, cfg *cfg.Config, db *Db, shallExpire ...bool) error {
return err
}
type Id struct {
Id string `json:"name" xml:"name" form:"name"`
}
// delete file, id dir and db entry
func DeleteUpload(c *fiber.Ctx, cfg *cfg.Config, db *Db) error {
func FileDelete(c *fiber.Ctx, cfg *cfg.Config, db *Db) error {
// delete file, id dir and db entry
id, err := Untaint(c.Params("id"), `[^a-zA-Z0-9\-]`)
id, err := Untaint(c.Params("id"), cfg.RegKey)
if err != nil {
return fiber.NewError(403, "Invalid id provided!")
}
@@ -184,23 +179,24 @@ func FileDelete(c *fiber.Ctx, cfg *cfg.Config, db *Db) error {
return nil
}
func List(c *fiber.Ctx, cfg *cfg.Config, db *Db) (string, error) {
apicontext, err := Untaint(c.Params("apicontext"), `[^a-zA-Z0-9\-]`)
// returns the whole list + error code, no post processing by server
func List(c *fiber.Ctx, cfg *cfg.Config, db *Db) error {
apicontext, err := Untaint(c.Params("apicontext"), cfg.RegKey)
if err != nil {
return "", fiber.NewError(403, "Invalid api context provided!")
return JsonStatus(c, fiber.StatusForbidden,
"Invalid api context provided!")
}
uploads, err := db.List(apicontext)
repr.Print(uploads)
if err != nil {
return "", fiber.NewError(500, "Unable to list uploads: "+err.Error())
return JsonStatus(c, fiber.StatusForbidden,
"Unable to list uploads: "+err.Error())
}
jsonlist, err := json.Marshal(uploads)
if err != nil {
return "", fiber.NewError(500, "json marshalling failure: "+err.Error())
}
// if we reached this point we can signal success
uploads.Success = true
uploads.Code = fiber.StatusOK
Log(string(jsonlist))
return string(jsonlist), nil
return c.Status(fiber.StatusOK).JSON(uploads)
}

148
upd/api/server.go
View File

@@ -20,6 +20,8 @@ package api
import (
"errors"
"github.com/gofiber/fiber/v2"
"github.com/gofiber/fiber/v2/middleware/compress"
"github.com/gofiber/fiber/v2/middleware/cors"
"github.com/gofiber/fiber/v2/middleware/logger"
"github.com/gofiber/fiber/v2/middleware/requestid"
"github.com/gofiber/fiber/v2/middleware/session"
@@ -30,84 +32,140 @@ import (
// sessions are context specific and can be global savely
var Sessionstore *session.Store
func Runserver(cfg *cfg.Config, args []string) error {
const shallExpire = true
func Runserver(conf *cfg.Config, args []string) error {
// required for authenticated routes, used to store the api context
Sessionstore = session.New()
router := fiber.New(fiber.Config{
CaseSensitive: true,
StrictRouting: true,
Immutable: true,
Prefork: cfg.Prefork,
ServerHeader: "upd",
AppName: cfg.AppName,
BodyLimit: cfg.BodyLimit,
Network: cfg.Network,
})
router.Use(requestid.New())
router.Use(logger.New(logger.Config{
Format: "${pid} ${locals:requestid} ${status} - ${method} ${path}​\n",
}))
db, err := NewDb(cfg.DbFile)
// bbolt db setup
db, err := NewDb(conf.DbFile)
if err != nil {
return err
}
defer db.Close()
AuthSetEndpoints(cfg.ApiPrefix, ApiVersion, []string{"/file"})
AuthSetApikeys(cfg.Apicontext)
// setup authenticated endpoints
auth := SetupAuthStore(conf)
auth := keyauth.New(keyauth.Config{
Validator: AuthValidateAPIKey,
ErrorHandler: AuthErrHandler,
})
// setup api server
router := SetupServer(conf)
shallExpire := true
api := router.Group(cfg.ApiPrefix + ApiVersion)
// authenticated routes
api := router.Group(conf.ApiPrefix + ApiVersion)
{
// authenticated routes
api.Post("/file/", auth, func(c *fiber.Ctx) error {
msg, err := FilePut(c, cfg, db)
msg, err := FilePut(c, conf, db)
return SendResponse(c, msg, err)
})
api.Get("/file/:id/:file", auth, func(c *fiber.Ctx) error {
return FileGet(c, cfg, db)
return FileGet(c, conf, db)
})
api.Get("/file/:id/", auth, func(c *fiber.Ctx) error {
return FileGet(c, cfg, db)
return FileGet(c, conf, db)
})
api.Delete("/file/:id/", auth, func(c *fiber.Ctx) error {
return FileDelete(c, cfg, db)
err := DeleteUpload(c, conf, db)
return SendResponse(c, "", err)
})
api.Get("/list/", auth, func(c *fiber.Ctx) error {
msg, err := List(c, cfg, db)
return SendResponse(c, msg, err)
return List(c, conf, db)
})
}
// public routes
router.Get("/", func(c *fiber.Ctx) error {
return c.Send([]byte("welcome to upload api, use /api enpoint!"))
{
router.Get("/", func(c *fiber.Ctx) error {
return c.Send([]byte("welcome to upload api, use /api enpoint!"))
})
router.Get("/download/:id/:file", func(c *fiber.Ctx) error {
return FileGet(c, conf, db, shallExpire)
})
router.Get("/download/:id/", func(c *fiber.Ctx) error {
return FileGet(c, conf, db, shallExpire)
})
}
// setup cleaner
quitcleaner := BackgroundCleaner(conf, db)
router.Hooks().OnShutdown(func() error {
Log("Shutting down cleaner")
close(quitcleaner)
return nil
})
router.Get("/download/:id/:file", func(c *fiber.Ctx) error {
return FileGet(c, cfg, db, shallExpire)
})
router.Get("/download/:id/", func(c *fiber.Ctx) error {
return FileGet(c, cfg, db, shallExpire)
})
return router.Listen(cfg.Listen)
return router.Listen(conf.Listen)
}
func SetupAuthStore(conf *cfg.Config) func(*fiber.Ctx) error {
AuthSetEndpoints(conf.ApiPrefix, ApiVersion, []string{"/file"})
AuthSetApikeys(conf.Apicontext)
return keyauth.New(keyauth.Config{
Validator: AuthValidateAPIKey,
ErrorHandler: AuthErrHandler,
})
}
func SetupServer(conf *cfg.Config) *fiber.App {
router := fiber.New(fiber.Config{
CaseSensitive: true,
StrictRouting: true,
Immutable: true,
Prefork: conf.Prefork,
ServerHeader: "upd",
AppName: conf.AppName,
BodyLimit: conf.BodyLimit,
Network: conf.Network,
})
router.Use(requestid.New())
router.Use(logger.New(logger.Config{
Format: "${pid} ${locals:requestid} ${status} - ${method} ${path}​\n",
}))
router.Use(cors.New(cors.Config{
AllowMethods: "GET,PUT,POST,DELETE",
ExposeHeaders: "Content-Type,Authorization,Accept",
}))
router.Use(compress.New(compress.Config{
Level: compress.LevelBestSpeed,
}))
return router
}
/*
Wrapper to respond with proper json status, message and code,
shall be prepared and called by the handlers directly.
*/
func JsonStatus(c *fiber.Ctx, code int, msg string) error {
success := true
if code != fiber.StatusOK {
success = false
}
return c.Status(code).JSON(Result{
Code: code,
Message: msg,
Success: success,
})
}
/*
Used for non json-aware handlers, called by server
*/
func SendResponse(c *fiber.Ctx, msg string, err error) error {
if err != nil {
code := fiber.StatusInternalServerError

16
upd/cfg/config.go
View File

@@ -18,7 +18,9 @@ package cfg
import (
"fmt"
"regexp"
"strings"
"time"
)
const Version string = "v0.0.1"
@@ -50,6 +52,13 @@ type Config struct {
// only settable via config
Apicontext []Apicontext `koanf:"apicontext"`
// Internals only
RegNormalizedFilename *regexp.Regexp
RegDuration *regexp.Regexp
RegKey *regexp.Regexp
CleanInterval time.Duration
DefaultExpire int
}
func Getversion() string {
@@ -88,4 +97,11 @@ func (c *Config) ApplyDefaults() {
c.Network = "tcp" // dual stack
}
}
c.RegNormalizedFilename = regexp.MustCompile(`[^\w\d\-_\.]`)
c.RegDuration = regexp.MustCompile(`[^dhms0-9]`)
c.RegKey = regexp.MustCompile(`[^a-zA-Z0-9\-]`)
c.CleanInterval = 10 * time.Second
c.DefaultExpire = 30 * 86400 // 1 month
}