scip/ephemerup
1
0
Fork 0
mirror of https://codeberg.org/scip/ephemerup.git synced 2025-12-16 20:20:58 +01:00
Code Issues Packages Projects Releases Wiki Activity

generalized DB engine to support multiple types (upload+form)

Browse Source
This commit is contained in:
Thomas von Dein
2023-03-25 16:20:42 +01:00
parent b6dfafd1c1
commit d028fb2db1
6 changed files with 432 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
api/db.go
View File

@@ -18,7 +18,6 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package api
import (
"encoding/json"
"fmt"
"github.com/tlinden/cenophane/cfg"
"github.com/tlinden/cenophane/common"
@@ -26,7 +25,7 @@ import (
bolt "go.etcd.io/bbolt"
)
const Bucket string = "uploads"
const Bucket string = "data"
// wrapper for bolt db
type Db struct {
@@ -44,14 +43,14 @@ func (db *Db) Close() {
db.bolt.Close()
}
func (db *Db) Insert(id string, entry *common.Upload) error {
func (db *Db) Insert(id string, entry common.Dbentry) error {
err := db.bolt.Update(func(tx *bolt.Tx) error {
bucket, err := tx.CreateBucketIfNotExists([]byte(Bucket))
if err != nil {
return fmt.Errorf("create bucket: %s", err)
}
jsonentry, err := json.Marshal(entry)
jsonentry, err := entry.Marshal()
if err != nil {
return fmt.Errorf("json marshalling failure: %s", err)
}
@@ -61,9 +60,6 @@ func (db *Db) Insert(id string, entry *common.Upload) error {
return fmt.Errorf("insert data: %s", err)
}
// results in:
// bbolt get /tmp/uploads.db uploads fb242922-86cb-43a8-92bc-b027c35f0586
// {"id":"fb242922-86cb-43a8-92bc-b027c35f0586","expire":"1d","file":"2023-02-17-13-09-data.zip"}
return nil
})
if err != nil {
@@ -87,12 +83,12 @@ func (db *Db) Delete(apicontext string, id string) error {
return fmt.Errorf("id %s not found", id)
}
upload := &common.Upload{}
if err := json.Unmarshal(j, &upload); err != nil {
entryContext, err := common.GetContext(j)
if err != nil {
return fmt.Errorf("unable to unmarshal json: %s", err)
}
if (apicontext != "" && (db.cfg.Super == apicontext || upload.Context == apicontext)) || apicontext == "" {
if (apicontext != "" && (db.cfg.Super == apicontext || entryContext == apicontext)) || apicontext == "" {
return bucket.Delete([]byte(id))
}
@@ -106,8 +102,8 @@ func (db *Db) Delete(apicontext string, id string) error {
return err
}
func (db *Db) UploadsList(apicontext string, filter string) (*common.Uploads, error) {
uploads := &common.Uploads{}
func (db *Db) UploadsList(apicontext string, filter string, t int) (*common.Response, error) {
response := &common.Response{}
err := db.bolt.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(Bucket))
@@ -116,24 +112,39 @@ func (db *Db) UploadsList(apicontext string, filter string) (*common.Uploads, er
}
err := bucket.ForEach(func(id, j []byte) error {
upload := &common.Upload{}
if err := json.Unmarshal(j, &upload); err != nil {
entry, err := common.Unmarshal(j, t)
if err != nil {
return fmt.Errorf("unable to unmarshal json: %s", err)
}
var entryContext string
if t == common.TypeUpload {
entryContext = entry.(common.Upload).Context
} else {
entryContext = entry.(common.Form).Context
}
fmt.Printf("apicontext: %s, filter: %s\n", apicontext, filter)
if apicontext != "" && db.cfg.Super != apicontext {
// only return the uploads for this context
if apicontext == upload.Context {
if apicontext == entryContext {
// unless a filter needed OR no filter specified
if (filter != "" && upload.Context == filter) || filter == "" {
uploads.Entries = append(uploads.Entries, upload)
if (filter != "" && entryContext == filter) || filter == "" {
if t == common.TypeUpload {
response.Uploads = append(response.Uploads, entry.(*common.Upload))
} else {
response.Forms = append(response.Forms, entry.(*common.Form))
}
}
}
} else {
// return all, because we operate a public service or current==super
if (filter != "" && upload.Context == filter) || filter == "" {
uploads.Entries = append(uploads.Entries, upload)
if (filter != "" && entryContext == filter) || filter == "" {
if t == common.TypeUpload {
response.Uploads = append(response.Uploads, entry.(*common.Upload))
} else {
response.Forms = append(response.Forms, entry.(*common.Form))
}
}
}
@@ -143,12 +154,12 @@ func (db *Db) UploadsList(apicontext string, filter string) (*common.Uploads, er
return err // might be nil as well
})
return uploads, err
return response, err
}
// we only return one obj here, but could return more later
func (db *Db) Get(apicontext string, id string) (*common.Uploads, error) {
uploads := &common.Uploads{}
func (db *Db) Get(apicontext string, id string, t int) (*common.Response, error) {
response := &common.Response{}
err := db.bolt.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(Bucket))
@@ -161,35 +172,46 @@ func (db *Db) Get(apicontext string, id string) (*common.Uploads, error) {
return fmt.Errorf("No upload object found with id %s", id)
}
upload := &common.Upload{}
if err := json.Unmarshal(j, &upload); err != nil {
entry, err := common.Unmarshal(j, t)
if err != nil {
return fmt.Errorf("unable to unmarshal json: %s", err)
}
if (apicontext != "" && (db.cfg.Super == apicontext || upload.Context == apicontext)) || apicontext == "" {
var entryContext string
if t == common.TypeUpload {
entryContext = entry.(common.Upload).Context
} else {
entryContext = entry.(common.Form).Context
}
if (apicontext != "" && (db.cfg.Super == apicontext || entryContext == apicontext)) || apicontext == "" {
// allowed if no context (public or download)
// or if context matches or if context==super
uploads.Entries = append(uploads.Entries, upload)
if t == common.TypeUpload {
response.Uploads = append(response.Uploads, entry.(*common.Upload))
} else {
response.Forms = append(response.Forms, entry.(*common.Form))
}
}
return nil
})
return uploads, err
return response, err
}
// a wrapper around Lookup() which extracts the 1st upload, if any
func (db *Db) Lookup(apicontext string, id string) (*common.Upload, error) {
uploads, err := db.Get(apicontext, id)
func (db *Db) Lookup(apicontext string, id string, t int) (*common.Response, error) {
response, err := db.Get(apicontext, id, t)
if err != nil {
// non existent db entry with that id, or other db error, see logs
return &common.Upload{}, fmt.Errorf("No upload object found with id %s", id)
return &common.Response{}, fmt.Errorf("No upload object found with id %s", id)
}
if len(uploads.Entries) == 0 {
return &common.Upload{}, fmt.Errorf("No upload object found with id %s", id)
if len(response.Uploads) == 0 {
return &common.Response{}, fmt.Errorf("No upload object found with id %s", id)
}
return uploads.Entries[0], nil
return response, nil
}

234
api/form_handlers.go Normal file
View File

@@ -0,0 +1,234 @@
/*
Copyright © 2023 Thomas von Dein
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package api
import (
//"github.com/alecthomas/repr"
"github.com/gofiber/fiber/v2"
"github.com/google/uuid"
"github.com/tlinden/cenophane/cfg"
"github.com/tlinden/cenophane/common"
"strings"
"time"
)
func FormCreate(c *fiber.Ctx, cfg *cfg.Config, db *Db) error {
id := uuid.NewString()
var formdata common.Form
// init form obj
entry := &common.Form{Id: id, Created: common.Timestamp{Time: time.Now()}}
// retrieve the API Context name from the session
apicontext, err := GetApicontext(c)
if err != nil {
return JsonStatus(c, fiber.StatusInternalServerError,
"Unable to initialize session store from context: "+err.Error())
}
entry.Context = apicontext
// extract auxilliary form data (expire field et al)
if err := c.BodyParser(&formdata); err != nil {
return JsonStatus(c, fiber.StatusInternalServerError,
"bodyparser error : "+err.Error())
}
// post process expire
if len(formdata.Expire) == 0 {
entry.Expire = "asap"
} else {
ex, err := common.Untaint(formdata.Expire, cfg.RegDuration) // duration or asap allowed
if err != nil {
return JsonStatus(c, fiber.StatusForbidden,
"Invalid data: "+err.Error())
}
entry.Expire = ex
}
// get url [and zip if there are multiple files]
returnUrl := strings.Join([]string{cfg.Url, "form", id}, "/")
entry.Url = returnUrl
Log("Now serving %s", returnUrl)
Log("Expire set to: %s", entry.Expire)
Log("Form created with API-Context %s", entry.Context)
// we do this in the background to not thwart the server
go db.Insert(id, entry)
// everything went well so far
res := &common.Response{Forms: []*common.Form{entry}}
res.Success = true
res.Code = fiber.StatusOK
return c.Status(fiber.StatusOK).JSON(res)
}
/*
func FormFetch(c *fiber.Ctx, cfg *cfg.Config, db *Db, shallExpire ...bool) error {
// deliver a file and delete it if expire is set to asap
// we ignore c.Params("file"), cause it may be malign. Also we've
// got it in the db anyway
id, err := common.Untaint(c.Params("id"), cfg.RegKey)
if err != nil {
return fiber.NewError(403, "Invalid id provided!")
}
// retrieve the API Context name from the session
apicontext, err := GetApicontext(c)
if err != nil {
return JsonStatus(c, fiber.StatusInternalServerError,
"Unable to initialize session store from context: "+err.Error())
}
upload, err := db.Lookup(apicontext, id)
if err != nil {
// non existent db entry with that id, or other db error, see logs
return fiber.NewError(404, "No download with that id could be found!")
}
file := upload.File
filename := filepath.Join(cfg.StorageDir, id, file)
if _, err := os.Stat(filename); err != nil {
// db entry is there, but file isn't (anymore?)
go db.Delete(apicontext, id)
return fiber.NewError(404, "No download with that id could be found!")
}
// finally put the file to the client
err = c.Download(filename, file)
if len(shallExpire) > 0 {
if shallExpire[0] == true {
go func() {
// check if we need to delete the file now and do it in the background
if upload.Expire == "asap" {
cleanup(filepath.Join(cfg.StorageDir, id))
db.Delete(apicontext, id)
}
}()
}
}
return err
}
// delete file, id dir and db entry
func FormDelete(c *fiber.Ctx, cfg *cfg.Config, db *Db) error {
id, err := common.Untaint(c.Params("id"), cfg.RegKey)
if err != nil {
return JsonStatus(c, fiber.StatusForbidden,
"Invalid id provided!")
}
if len(id) == 0 {
return JsonStatus(c, fiber.StatusForbidden,
"No id specified!")
}
// retrieve the API Context name from the session
apicontext, err := GetApicontext(c)
if err != nil {
return JsonStatus(c, fiber.StatusInternalServerError,
"Unable to initialize session store from context: "+err.Error())
}
err = db.Delete(apicontext, id)
if err != nil {
// non existent db entry with that id, or other db error, see logs
return JsonStatus(c, fiber.StatusForbidden,
"No upload with that id could be found!")
}
cleanup(filepath.Join(cfg.StorageDir, id))
return nil
}
// returns the whole list + error code, no post processing by server
func FormsList(c *fiber.Ctx, cfg *cfg.Config, db *Db) error {
// fetch filter from body(json expected)
setcontext := new(SetContext)
if err := c.BodyParser(setcontext); err != nil {
return JsonStatus(c, fiber.StatusForbidden,
"Unable to parse body: "+err.Error())
}
filter, err := common.Untaint(setcontext.Apicontext, cfg.RegKey)
if err != nil {
return JsonStatus(c, fiber.StatusForbidden,
"Invalid api context filter provided!")
}
// retrieve the API Context name from the session
apicontext, err := GetApicontext(c)
if err != nil {
return JsonStatus(c, fiber.StatusInternalServerError,
"Unable to initialize session store from context: "+err.Error())
}
// get list
uploads, err := db.FormsList(apicontext, filter)
if err != nil {
return JsonStatus(c, fiber.StatusForbidden,
"Unable to list uploads: "+err.Error())
}
// if we reached this point we can signal success
uploads.Success = true
uploads.Code = fiber.StatusOK
return c.Status(fiber.StatusOK).JSON(uploads)
}
// returns just one upload obj + error code, no post processing by server
func FormDescribe(c *fiber.Ctx, cfg *cfg.Config, db *Db) error {
id, err := common.Untaint(c.Params("id"), cfg.RegKey)
if err != nil {
return JsonStatus(c, fiber.StatusForbidden,
"Invalid id provided!")
}
// retrieve the API Context name from the session
apicontext, err := GetApicontext(c)
if err != nil {
return JsonStatus(c, fiber.StatusInternalServerError,
"Unable to initialize session store from context: "+err.Error())
}
uploads, err := db.Get(apicontext, id)
if err != nil {
return JsonStatus(c, fiber.StatusForbidden,
"No upload with that id could be found!")
}
for _, upload := range uploads.Entries {
upload.Url = strings.Join([]string{cfg.Url, "download", id, upload.File}, "/")
}
// if we reached this point we can signal success
uploads.Success = true
uploads.Code = fiber.StatusOK
return c.Status(fiber.StatusOK).JSON(uploads)
}
*/

29
api/server.go
View File

@@ -80,6 +80,29 @@ func Runserver(conf *cfg.Config, args []string) error {
api.Get("/uploads/:id/file", auth, func(c *fiber.Ctx) error {
return UploadFetch(c, conf, db)
})
// same for forms ************
api.Post("/forms", auth, func(c *fiber.Ctx) error {
return FormCreate(c, conf, db)
})
/*
// remove
api.Delete("/forms/:id", auth, func(c *fiber.Ctx) error {
err := FormDelete(c, conf, db)
return SendResponse(c, "", err)
})
// listing
api.Get("/forms", auth, func(c *fiber.Ctx) error {
return FormsList(c, conf, db)
})
// info/describe
api.Get("/forms/:id", auth, func(c *fiber.Ctx) error {
return FormDescribe(c, conf, db)
})
*/
}
// public routes
@@ -95,6 +118,12 @@ func Runserver(conf *cfg.Config, args []string) error {
router.Get("/download/:id", func(c *fiber.Ctx) error {
return UploadFetch(c, conf, db, shallExpire)
})
/*
router.Get("/form/:id", func(c *fiber.Ctx) error {
return FormFetch(c, conf, db, shallExpire)
})
*/
}
// setup cleaner

21
api/handlers.go → api/upload_handlers.go
View File

@@ -116,7 +116,7 @@ func UploadPost(c *fiber.Ctx, cfg *cfg.Config, db *Db) error {
go db.Insert(id, entry)
// everything went well so far
res := &common.Uploads{Entries: []*common.Upload{entry}}
res := &common.Response{Uploads: []*common.Upload{entry}}
res.Success = true
res.Code = fiber.StatusOK
return c.Status(fiber.StatusOK).JSON(res)
@@ -139,12 +139,17 @@ func UploadFetch(c *fiber.Ctx, cfg *cfg.Config, db *Db, shallExpire ...bool) err
"Unable to initialize session store from context: "+err.Error())
}
upload, err := db.Lookup(apicontext, id)
response, err := db.Lookup(apicontext, id, common.TypeUpload)
if err != nil {
// non existent db entry with that id, or other db error, see logs
return fiber.NewError(404, "No download with that id could be found!")
}
var upload *common.Upload
if len(response.Uploads) > 0 {
upload = response.Uploads[0]
}
file := upload.File
filename := filepath.Join(cfg.StorageDir, id, file)
@@ -228,7 +233,7 @@ func UploadsList(c *fiber.Ctx, cfg *cfg.Config, db *Db) error {
}
// get list
uploads, err := db.UploadsList(apicontext, filter)
uploads, err := db.UploadsList(apicontext, filter, common.TypeUpload)
if err != nil {
return JsonStatus(c, fiber.StatusForbidden,
"Unable to list uploads: "+err.Error())
@@ -256,19 +261,19 @@ func UploadDescribe(c *fiber.Ctx, cfg *cfg.Config, db *Db) error {
"Unable to initialize session store from context: "+err.Error())
}
uploads, err := db.Get(apicontext, id)
response, err := db.Get(apicontext, id, common.TypeUpload)
if err != nil {
return JsonStatus(c, fiber.StatusForbidden,
"No upload with that id could be found!")
}
for _, upload := range uploads.Entries {
for _, upload := range response.Uploads {
upload.Url = strings.Join([]string{cfg.Url, "download", id, upload.File}, "/")
}
// if we reached this point we can signal success
uploads.Success = true
uploads.Code = fiber.StatusOK
response.Success = true
response.Code = fiber.StatusOK
return c.Status(fiber.StatusOK).JSON(uploads)
return c.Status(fiber.StatusOK).JSON(response)
}

102
common/types.go
View File

@@ -17,6 +17,11 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package common
import (
"encoding/json"
"fmt"
)
// used to return to the api client
type Result struct {
Success bool `json:"success"`
@@ -24,6 +29,12 @@ type Result struct {
Code int `json:"code"`
}
// upload or form structs
type Dbentry interface {
Getcontext(j []byte) (string, error)
Marshal() ([]byte, error)
}
type Upload struct {
Id string `json:"id"`
Expire string `json:"expire"`
@@ -35,9 +46,96 @@ type Upload struct {
}
// this one is also used for marshalling to the client
type Uploads struct {
Entries []*Upload `json:"uploads"`
type Response struct {
Uploads []*Upload `json:"uploads"`
Forms []*Form `json:"forms"`
// integrate the Result struct so we can signal success
Result
}
type Form struct {
Id string `json:"id"`
Expire string `json:"expire"`
Description string `json:"description"`
Created Timestamp `json:"uploaded"`
Context string `json:"context"`
Url string `json:"url"`
}
const (
TypeUpload = iota
TypeForm
)
/*
implement Dbentry interface
*/
func (upload Upload) Getcontext(j []byte) (string, error) {
if err := json.Unmarshal(j, &upload); err != nil {
return "", fmt.Errorf("unable to unmarshal json: %s", err)
}
return upload.Context, nil
}
func (form Form) Getcontext(j []byte) (string, error) {
if err := json.Unmarshal(j, &form); err != nil {
return "", fmt.Errorf("unable to unmarshal json: %s", err)
}
return form.Context, nil
}
func (upload Upload) Marshal() ([]byte, error) {
jsonentry, err := json.Marshal(upload)
if err != nil {
return nil, fmt.Errorf("json marshalling failure: %s", err)
}
return jsonentry, nil
}
func (form Form) Marshal() ([]byte, error) {
jsonentry, err := json.Marshal(form)
if err != nil {
return nil, fmt.Errorf("json marshalling failure: %s", err)
}
return jsonentry, nil
}
/*
Extract context, whatever kind entry is, but we don't know in
advance, only after unmarshalling. So try Upload first, if that
fails, try Form.
*/
func GetContext(j []byte) (string, error) {
upload := &Upload{}
entryContext, err := upload.Getcontext(j)
if err != nil {
form := &Form{}
entryContext, err = form.Getcontext(j)
if err != nil {
return "", fmt.Errorf("unable to unmarshal json: %s", err)
}
}
return entryContext, nil
}
func Unmarshal(j []byte, t int) (Dbentry, error) {
if t == TypeUpload {
upload := &Upload{}
if err := json.Unmarshal(j, &upload); err != nil {
return upload, fmt.Errorf("unable to unmarshal json: %s", err)
}
return upload, nil
} else {
form := &Form{}
if err := json.Unmarshal(j, &form); err != nil {
return form, fmt.Errorf("unable to unmarshal json: %s", err)
}
return form, nil
}
}

2
upctl/lib/client.go
View File

@@ -168,7 +168,7 @@ func HandleResponse(c *cfg.Config, resp *req.Response) error {
func UploadFiles(w io.Writer, c *cfg.Config, args []string) error {
// setup url, req.Request, timeout handling etc
rq := Setup(c, "/uploads/")
rq := Setup(c, "/uploads")
// collect files to upload from @argv
if err := GatherFiles(rq, args); err != nil {