From d028fb2db10ebf6b0d61f14c385c2d3b0fe0f4c4 Mon Sep 17 00:00:00 2001 From: Thomas von Dein Date: Sat, 25 Mar 2023 16:20:42 +0100 Subject: [PATCH] generalized DB engine to support multiple types (upload+form) --- api/db.go | 88 +++++---- api/form_handlers.go | 234 ++++++++++++++++++++++++ api/server.go | 29 +++ api/{handlers.go => upload_handlers.go} | 21 ++- common/types.go | 102 ++++++++++- upctl/lib/client.go | 2 +- 6 files changed, 432 insertions(+), 44 deletions(-) create mode 100644 api/form_handlers.go rename api/{handlers.go => upload_handlers.go} (93%) diff --git a/api/db.go b/api/db.go index 11ad1f9..1b9c322 100644 --- a/api/db.go +++ b/api/db.go @@ -18,7 +18,6 @@ along with this program. If not, see . package api import ( - "encoding/json" "fmt" "github.com/tlinden/cenophane/cfg" "github.com/tlinden/cenophane/common" @@ -26,7 +25,7 @@ import ( bolt "go.etcd.io/bbolt" ) -const Bucket string = "uploads" +const Bucket string = "data" // wrapper for bolt db type Db struct { @@ -44,14 +43,14 @@ func (db *Db) Close() { db.bolt.Close() } -func (db *Db) Insert(id string, entry *common.Upload) error { +func (db *Db) Insert(id string, entry common.Dbentry) error { err := db.bolt.Update(func(tx *bolt.Tx) error { bucket, err := tx.CreateBucketIfNotExists([]byte(Bucket)) if err != nil { return fmt.Errorf("create bucket: %s", err) } - jsonentry, err := json.Marshal(entry) + jsonentry, err := entry.Marshal() if err != nil { return fmt.Errorf("json marshalling failure: %s", err) } @@ -61,9 +60,6 @@ func (db *Db) Insert(id string, entry *common.Upload) error { return fmt.Errorf("insert data: %s", err) } - // results in: - // bbolt get /tmp/uploads.db uploads fb242922-86cb-43a8-92bc-b027c35f0586 - // {"id":"fb242922-86cb-43a8-92bc-b027c35f0586","expire":"1d","file":"2023-02-17-13-09-data.zip"} return nil }) if err != nil { @@ -87,12 +83,12 @@ func (db *Db) Delete(apicontext string, id string) error { return fmt.Errorf("id %s not found", id) } - upload := &common.Upload{} - if err := json.Unmarshal(j, &upload); err != nil { + entryContext, err := common.GetContext(j) + if err != nil { return fmt.Errorf("unable to unmarshal json: %s", err) } - if (apicontext != "" && (db.cfg.Super == apicontext || upload.Context == apicontext)) || apicontext == "" { + if (apicontext != "" && (db.cfg.Super == apicontext || entryContext == apicontext)) || apicontext == "" { return bucket.Delete([]byte(id)) } @@ -106,8 +102,8 @@ func (db *Db) Delete(apicontext string, id string) error { return err } -func (db *Db) UploadsList(apicontext string, filter string) (*common.Uploads, error) { - uploads := &common.Uploads{} +func (db *Db) UploadsList(apicontext string, filter string, t int) (*common.Response, error) { + response := &common.Response{} err := db.bolt.View(func(tx *bolt.Tx) error { bucket := tx.Bucket([]byte(Bucket)) @@ -116,24 +112,39 @@ func (db *Db) UploadsList(apicontext string, filter string) (*common.Uploads, er } err := bucket.ForEach(func(id, j []byte) error { - upload := &common.Upload{} - if err := json.Unmarshal(j, &upload); err != nil { + entry, err := common.Unmarshal(j, t) + if err != nil { return fmt.Errorf("unable to unmarshal json: %s", err) } + var entryContext string + if t == common.TypeUpload { + entryContext = entry.(common.Upload).Context + } else { + entryContext = entry.(common.Form).Context + } + fmt.Printf("apicontext: %s, filter: %s\n", apicontext, filter) if apicontext != "" && db.cfg.Super != apicontext { // only return the uploads for this context - if apicontext == upload.Context { + if apicontext == entryContext { // unless a filter needed OR no filter specified - if (filter != "" && upload.Context == filter) || filter == "" { - uploads.Entries = append(uploads.Entries, upload) + if (filter != "" && entryContext == filter) || filter == "" { + if t == common.TypeUpload { + response.Uploads = append(response.Uploads, entry.(*common.Upload)) + } else { + response.Forms = append(response.Forms, entry.(*common.Form)) + } } } } else { // return all, because we operate a public service or current==super - if (filter != "" && upload.Context == filter) || filter == "" { - uploads.Entries = append(uploads.Entries, upload) + if (filter != "" && entryContext == filter) || filter == "" { + if t == common.TypeUpload { + response.Uploads = append(response.Uploads, entry.(*common.Upload)) + } else { + response.Forms = append(response.Forms, entry.(*common.Form)) + } } } @@ -143,12 +154,12 @@ func (db *Db) UploadsList(apicontext string, filter string) (*common.Uploads, er return err // might be nil as well }) - return uploads, err + return response, err } // we only return one obj here, but could return more later -func (db *Db) Get(apicontext string, id string) (*common.Uploads, error) { - uploads := &common.Uploads{} +func (db *Db) Get(apicontext string, id string, t int) (*common.Response, error) { + response := &common.Response{} err := db.bolt.View(func(tx *bolt.Tx) error { bucket := tx.Bucket([]byte(Bucket)) @@ -161,35 +172,46 @@ func (db *Db) Get(apicontext string, id string) (*common.Uploads, error) { return fmt.Errorf("No upload object found with id %s", id) } - upload := &common.Upload{} - if err := json.Unmarshal(j, &upload); err != nil { + entry, err := common.Unmarshal(j, t) + if err != nil { return fmt.Errorf("unable to unmarshal json: %s", err) } - if (apicontext != "" && (db.cfg.Super == apicontext || upload.Context == apicontext)) || apicontext == "" { + var entryContext string + if t == common.TypeUpload { + entryContext = entry.(common.Upload).Context + } else { + entryContext = entry.(common.Form).Context + } + + if (apicontext != "" && (db.cfg.Super == apicontext || entryContext == apicontext)) || apicontext == "" { // allowed if no context (public or download) // or if context matches or if context==super - uploads.Entries = append(uploads.Entries, upload) + if t == common.TypeUpload { + response.Uploads = append(response.Uploads, entry.(*common.Upload)) + } else { + response.Forms = append(response.Forms, entry.(*common.Form)) + } } return nil }) - return uploads, err + return response, err } // a wrapper around Lookup() which extracts the 1st upload, if any -func (db *Db) Lookup(apicontext string, id string) (*common.Upload, error) { - uploads, err := db.Get(apicontext, id) +func (db *Db) Lookup(apicontext string, id string, t int) (*common.Response, error) { + response, err := db.Get(apicontext, id, t) if err != nil { // non existent db entry with that id, or other db error, see logs - return &common.Upload{}, fmt.Errorf("No upload object found with id %s", id) + return &common.Response{}, fmt.Errorf("No upload object found with id %s", id) } - if len(uploads.Entries) == 0 { - return &common.Upload{}, fmt.Errorf("No upload object found with id %s", id) + if len(response.Uploads) == 0 { + return &common.Response{}, fmt.Errorf("No upload object found with id %s", id) } - return uploads.Entries[0], nil + return response, nil } diff --git a/api/form_handlers.go b/api/form_handlers.go new file mode 100644 index 0000000..de62a9e --- /dev/null +++ b/api/form_handlers.go @@ -0,0 +1,234 @@ +/* +Copyright © 2023 Thomas von Dein + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package api + +import ( + //"github.com/alecthomas/repr" + "github.com/gofiber/fiber/v2" + "github.com/google/uuid" + "github.com/tlinden/cenophane/cfg" + "github.com/tlinden/cenophane/common" + + "strings" + "time" +) + +func FormCreate(c *fiber.Ctx, cfg *cfg.Config, db *Db) error { + id := uuid.NewString() + + var formdata common.Form + + // init form obj + entry := &common.Form{Id: id, Created: common.Timestamp{Time: time.Now()}} + + // retrieve the API Context name from the session + apicontext, err := GetApicontext(c) + if err != nil { + return JsonStatus(c, fiber.StatusInternalServerError, + "Unable to initialize session store from context: "+err.Error()) + } + entry.Context = apicontext + + // extract auxilliary form data (expire field et al) + if err := c.BodyParser(&formdata); err != nil { + return JsonStatus(c, fiber.StatusInternalServerError, + "bodyparser error : "+err.Error()) + } + + // post process expire + if len(formdata.Expire) == 0 { + entry.Expire = "asap" + } else { + ex, err := common.Untaint(formdata.Expire, cfg.RegDuration) // duration or asap allowed + if err != nil { + return JsonStatus(c, fiber.StatusForbidden, + "Invalid data: "+err.Error()) + } + entry.Expire = ex + } + + // get url [and zip if there are multiple files] + returnUrl := strings.Join([]string{cfg.Url, "form", id}, "/") + entry.Url = returnUrl + + Log("Now serving %s", returnUrl) + Log("Expire set to: %s", entry.Expire) + Log("Form created with API-Context %s", entry.Context) + + // we do this in the background to not thwart the server + go db.Insert(id, entry) + + // everything went well so far + res := &common.Response{Forms: []*common.Form{entry}} + res.Success = true + res.Code = fiber.StatusOK + return c.Status(fiber.StatusOK).JSON(res) +} + +/* +func FormFetch(c *fiber.Ctx, cfg *cfg.Config, db *Db, shallExpire ...bool) error { + // deliver a file and delete it if expire is set to asap + + // we ignore c.Params("file"), cause it may be malign. Also we've + // got it in the db anyway + id, err := common.Untaint(c.Params("id"), cfg.RegKey) + if err != nil { + return fiber.NewError(403, "Invalid id provided!") + } + + // retrieve the API Context name from the session + apicontext, err := GetApicontext(c) + if err != nil { + return JsonStatus(c, fiber.StatusInternalServerError, + "Unable to initialize session store from context: "+err.Error()) + } + + upload, err := db.Lookup(apicontext, id) + if err != nil { + // non existent db entry with that id, or other db error, see logs + return fiber.NewError(404, "No download with that id could be found!") + } + + file := upload.File + filename := filepath.Join(cfg.StorageDir, id, file) + + if _, err := os.Stat(filename); err != nil { + // db entry is there, but file isn't (anymore?) + go db.Delete(apicontext, id) + return fiber.NewError(404, "No download with that id could be found!") + } + + // finally put the file to the client + err = c.Download(filename, file) + + if len(shallExpire) > 0 { + if shallExpire[0] == true { + go func() { + // check if we need to delete the file now and do it in the background + if upload.Expire == "asap" { + cleanup(filepath.Join(cfg.StorageDir, id)) + db.Delete(apicontext, id) + } + }() + } + } + + return err +} + +// delete file, id dir and db entry +func FormDelete(c *fiber.Ctx, cfg *cfg.Config, db *Db) error { + + id, err := common.Untaint(c.Params("id"), cfg.RegKey) + if err != nil { + return JsonStatus(c, fiber.StatusForbidden, + "Invalid id provided!") + } + + if len(id) == 0 { + return JsonStatus(c, fiber.StatusForbidden, + "No id specified!") + } + + // retrieve the API Context name from the session + apicontext, err := GetApicontext(c) + if err != nil { + return JsonStatus(c, fiber.StatusInternalServerError, + "Unable to initialize session store from context: "+err.Error()) + } + + err = db.Delete(apicontext, id) + if err != nil { + // non existent db entry with that id, or other db error, see logs + return JsonStatus(c, fiber.StatusForbidden, + "No upload with that id could be found!") + } + + cleanup(filepath.Join(cfg.StorageDir, id)) + + return nil +} + +// returns the whole list + error code, no post processing by server +func FormsList(c *fiber.Ctx, cfg *cfg.Config, db *Db) error { + // fetch filter from body(json expected) + setcontext := new(SetContext) + if err := c.BodyParser(setcontext); err != nil { + return JsonStatus(c, fiber.StatusForbidden, + "Unable to parse body: "+err.Error()) + } + + filter, err := common.Untaint(setcontext.Apicontext, cfg.RegKey) + if err != nil { + return JsonStatus(c, fiber.StatusForbidden, + "Invalid api context filter provided!") + } + + // retrieve the API Context name from the session + apicontext, err := GetApicontext(c) + if err != nil { + return JsonStatus(c, fiber.StatusInternalServerError, + "Unable to initialize session store from context: "+err.Error()) + } + + // get list + uploads, err := db.FormsList(apicontext, filter) + if err != nil { + return JsonStatus(c, fiber.StatusForbidden, + "Unable to list uploads: "+err.Error()) + } + + // if we reached this point we can signal success + uploads.Success = true + uploads.Code = fiber.StatusOK + + return c.Status(fiber.StatusOK).JSON(uploads) +} + +// returns just one upload obj + error code, no post processing by server +func FormDescribe(c *fiber.Ctx, cfg *cfg.Config, db *Db) error { + id, err := common.Untaint(c.Params("id"), cfg.RegKey) + if err != nil { + return JsonStatus(c, fiber.StatusForbidden, + "Invalid id provided!") + } + + // retrieve the API Context name from the session + apicontext, err := GetApicontext(c) + if err != nil { + return JsonStatus(c, fiber.StatusInternalServerError, + "Unable to initialize session store from context: "+err.Error()) + } + + uploads, err := db.Get(apicontext, id) + if err != nil { + return JsonStatus(c, fiber.StatusForbidden, + "No upload with that id could be found!") + } + + for _, upload := range uploads.Entries { + upload.Url = strings.Join([]string{cfg.Url, "download", id, upload.File}, "/") + } + + // if we reached this point we can signal success + uploads.Success = true + uploads.Code = fiber.StatusOK + + return c.Status(fiber.StatusOK).JSON(uploads) +} +*/ diff --git a/api/server.go b/api/server.go index aa5e4ef..ee3f20f 100644 --- a/api/server.go +++ b/api/server.go @@ -80,6 +80,29 @@ func Runserver(conf *cfg.Config, args []string) error { api.Get("/uploads/:id/file", auth, func(c *fiber.Ctx) error { return UploadFetch(c, conf, db) }) + + // same for forms ************ + api.Post("/forms", auth, func(c *fiber.Ctx) error { + return FormCreate(c, conf, db) + }) + + /* + // remove + api.Delete("/forms/:id", auth, func(c *fiber.Ctx) error { + err := FormDelete(c, conf, db) + return SendResponse(c, "", err) + }) + + // listing + api.Get("/forms", auth, func(c *fiber.Ctx) error { + return FormsList(c, conf, db) + }) + + // info/describe + api.Get("/forms/:id", auth, func(c *fiber.Ctx) error { + return FormDescribe(c, conf, db) + }) + */ } // public routes @@ -95,6 +118,12 @@ func Runserver(conf *cfg.Config, args []string) error { router.Get("/download/:id", func(c *fiber.Ctx) error { return UploadFetch(c, conf, db, shallExpire) }) + + /* + router.Get("/form/:id", func(c *fiber.Ctx) error { + return FormFetch(c, conf, db, shallExpire) + }) + */ } // setup cleaner diff --git a/api/handlers.go b/api/upload_handlers.go similarity index 93% rename from api/handlers.go rename to api/upload_handlers.go index 43c54f2..730c5f0 100644 --- a/api/handlers.go +++ b/api/upload_handlers.go @@ -116,7 +116,7 @@ func UploadPost(c *fiber.Ctx, cfg *cfg.Config, db *Db) error { go db.Insert(id, entry) // everything went well so far - res := &common.Uploads{Entries: []*common.Upload{entry}} + res := &common.Response{Uploads: []*common.Upload{entry}} res.Success = true res.Code = fiber.StatusOK return c.Status(fiber.StatusOK).JSON(res) @@ -139,12 +139,17 @@ func UploadFetch(c *fiber.Ctx, cfg *cfg.Config, db *Db, shallExpire ...bool) err "Unable to initialize session store from context: "+err.Error()) } - upload, err := db.Lookup(apicontext, id) + response, err := db.Lookup(apicontext, id, common.TypeUpload) if err != nil { // non existent db entry with that id, or other db error, see logs return fiber.NewError(404, "No download with that id could be found!") } + var upload *common.Upload + if len(response.Uploads) > 0 { + upload = response.Uploads[0] + } + file := upload.File filename := filepath.Join(cfg.StorageDir, id, file) @@ -228,7 +233,7 @@ func UploadsList(c *fiber.Ctx, cfg *cfg.Config, db *Db) error { } // get list - uploads, err := db.UploadsList(apicontext, filter) + uploads, err := db.UploadsList(apicontext, filter, common.TypeUpload) if err != nil { return JsonStatus(c, fiber.StatusForbidden, "Unable to list uploads: "+err.Error()) @@ -256,19 +261,19 @@ func UploadDescribe(c *fiber.Ctx, cfg *cfg.Config, db *Db) error { "Unable to initialize session store from context: "+err.Error()) } - uploads, err := db.Get(apicontext, id) + response, err := db.Get(apicontext, id, common.TypeUpload) if err != nil { return JsonStatus(c, fiber.StatusForbidden, "No upload with that id could be found!") } - for _, upload := range uploads.Entries { + for _, upload := range response.Uploads { upload.Url = strings.Join([]string{cfg.Url, "download", id, upload.File}, "/") } // if we reached this point we can signal success - uploads.Success = true - uploads.Code = fiber.StatusOK + response.Success = true + response.Code = fiber.StatusOK - return c.Status(fiber.StatusOK).JSON(uploads) + return c.Status(fiber.StatusOK).JSON(response) } diff --git a/common/types.go b/common/types.go index bfa9cfa..394bb01 100644 --- a/common/types.go +++ b/common/types.go @@ -17,6 +17,11 @@ along with this program. If not, see . package common +import ( + "encoding/json" + "fmt" +) + // used to return to the api client type Result struct { Success bool `json:"success"` @@ -24,6 +29,12 @@ type Result struct { Code int `json:"code"` } +// upload or form structs +type Dbentry interface { + Getcontext(j []byte) (string, error) + Marshal() ([]byte, error) +} + type Upload struct { Id string `json:"id"` Expire string `json:"expire"` @@ -35,9 +46,96 @@ type Upload struct { } // this one is also used for marshalling to the client -type Uploads struct { - Entries []*Upload `json:"uploads"` +type Response struct { + Uploads []*Upload `json:"uploads"` + Forms []*Form `json:"forms"` // integrate the Result struct so we can signal success Result } + +type Form struct { + Id string `json:"id"` + Expire string `json:"expire"` + Description string `json:"description"` + Created Timestamp `json:"uploaded"` + Context string `json:"context"` + Url string `json:"url"` +} + +const ( + TypeUpload = iota + TypeForm +) + +/* + implement Dbentry interface +*/ +func (upload Upload) Getcontext(j []byte) (string, error) { + if err := json.Unmarshal(j, &upload); err != nil { + return "", fmt.Errorf("unable to unmarshal json: %s", err) + } + + return upload.Context, nil +} + +func (form Form) Getcontext(j []byte) (string, error) { + if err := json.Unmarshal(j, &form); err != nil { + return "", fmt.Errorf("unable to unmarshal json: %s", err) + } + + return form.Context, nil +} + +func (upload Upload) Marshal() ([]byte, error) { + jsonentry, err := json.Marshal(upload) + if err != nil { + return nil, fmt.Errorf("json marshalling failure: %s", err) + } + + return jsonentry, nil +} + +func (form Form) Marshal() ([]byte, error) { + jsonentry, err := json.Marshal(form) + if err != nil { + return nil, fmt.Errorf("json marshalling failure: %s", err) + } + + return jsonentry, nil +} + +/* + Extract context, whatever kind entry is, but we don't know in + advance, only after unmarshalling. So try Upload first, if that + fails, try Form. +*/ +func GetContext(j []byte) (string, error) { + upload := &Upload{} + entryContext, err := upload.Getcontext(j) + if err != nil { + form := &Form{} + entryContext, err = form.Getcontext(j) + if err != nil { + return "", fmt.Errorf("unable to unmarshal json: %s", err) + } + } + + return entryContext, nil +} + +func Unmarshal(j []byte, t int) (Dbentry, error) { + if t == TypeUpload { + upload := &Upload{} + if err := json.Unmarshal(j, &upload); err != nil { + return upload, fmt.Errorf("unable to unmarshal json: %s", err) + } + return upload, nil + } else { + form := &Form{} + if err := json.Unmarshal(j, &form); err != nil { + return form, fmt.Errorf("unable to unmarshal json: %s", err) + } + return form, nil + } +} diff --git a/upctl/lib/client.go b/upctl/lib/client.go index dc7b330..6085ae6 100644 --- a/upctl/lib/client.go +++ b/upctl/lib/client.go @@ -168,7 +168,7 @@ func HandleResponse(c *cfg.Config, resp *req.Response) error { func UploadFiles(w io.Writer, c *cfg.Config, args []string) error { // setup url, req.Request, timeout handling etc - rq := Setup(c, "/uploads/") + rq := Setup(c, "/uploads") // collect files to upload from @argv if err := GatherFiles(rq, args); err != nil {