diff --git a/config.go b/config.go
index be53e30..c929576 100644
--- a/config.go
+++ b/config.go
@@ -1,5 +1,5 @@
/*
-Copyright © 2023 Thomas von Dein
+Copyright © 2023-2024 Thomas von Dein
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -33,7 +33,7 @@ import (
)
const (
- VERSION string = "0.1.0"
+ VERSION string = "0.1.1"
Baseuri string = "https://www.kleinanzeigen.de"
Listuri string = "/s-bestandsliste.html"
Defaultdir string = "."
@@ -43,6 +43,7 @@ const (
"Category: {{.Category}}\r\nCondition: {{.Condition}}\r\nCreated: {{.Created}}\r\n\r\n{{.Text}}\r\n"
Useragent string = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " +
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+ DefaultAdNameTemplate string = "{{.Slug}}-{{.Id}}"
)
const Usage string = `This is kleingebaeck, the kleinanzeigen.de backup tool.
@@ -72,6 +73,7 @@ type Config struct {
User int `koanf:"user"`
Outdir string `koanf:"outdir"`
Template string `koanf:"template"`
+ Adnametemplate string `koanf:"adnametemplate"`
Loglevel string `koanf:"loglevel"`
Limit int `koanf:"limit"`
Adlinks []string
@@ -99,10 +101,11 @@ func InitConfig(w io.Writer) (*Config, error) {
// Load default values using the confmap provider.
if err := k.Load(confmap.Provider(map[string]interface{}{
- "template": template,
- "outdir": ".",
- "loglevel": "notice",
- "userid": 0,
+ "template": template,
+ "outdir": ".",
+ "loglevel": "notice",
+ "userid": 0,
+ "adnametemplate": DefaultAdNameTemplate,
}, "."), nil); err != nil {
return nil, err
}
diff --git a/kleingebaeck.pod b/kleingebaeck.pod
index d8aaeac..25cdc59 100644
--- a/kleingebaeck.pod
+++ b/kleingebaeck.pod
@@ -96,7 +96,20 @@ change in the future.
=head1 LICENSE
-Licensed under the GNU GENERAL PUBLIC LICENSE version 3.
+Copyright © 2023-2024 Thomas von Dein
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see L.
=head1 Author
diff --git a/main_test.go b/main_test.go
index bceb8af..9b7dc5e 100644
--- a/main_test.go
+++ b/main_test.go
@@ -1,5 +1,5 @@
/*
-Copyright © 2023 Thomas von Dein
+Copyright © 2023-2024 Thomas von Dein
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -440,7 +440,15 @@ func SetIntercept(ads []Adsource) {
func VerifyAd(ad AdConfig) error {
body := ad.Title + ad.Price + ad.Id + ad.Category + ad.Condition + ad.Created
- file := fmt.Sprintf("t/out/%s/Adlisting.txt", ad.Slug)
+ // prepare ad dir name using DefaultAdNameTemplate
+ c := Config{Adnametemplate: DefaultAdNameTemplate}
+ adstruct := Ad{Slug: ad.Slug, Id: ad.Id}
+ addir, err := AdDirName(&c, &adstruct)
+ if err != nil {
+ return err
+ }
+
+ file := fmt.Sprintf("t/out/%s/Adlisting.txt", addir)
content, err := os.ReadFile(file)
if err != nil {
return err
diff --git a/scrape.go b/scrape.go
index 25e7d73..895fb20 100644
--- a/scrape.go
+++ b/scrape.go
@@ -1,5 +1,5 @@
/*
-Copyright © 2023 Thomas von Dein
+Copyright © 2023-2024 Thomas von Dein
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -147,24 +147,24 @@ func Scrape(c *Config, uri string) error {
slog.Debug("extracted ad listing", "ad", ad)
// write listing
- err = WriteAd(c.Outdir, ad, c.Template)
+ addir, err := WriteAd(c, ad)
if err != nil {
return err
}
c.IncrAds()
- return ScrapeImages(c, ad)
+ return ScrapeImages(c, ad, addir)
}
-func ScrapeImages(c *Config, ad *Ad) error {
+func ScrapeImages(c *Config, ad *Ad, addir string) error {
// fetch images
img := 1
g := new(errgroup.Group)
for _, imguri := range ad.Images {
imguri := imguri
- file := filepath.Join(c.Outdir, ad.Slug, fmt.Sprintf("%d.jpg", img))
+ file := filepath.Join(c.Outdir, addir, fmt.Sprintf("%d.jpg", img))
g.Go(func() error {
err := Getimage(imguri, file)
if err != nil {
diff --git a/store.go b/store.go
index e159f95..ad71c3e 100644
--- a/store.go
+++ b/store.go
@@ -1,5 +1,5 @@
/*
-Copyright © 2023 Thomas von Dein
+Copyright © 2023-2024 Thomas von Dein
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -18,6 +18,7 @@ along with this program. If not, see .
package main
import (
+ "bytes"
"io"
"log/slog"
"os"
@@ -27,19 +28,40 @@ import (
tpl "text/template"
)
-func WriteAd(dir string, ad *Ad, template string) error {
- // prepare output dir
- dir = filepath.Join(dir, ad.Slug)
- err := Mkdir(dir)
+func AdDirName(c *Config, ad *Ad) (string, error) {
+ tmpl, err := tpl.New("adname").Parse(c.Adnametemplate)
if err != nil {
- return err
+ return "", err
+ }
+
+ buf := bytes.Buffer{}
+ err = tmpl.Execute(&buf, ad)
+ if err != nil {
+ return "", err
+ }
+
+ return buf.String(), nil
+}
+
+func WriteAd(c *Config, ad *Ad) (string, error) {
+ // prepare ad dir name
+ addir, err := AdDirName(c, ad)
+ if err != nil {
+ return "", err
+ }
+
+ // prepare output dir
+ dir := filepath.Join(c.Outdir, addir)
+ err = Mkdir(dir)
+ if err != nil {
+ return "", err
}
// write ad file
listingfile := filepath.Join(dir, "Adlisting.txt")
f, err := os.Create(listingfile)
if err != nil {
- return err
+ return "", err
}
defer f.Close()
@@ -49,19 +71,19 @@ func WriteAd(dir string, ad *Ad, template string) error {
ad.Text = strings.ReplaceAll(ad.Text, "
", "\n")
}
- tmpl, err := tpl.New("adlisting").Parse(template)
+ tmpl, err := tpl.New("adlisting").Parse(c.Template)
if err != nil {
- return err
+ return "", err
}
err = tmpl.Execute(f, ad)
if err != nil {
- return err
+ return "", err
}
slog.Info("wrote ad listing", "listingfile", listingfile)
- return nil
+ return addir, nil
}
func WriteImage(filename string, reader io.ReadCloser) error {