add test files in other formats

Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 6.9.0 to 6.10.0.
- [Release notes](https://github.com/docker/build-push-action/releases)
- [Commits](4f58ea7922...48aba3b46d)

---
updated-dependencies:
- dependency-name: docker/build-push-action
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "monthly"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+        interval: "monthly"

.github/workflows/ci.yaml

@@ -10,13 +10,13 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
     - name: Set up Go
-      uses: actions/setup-go@v3
+      uses: actions/setup-go@v5
       with:
         go-version: ${{ matrix.version }}
       id: go
 
     - name: checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: build
       run: go build
@@ -39,9 +39,9 @@ jobs:
     name: lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v5
         with:
           go-version: 1.21
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
+        uses: golangci/golangci-lint-action@v6

.github/workflows/pushimage.yaml

@@ -15,20 +15,20 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Log in to the Container registry
-        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567
         with:
           registry: https://ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355
         with:
           push: true
           tags: ghcr.io/tlinden/kleingebaeck:${{ github.ref_name}}
 
       - name: Build and push latest Docker image
-        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355
         with:
           push: true
           tags: ghcr.io/tlinden/kleingebaeck:latest

Makefile

@@ -63,6 +63,7 @@ lint:
 
 lint-full:
 	golangci-lint run --enable-all --exclude-use-default --disable exhaustivestruct,exhaustruct,depguard,interfacer,deadcode,golint,structcheck,scopelint,varcheck,ifshort,maligned,nosnakecase,godot,funlen,gofumpt,cyclop,noctx,gochecknoglobals,paralleltest
+	gocritic check -enableAll *.go
 
 testfuzzy: clean
 	go test -fuzz ./... $(ARGS)

README-de.md

@@ -222,6 +222,49 @@ Sowie alle Bilder.
 Das Format kann man mit der Variable `template` in der Konfiguration
 ändern. Die `example.conf` enthält ein Beispiel für das Standard Template.
 
+## Verhalten des Tools
+
+Es gibt einige Dinge über das Verhalten von kleingebäck, über die Du
+Bescheid wissen solltest:
+
+- alle HTML Seiten und Bilder werden immer heruntergeladen
+- es wird ein (konfigurierbarer) Useragent verwendet
+- HTTP Cookies werden beachtet
+- bei Fehlern wird dreimal mit unterschiedlichem Abstand erneut
+  versucht
+- Bilder Downloads laufen parallelisiert mit leicht unterschiedlichen
+  zeitlichen Abständen ab
+- Gleich aussehende Bilder werden nicht überschrieben
+
+Der letzte Punkt muss genauer erläutert werden:
+
+Wenn man bei Kleinanzeigen.de eine Anzeige einstellt und Bilder
+postet, werden diese dort in ihrer Grösse reduziert (durch Kompression
+und Verkleinerung der Bilder usw.). Diese reduzierten Bilder werden
+dann von kleingebäck heruntergeladen. Falls Du Deine original Bilder
+behalten hast, kannst Du diese danach in das Backupverzeichnis
+kopieren. Bei einem erneuten kleingebäck-Lauf werden diese Bilder dann
+nicht überschrieben.
+
+Wir verwenden dafür einen Algorythmus namens [distance
+hashing](https://github.com/corona10/goimagehash). Dieser Algorithmus
+prüft die Ähnlichkeit von Bildern. Diese können in ihrer Auflösung,
+Kompression, Farbtiefe und vielem mehr manipuliert worden sein und
+trotzdem als das "gleiche Bild" erkannt werden (wohlgemerkt nicht "das
+selbe": die Dateien sind durchaus unterschiedlich!). Bis zu einer
+Distance von 5 überschreiben wir keine Bilder, weil wir dann davon
+ausgehen, dass das lokal Vorhandene das Original ist.
+
+Bitte beachte aber, dass dies KEIN Cachingmechanismus ist: die Bilder
+werden trotzdem immer alle heruntergeladen. Das muss so sein, da wir
+uns nicht die Dateinamen anschauen können, da kleinanzeigen.de diese
+nämlich zu Zahlen umbenennt. Und die Dateinamen können sich auch
+ändern, wenn der User in der Anzeige die Bilder umarrangiert hat.
+
+Du kannst dieses Verhalten mit der Option **--force** ausschalten. Du
+kannst ausserdem mit der Option **--ignoreerrors** auch alle Fehler
+ignorieren, die beim Bilderdownload auftreten könnten.
+
 ## Documentation
 
 Die Dokumentation kann man

README.md

@@ -207,6 +207,48 @@ variable. The supplied sample config contains the default template.
 
 All images will be stored in the same directory.
 
+## Tool Behavior
+
+There are a bunch of things you might want to know about the behavior
+of the kleingebäck tool:
+
+- all HTML pages and IMAGEs are always being downloaded
+- we use a (customizable) user agent
+- we respect HTTP cookies
+- in the case of an error, the tool does 3 retries, the time it waits
+  between tries is longer for each retry
+- image download is parallized using small time differences to look
+  more natural
+- same images are not being overwritten on subsequent download
+
+
+The latter needs to be elaborated a bit more:
+
+If you publish an ad on kleinanzeigen.de and post images, those images
+will be reduced in size by the site (by compressing and down sizing
+them). This reduced images will be downloaded by kleingebäck. However,
+you may still own the original images and may want to put them into
+that backup directory so that you have all things for one ad together.
+
+You can easily do that, because kleingebäck won't overwrite those
+original images. It uses something called a distance hash using
+[goimagehash](https://github.com/corona10/goimagehash). This
+algorithmus checks the similarity of images. If an image has been
+resized it is still very similar to the original one. We accept a
+maximum of a distance of 5, everything above leads to overwrite.
+
+This works with resizes, cropped and otherwise manipulated images as
+long as the image still shows the original contents good enough.
+
+Also note, that this is NOT a caching mechanism: the images will be
+downloaded anyway during each run. We also can't look at the file
+names because kleinanzeigen.de renames all images to numbers. And
+those might even change if the user re-arranges the images.
+
+You can override this behavior using the **--force** option. Another
+option, **--ignoreerrors**, can be used to ignore all kinds of image
+errors. 
+
 ## Documentation
 
 You can read the documentation [online](https://github.com/TLINDEN/kleingebaeck/blob/main/kleingebaeck.pod) or locally once you have installed kleingebaeck with: `kleingebaeck --manual`.

SECURITY.md

@@ -0,0 +1,17 @@
+# Security Policy
+
+## Supported Versions
+
+Only  the latest  release  is supported.  If you  find  an issue  (any
+issue!), please check with the latest release first.
+
+## Reporting a Vulnerability
+
+I don't agree with the "responsible disclosure" process most projects
+(and companies) work these days.
+
+So, if you find a vulnerability of any kind, please just open an
+[issue](https://github.com/TLINDEN/kleingebaeck/issues). Please add
+all details required to reproduce the vulnerability. You won't be chased.
+
+That's just all about it.

ad.go

@@ -73,7 +73,7 @@ func (ad *Ad) Incomplete() bool {
 }
 
 func (ad *Ad) CalculateExpire() {
-	if len(ad.Created) > 0 {
+	if ad.Created != "" {
 		ts, err := time.Parse("02.01.2006", ad.Created)
 		if err == nil {
 			ad.Expire = ts.AddDate(0, ExpireMonths, ExpireDays).Format("02.01.2006")

config.go

@@ -34,7 +34,7 @@ import (
 )
 
 const (
-	VERSION    string = "0.3.3"
+	VERSION    string = "0.3.11"
 	Baseuri    string = "https://www.kleinanzeigen.de"
 	Listuri    string = "/s-bestandsliste.html"
 	Defaultdir string = "."
@@ -52,6 +52,8 @@ const (
 
 	DefaultAdNameTemplate string = "{{.Slug}}"
 
+	DefaultOutdirTemplate string = "."
+
 	// for image download throttling
 	MinThrottle int = 2
 	MaxThrottle int = 20
@@ -65,6 +67,8 @@ const (
 	WIN string = "windows"
 )
 
+var DirsVisited map[string]int
+
 const Usage string = `This is kleingebaeck, the kleinanzeigen.de backup tool.
 
 Usage: kleingebaeck [-dvVhmoclu] [<ad-listing-url>,...]
@@ -77,7 +81,7 @@ Options:
 -l --limit   <num>      Limit the ads to download to <num>, default: load all.
 -c --config  <file>     Use config file <file> (default: ~/.kleingebaeck).
    --ignoreerrors       Ignore HTTP errors, may lead to incomplete ad backup.
--f --force              Download images even if they already exist.
+-f --force              Overwrite images and ads even if the already exist.
 -m --manual             Show manual.
 -h --help               Show usage.
 -V --version            Show program version.
@@ -126,7 +130,7 @@ func InitConfig(output io.Writer) (*Config, error) {
 	// Load default values using the confmap provider.
 	if err := kloader.Load(confmap.Provider(map[string]interface{}{
 		"template":       template,
-		"outdir":         ".",
+		"outdir":         DefaultOutdirTemplate,
 		"loglevel":       "notice",
 		"userid":         0,
 		"adnametemplate": DefaultAdNameTemplate,
@@ -179,13 +183,22 @@ func InitConfig(output io.Writer) (*Config, error) {
 
 	// Load the config file[s]
 	for _, cfgfile := range configfiles {
-		if path, err := os.Stat(cfgfile); !os.IsNotExist(err) {
+		path, err := os.Stat(cfgfile)
-			if !path.IsDir() {
+
-				if err := kloader.Load(file.Provider(cfgfile), toml.Parser()); err != nil {
+		if err != nil {
-					return nil, fmt.Errorf("error loading config file: %w", err)
+			// ignore non-existent files, but bail out on any other errors
-				}
+			if !os.IsNotExist(err) {
+				return nil, fmt.Errorf("failed to stat config file: %w", err)
 			}
-		} // else: we ignore the file if it doesn't exists
+
+			continue
+		}
+
+		if !path.IsDir() {
+			if err := kloader.Load(file.Provider(cfgfile), toml.Parser()); err != nil {
+				return nil, fmt.Errorf("error loading config file: %w", err)
+			}
+		}
 	}
 
 	// env overrides config file

fetch.go

@@ -52,7 +52,7 @@ func NewFetcher(conf *Config) (*Fetcher, error) {
 }
 
 func (f *Fetcher) Get(uri string) (io.ReadCloser, error) {
-	req, err := http.NewRequest(http.MethodGet, uri, nil)
+	req, err := http.NewRequest(http.MethodGet, uri, http.NoBody)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create a new HTTP request obj: %w", err)
 	}

go.mod

@@ -4,36 +4,37 @@ go 1.21
 
 require (
 	astuart.co/goq v1.0.0
+	github.com/corona10/goimagehash v1.1.0
+	github.com/inconshreveable/mousetrap v1.1.0
 	github.com/jarcoal/httpmock v1.3.1
 	github.com/knadh/koanf/parsers/toml v0.1.0
 	github.com/knadh/koanf/providers/confmap v0.1.0
-	github.com/knadh/koanf/providers/env v0.1.0
+	github.com/knadh/koanf/providers/env v1.0.0
-	github.com/knadh/koanf/providers/file v0.1.0
+	github.com/knadh/koanf/providers/file v1.1.2
 	github.com/knadh/koanf/providers/posflag v0.1.0
-	github.com/knadh/koanf/v2 v2.0.1
+	github.com/knadh/koanf/v2 v2.1.2
-	github.com/lmittmann/tint v1.0.4
+	github.com/lmittmann/tint v1.0.5
 	github.com/mattn/go-isatty v0.0.20
 	github.com/spf13/pflag v1.0.5
-	github.com/tlinden/yadu v0.1.1
+	github.com/tlinden/yadu v0.1.3
-	golang.org/x/sync v0.5.0
+	golang.org/x/sync v0.9.0
 )
 
 require (
 	github.com/PuerkitoBio/goquery v1.5.1 // indirect
 	github.com/andybalholm/cascadia v1.1.0 // indirect
-	github.com/corona10/goimagehash v1.1.0 // indirect
 	github.com/fatih/color v1.16.0 // indirect
-	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
 	github.com/knadh/koanf/maps v0.1.1 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
-	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
+	golang.org/x/image v0.23.0 // indirect
-	golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect
+	golang.org/x/net v0.23.0 // indirect
-	golang.org/x/sys v0.14.0 // indirect
+	golang.org/x/sys v0.21.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 
 )

go.sum

@@ -13,8 +13,12 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
-github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
+github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
-github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
+github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
+github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss=
+github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww=
 github.com/jarcoal/httpmock v1.3.1/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg=
 github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NIs=
@@ -23,16 +27,20 @@ github.com/knadh/koanf/parsers/toml v0.1.0 h1:S2hLqS4TgWZYj4/7mI5m1CQQcWurxUz6OD
 github.com/knadh/koanf/parsers/toml v0.1.0/go.mod h1:yUprhq6eo3GbyVXFFMdbfZSo928ksS+uo0FFqNMnO18=
 github.com/knadh/koanf/providers/confmap v0.1.0 h1:gOkxhHkemwG4LezxxN8DMOFopOPghxRVp7JbIvdvqzU=
 github.com/knadh/koanf/providers/confmap v0.1.0/go.mod h1:2uLhxQzJnyHKfxG927awZC7+fyHFdQkd697K4MdLnIU=
-github.com/knadh/koanf/providers/env v0.1.0 h1:LqKteXqfOWyx5Ab9VfGHmjY9BvRXi+clwyZozgVRiKg=
+github.com/knadh/koanf/providers/env v1.0.0 h1:ufePaI9BnWH+ajuxGGiJ8pdTG0uLEUWC7/HDDPGLah0=
-github.com/knadh/koanf/providers/env v0.1.0/go.mod h1:RE8K9GbACJkeEnkl8L/Qcj8p4ZyPXZIQ191HJi44ZaQ=
+github.com/knadh/koanf/providers/env v1.0.0/go.mod h1:mzFyRZueYhb37oPmC1HAv/oGEEuyvJDA98r3XAa8Gak=
-github.com/knadh/koanf/providers/file v0.1.0 h1:fs6U7nrV58d3CFAFh8VTde8TM262ObYf3ODrc//Lp+c=
+github.com/knadh/koanf/providers/file v1.1.2 h1:aCC36YGOgV5lTtAFz2qkgtWdeQsgfxUkxDOe+2nQY3w=
-github.com/knadh/koanf/providers/file v0.1.0/go.mod h1:rjJ/nHQl64iYCtAW2QQnF0eSmDEX/YZ/eNFj5yR6BvA=
+github.com/knadh/koanf/providers/file v1.1.2/go.mod h1:/faSBcv2mxPVjFrXck95qeoyoZ5myJ6uxN8OOVNJJCI=
 github.com/knadh/koanf/providers/posflag v0.1.0 h1:mKJlLrKPcAP7Ootf4pBZWJ6J+4wHYujwipe7Ie3qW6U=
 github.com/knadh/koanf/providers/posflag v0.1.0/go.mod h1:SYg03v/t8ISBNrMBRMlojH8OsKowbkXV7giIbBVgbz0=
-github.com/knadh/koanf/v2 v2.0.1 h1:1dYGITt1I23x8cfx8ZnldtezdyaZtfAuRtIFOiRzK7g=
+github.com/knadh/koanf/v2 v2.1.2 h1:I2rtLRqXRy1p01m/utEtpZSSA6dcJbgGVuE27kW2PzQ=
-github.com/knadh/koanf/v2 v2.0.1/go.mod h1:ZeiIlIDXTE7w1lMT6UVcNiRAS2/rCeLn/GdLNvY1Dus=
+github.com/knadh/koanf/v2 v2.1.2/go.mod h1:Gphfaen0q1Fc1HTgJgSTC4oRX9R2R5ErYMZJy8fLJBo=
-github.com/lmittmann/tint v1.0.4 h1:LeYihpJ9hyGvE0w+K2okPTGUdVLfng1+nDNVR4vWISc=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
-github.com/lmittmann/tint v1.0.4/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/lmittmann/tint v1.0.5 h1:NQclAutOfYsqs2F1Lenue6OoWCajs5wJcP3DfWVpePw=
+github.com/lmittmann/tint v1.0.5/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
@@ -42,47 +50,41 @@ github.com/maxatome/go-testdeep v1.12.0 h1:Ql7Go8Tg0C1D/uMMX59LAoYK7LffeJQ6X2T04
 github.com/maxatome/go-testdeep v1.12.0/go.mod h1:lPZc/HAcJMP92l7yI6TRz1aZN5URwUBUAfUNvrclaNM=
 github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
 github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
-github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
-github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
 github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
 github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
 github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
-github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
-github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/tlinden/yadu v0.0.0-20240118202225-ec3f0b7fc355 h1:EmgK+IGUz2m42bFKteLY5SYJLn/CyBrz6nkgS22K8Bk=
+github.com/tlinden/yadu v0.1.3 h1:5cRCUmj+l5yvlM2irtpFBIJwVV2DPEgYSaWvF19FtcY=
-github.com/tlinden/yadu v0.0.0-20240118202225-ec3f0b7fc355/go.mod h1:l3bRmHKL9zGAR6pnBHY2HRPxBecf7L74BoBgOOpTcUA=
+github.com/tlinden/yadu v0.1.3/go.mod h1:l3bRmHKL9zGAR6pnBHY2HRPxBecf7L74BoBgOOpTcUA=
-github.com/tlinden/yadu v0.1.0 h1:qtCi1jxg392qVRLFyrJ2LYu6/PiKSp1LT02EX+mNLME=
-github.com/tlinden/yadu v0.1.0/go.mod h1:l3bRmHKL9zGAR6pnBHY2HRPxBecf7L74BoBgOOpTcUA=
-github.com/tlinden/yadu v0.1.1 h1:116oEUy9b4PcMF5wLL2dCFA/sn/praYutOnao07MROw=
-github.com/tlinden/yadu v0.1.1/go.mod h1:l3bRmHKL9zGAR6pnBHY2HRPxBecf7L74BoBgOOpTcUA=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/image v0.23.0 h1:HseQ7c2OpPKTPVzNjG5fwJsOTCiiwS4QdsYi5XU6H68=
+golang.org/x/image v0.23.0/go.mod h1:wJJBTdLfCCf3tiHa1fNxpZmUI4mmoZvwMCPP0ddoNKY=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190606173856-1492cefac77f/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
-golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
-golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
+golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ=
-golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
+golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
-golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

image.go

@@ -20,11 +20,16 @@ package main
 import (
 	"bytes"
 	"fmt"
-	"image/jpeg"
+	"image"
+	_ "image/gif"
+	_ "image/jpeg"
+	_ "image/png"
 	"log/slog"
 	"os"
 	"path/filepath"
 
+	_ "golang.org/x/image/webp"
+
 	"github.com/corona10/goimagehash"
 )
 
@@ -35,6 +40,7 @@ type Image struct {
 	Hash     *goimagehash.ImageHash
 	Data     *bytes.Reader
 	URI      string
+	Mime     string
 }
 
 // used for logging to avoid printing Data
@@ -49,21 +55,52 @@ func (img *Image) LogValue() slog.Value {
 // holds all images of an ad
 type Cache []*goimagehash.ImageHash
 
-func NewImage(buf *bytes.Reader, filename string, uri string) *Image {
+// filename comes from the scraper, it contains directory/base w/o suffix
+func NewImage(buf *bytes.Reader, filename, uri string) (*Image, error) {
+	_, imgconfig, err := image.DecodeConfig(buf)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode image: %w", err)
+	}
+
+	_, err = buf.Seek(0, 0)
+	if err != nil {
+		return nil, fmt.Errorf("failed to seek(0) on image buffer: %w", err)
+	}
+
+	if imgconfig == "jpeg" {
+		// we're  using the  format as file  extension, but  have used
+		// "jpg" in the past, so to be backwards compatible, stay with
+		// it.
+		imgconfig = "jpg"
+	}
+
+	if imgconfig == "" {
+		return nil, fmt.Errorf("failed to process image: unknown or unsupported image format (supported: jpg,png,gif,webp)")
+	}
+
+	filename += "." + imgconfig
+
 	img := &Image{
 		Filename: filename,
 		URI:      uri,
 		Data:     buf,
+		Mime:     imgconfig,
 	}
 
-	return img
+	slog.Debug("image MIME", "mime", img.Mime)
+
+	return img, nil
 }
 
 // Calculate diff hash of the image
 func (img *Image) CalcHash() error {
-	jpgdata, err := jpeg.Decode(img.Data)
+	jpgdata, format, err := image.Decode(img.Data)
 	if err != nil {
-		return fmt.Errorf("failed to decode JPEG image: %w", err)
+		return fmt.Errorf("failed to decode image: %w", err)
+	}
+
+	if format == "" {
+		return fmt.Errorf("failed to decode image: unknown or unsupported image format (supported: jpg,png,gif,webp)")
 	}
 
 	hash1, err := goimagehash.DifferenceHash(jpgdata)
@@ -133,12 +170,19 @@ func ReadImages(addir string, dont bool) (Cache, error) {
 
 			reader := bytes.NewReader(data.Bytes())
 
-			img := NewImage(reader, filename, "")
+			img, err := NewImage(reader, filename, "")
-			if err = img.CalcHash(); err != nil {
+			if err != nil {
 				return nil, err
 			}
 
-			slog.Debug("Caching image from file system", "image", img, "hash", img.Hash.ToString())
+			if err := img.CalcHash(); err != nil {
+				return nil, err
+			}
+
+			if img.Hash != nil {
+				slog.Debug("Caching image from file system", "image", img, "hash", img.Hash.ToString())
+			}
+
 			cache = append(cache, img.Hash)
 		}
 	}

kleingebaeck.1

@@ -133,7 +133,7 @@
 .\" ========================================================================
 .\"
 .IX Title "KLEINGEBAECK 1"
-.TH KLEINGEBAECK 1 "2024-01-25" "1" "User Commands"
+.TH KLEINGEBAECK 1 "2024-02-10" "1" "User Commands"
 .\" For nroff, turn off justification.  Always turn off hyphenation; it makes
 .\" way too many mistakes in technical documents.
 .if n .ad l
@@ -152,7 +152,7 @@ kleingebaeck \- kleinanzeigen.de backup tool
 \&    \-l \-\-limit   <num>      Limit the ads to download to <num>, default: load all.
 \&    \-c \-\-config  <file>     Use config file <file> (default: ~/.kleingebaeck).
 \&       \-\-ignoreerrors       Ignore HTTP errors, may lead to incomplete ad backup.
-\&    \-f \-\-force              Download images even if they already exist.
+\&    \-f \-\-force              Overwrite images and ads even if the already exist.
 \&    \-m \-\-manual             Show manual.
 \&    \-h \-\-help               Show usage.
 \&    \-V \-\-version            Show program version.
@@ -195,7 +195,7 @@ Be careful  if you want  to change the  template.  The variable  is a
 multiline string surrounded  by three double quotes. You  can left out
 certain   fields  and   use  any   formatting  you   like.  Refer   to
 <https://pkg.go.dev/text/template>  for   details  how  to   write  a
-template.
+template. Also read the \s-1TEMPLATES\s0 section below.
 .PP
 If you're on  windows and want to customize the  output directory, put
 it into single  quotes to avoid the backslashes  interpreted as escape
@@ -204,6 +204,94 @@ chars like this:
 .Vb 1
 \&    outdir = \*(AqC:\eData\eAds\*(Aq
 .Ve
+.SH "TEMPLATES"
+.IX Header "TEMPLATES"
+Various parts  of the configuration  can be modified  using templates:
+the output directory, the ad directory and the ad listing itself.
+.SS "\s-1OUTPUT DIR TEMPLATE\s0"
+.IX Subsection "OUTPUT DIR TEMPLATE"
+The config varialbe \f(CW\*(C`outdir\*(C'\fR or the command line parameter \f(CW\*(C`\-o\*(C'\fR take a
+template which may contain:
+.ie n .IP """{{.Year}}""" 4
+.el .IP "\f(CW{{.Year}}\fR" 4
+.IX Item "{{.Year}}"
+.PD 0
+.ie n .IP """{{.Month}}""" 4
+.el .IP "\f(CW{{.Month}}\fR" 4
+.IX Item "{{.Month}}"
+.ie n .IP """{{.Day}}""" 4
+.el .IP "\f(CW{{.Day}}\fR" 4
+.IX Item "{{.Day}}"
+.PD
+.PP
+That way you can create a new output directory for every backup
+run. For example:
+.PP
+.Vb 1
+\&    outdir = "/home/backups/ads\-{{.Year}}\-{{.Month}}\-{{.Day}}"
+.Ve
+.PP
+Or using the command line flag:
+.PP
+.Vb 1
+\&    \-o "/home/backups/ads\-{{.Year}}\-{{.Month}}\-{{.Day}}"
+.Ve
+.PP
+The default value is \f(CW\*(C`.\*(C'\fR \- the current directory.
+.SS "\s-1AD DIRECTORY TEMPLATE\s0"
+.IX Subsection "AD DIRECTORY TEMPLATE"
+The ad directory name can be modified using the following ad values:
+.IP "{{.Price}}" 4
+.IX Item "{{.Price}}"
+.PD 0
+.IP "{{.ID}}" 4
+.IX Item "{{.ID}}"
+.IP "{{.Category}}" 4
+.IX Item "{{.Category}}"
+.IP "{{.Condition}}" 4
+.IX Item "{{.Condition}}"
+.IP "{{.Created}}" 4
+.IX Item "{{.Created}}"
+.IP "{{.Slug}}" 4
+.IX Item "{{.Slug}}"
+.IP "{{.Text}}" 4
+.IX Item "{{.Text}}"
+.PD
+.PP
+It can only be configured in the config file. By default only
+\&\f(CW\*(C`{{.Slug}}\*(C'\fR is being used, this is the title of the ad in url format.
+.SS "\s-1AD TEMPLATE\s0"
+.IX Subsection "AD TEMPLATE"
+The ad listing itself can be modified as well, using the same
+variables as the ad name template above.
+.PP
+This is the default template:
+.PP
+.Vb 7
+\&    Title: {{.Title}}
+\&    Price: {{.Price}}
+\&    Id: {{.ID}}
+\&    Category: {{.Category}}
+\&    Condition: {{.Condition}}
+\&    Created: {{.Created}}
+\&    Expire: {{.Expire}}
+\&    
+\&    {{.Text}}
+.Ve
+.PP
+The config parameter to modify is \f(CW\*(C`template\*(C'\fR. See example.conf in the
+source repository. Please take care, since this is a multiline
+string. This is how it shall look if you modify it:
+.PP
+.Vb 2
+\&    template="""
+\&    Title: {{.Title}}
+\&    
+\&    {{.Text}}
+\&    """
+.Ve
+.PP
+That is, the content between the two \f(CW"""\fR chars is the template.
 .SH "SETUP"
 .IX Header "SETUP"
 To setup the tool, you need to lookup your userid on

kleingebaeck.go

@@ -14,7 +14,7 @@ SYNOPSYS
         -l --limit   <num>      Limit the ads to download to <num>, default: load all.
         -c --config  <file>     Use config file <file> (default: ~/.kleingebaeck).
            --ignoreerrors       Ignore HTTP errors, may lead to incomplete ad backup.
-        -f --force              Download images even if they already exist.
+        -f --force              Overwrite images and ads even if the already exist.
         -m --manual             Show manual.
         -h --help               Show usage.
         -V --version            Show program version.
@@ -55,6 +55,7 @@ CONFIGURATION
     multiline string surrounded by three double quotes. You can left out
     certain fields and use any formatting you like. Refer to
     <https://pkg.go.dev/text/template> for details how to write a template.
+    Also read the TEMPLATES section below.
 
     If you're on windows and want to customize the output directory, put it
     into single quotes to avoid the backslashes interpreted as escape chars
@@ -62,6 +63,71 @@ CONFIGURATION
 
         outdir = 'C:\Data\Ads'
 
+TEMPLATES
+    Various parts of the configuration can be modified using templates: the
+    output directory, the ad directory and the ad listing itself.
+
+  OUTPUT DIR TEMPLATE
+    The config varialbe "outdir" or the command line parameter "-o" take a
+    template which may contain:
+
+    "{{.Year}}"
+    "{{.Month}}"
+    "{{.Day}}"
+
+    That way you can create a new output directory for every backup run. For
+    example:
+
+        outdir = "/home/backups/ads-{{.Year}}-{{.Month}}-{{.Day}}"
+
+    Or using the command line flag:
+
+        -o "/home/backups/ads-{{.Year}}-{{.Month}}-{{.Day}}"
+
+    The default value is "." - the current directory.
+
+  AD DIRECTORY TEMPLATE
+    The ad directory name can be modified using the following ad values:
+
+    {{.Price}}
+    {{.ID}}
+    {{.Category}}
+    {{.Condition}}
+    {{.Created}}
+    {{.Slug}}
+    {{.Text}}
+
+    It can only be configured in the config file. By default only
+    "{{.Slug}}" is being used, this is the title of the ad in url format.
+
+  AD TEMPLATE
+    The ad listing itself can be modified as well, using the same variables
+    as the ad name template above.
+
+    This is the default template:
+
+        Title: {{.Title}}
+        Price: {{.Price}}
+        Id: {{.ID}}
+        Category: {{.Category}}
+        Condition: {{.Condition}}
+        Created: {{.Created}}
+        Expire: {{.Expire}}
+    
+        {{.Text}}
+
+    The config parameter to modify is "template". See example.conf in the
+    source repository. Please take care, since this is a multiline string.
+    This is how it shall look if you modify it:
+
+        template="""
+        Title: {{.Title}}
+    
+        {{.Text}}
+        """
+
+    That is, the content between the two """ chars is the template.
+
 SETUP
     To setup the tool, you need to lookup your userid on kleinanzeigen.de.
     Go to your ad overview page while NOT being logged in:

kleingebaeck.pod

@@ -13,7 +13,7 @@ kleingebaeck - kleinanzeigen.de backup tool
     -l --limit   <num>      Limit the ads to download to <num>, default: load all.
     -c --config  <file>     Use config file <file> (default: ~/.kleingebaeck).
        --ignoreerrors       Ignore HTTP errors, may lead to incomplete ad backup.
-    -f --force              Download images even if they already exist.
+    -f --force              Overwrite images and ads even if the already exist.
     -m --manual             Show manual.
     -h --help               Show usage.
     -V --version            Show program version.
@@ -55,7 +55,7 @@ Be careful  if you want  to change the  template.  The variable  is a
 multiline string surrounded  by three double quotes. You  can left out
 certain   fields  and   use  any   formatting  you   like.  Refer   to
 L<https://pkg.go.dev/text/template>  for   details  how  to   write  a
-template.
+template. Also read the TEMPLATES section below.
 
 If you're on  windows and want to customize the  output directory, put
 it into single  quotes to avoid the backslashes  interpreted as escape
@@ -63,6 +63,91 @@ chars like this:
 
     outdir = 'C:\Data\Ads'
 
+=head1 TEMPLATES
+
+Various parts  of the configuration  can be modified  using templates:
+the output directory, the ad directory and the ad listing itself.
+
+=head2 OUTPUT DIR TEMPLATE
+
+The config varialbe C<outdir> or the command line parameter C<-o> take a
+template which may contain:
+
+=over
+
+=item C<{{.Year}}>
+
+=item C<{{.Month}}>
+
+=item C<{{.Day}}>
+
+=back
+
+That way you can create a new output directory for every backup
+run. For example:
+
+    outdir = "/home/backups/ads-{{.Year}}-{{.Month}}-{{.Day}}"
+
+Or using the command line flag:
+
+    -o "/home/backups/ads-{{.Year}}-{{.Month}}-{{.Day}}"
+
+The default value is C<.> - the current directory.
+
+=head2 AD DIRECTORY TEMPLATE
+
+The ad directory name can be modified using the following ad values:
+
+=over
+
+=item {{.Price}}
+
+=item {{.ID}}
+
+=item {{.Category}}
+
+=item {{.Condition}}
+
+=item {{.Created}}
+
+=item {{.Slug}}
+
+=item {{.Text}}
+
+=back
+
+It can only be configured in the config file. By default only
+C<{{.Slug}}> is being used, this is the title of the ad in url format.
+
+=head2 AD TEMPLATE
+
+The ad listing itself can be modified as well, using the same
+variables as the ad name template above.
+
+This is the default template:
+
+    Title: {{.Title}}
+    Price: {{.Price}}
+    Id: {{.ID}}
+    Category: {{.Category}}
+    Condition: {{.Condition}}
+    Created: {{.Created}}
+    Expire: {{.Expire}}
+    
+    {{.Text}}
+
+The config parameter to modify is C<template>. See example.conf in the
+source repository. Please take care, since this is a multiline
+string. This is how it shall look if you modify it:
+
+    template="""
+    Title: {{.Title}}
+    
+    {{.Text}}
+    """
+
+That is, the content between the two C<"""> chars is the template.
+
 =head1 SETUP
 
 To setup the tool, you need to lookup your userid on

main.go

@@ -18,13 +18,16 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
 package main
 
 import (
+	"bufio"
 	"errors"
 	"fmt"
 	"io"
 	"log/slog"
 	"os"
+	"runtime"
 	"runtime/debug"
 
+	"github.com/inconshreveable/mousetrap"
 	"github.com/lmittmann/tint"
 	"github.com/tlinden/yadu"
 )
@@ -35,6 +38,25 @@ func main() {
 	os.Exit(Main(os.Stdout))
 }
 
+func init() {
+	// if we're running on Windows  AND if the user double clicked the
+	// exe  file from explorer, we  tell them and then  wait until any
+	// key has been hit, which  will make the cmd window disappear and
+	// thus give the user time to read it.
+	if runtime.GOOS == "windows" {
+		if mousetrap.StartedByExplorer() {
+			fmt.Println("Do no double click kleingebaeck.exe!")
+			fmt.Println("Please open a command shell and run it from there.")
+			fmt.Println()
+			fmt.Print("Press any key to quit: ")
+			_, err := bufio.NewReader(os.Stdin).ReadString('\n')
+			if err != nil {
+				panic(err)
+			}
+		}
+	}
+}
+
 func Main(output io.Writer) int {
 	logLevel := &slog.LevelVar{}
 	opts := &tint.Options{
@@ -112,10 +134,11 @@ func Main(output io.Writer) int {
 	slog.Debug("config", "conf", conf)
 
 	// prepare output dir
-	err = Mkdir(conf.Outdir)
+	outdir, err := OutDirName(conf)
 	if err != nil {
 		return Die(err)
 	}
+	conf.Outdir = outdir
 
 	// used for all HTTP requests
 	fetch, err := NewFetcher(conf)
@@ -123,6 +146,9 @@ func Main(output io.Writer) int {
 		return Die(err)
 	}
 
+	// setup ad dir registry, needed to check for duplicates
+	DirsVisited = make(map[string]int)
+
 	switch {
 	case len(conf.Adlinks) >= 1:
 		// directly backup ad listing[s]

main_test.go

@@ -182,13 +182,13 @@ var tests = []Tests{
 	{
 		name:     "download-all-ads",
 		args:     base + " -o t/out -u 1",
-		expect:   "Successfully downloaded 6 ads with 12 images to t/out",
+		expect:   "Successfully downloaded 7 ads with 16 images to t/out",
 		exitcode: 0,
 	},
 	{
 		name:     "download-all-ads-using-config",
 		args:     "kleingebaeck -c t/fullconfig.conf",
-		expect:   "Successfully downloaded 6 ads with 12 images to t/out",
+		expect:   "Successfully downloaded 7 ads with 16 images to t/out",
 		exitcode: 0,
 	},
 }
@@ -321,6 +321,17 @@ var adsrc = []AdConfig{
 		Created:   "Yesterday",
 		Images:    []string{"t/1.jpg", "t/2.jpg"},
 	},
+	{
+		Title:     "Ad with multiple img formats",
+		ID:        "7",
+		Price:     "5€",
+		Category:  "Klunker",
+		Text:      "Thing to sale",
+		Slug:      "seventh-ad",
+		Condition: "works",
+		Created:   "Yesterday",
+		Images:    []string{"t/1.png", "t/1.gif", "t/1.webp", "t/1.jpg"},
+	},
 }
 
 // An  Adsource  is used  to  construct  a  httpmock responder  for  a
@@ -334,14 +345,14 @@ type Adsource struct {
 }
 
 // Render a HTML template for an adlisting or an ad
-func GetTemplate(adconfigs []AdConfig, adconfig AdConfig, htmltemplate string) string {
+func GetTemplate(adconfigs []AdConfig, adconfig *AdConfig, htmltemplate string) string {
 	tmpl, err := tpl.New("template").Parse(htmltemplate)
 	if err != nil {
 		panic(err)
 	}
 
 	var out bytes.Buffer
-	if len(adconfig.ID) == 0 {
+	if adconfig.ID == "" {
 		err = tmpl.Execute(&out, adconfigs)
 	} else {
 		err = tmpl.Execute(&out, adconfig)
@@ -363,7 +374,7 @@ func InitValidSources() []Adsource {
 
 	// valid ad listing page 2
 	list2 := []AdConfig{
-		adsrc[3], adsrc[4], adsrc[5],
+		adsrc[3], adsrc[4], adsrc[5], adsrc[6],
 	}
 
 	// valid ad listing page 3, which is empty
@@ -376,15 +387,15 @@ func InitValidSources() []Adsource {
 	ads := []Adsource{
 		{
 			uri:     fmt.Sprintf("%s%s?userId=1", Baseuri, Listuri),
-			content: GetTemplate(list1, empty, LISTTPL),
+			content: GetTemplate(list1, &empty, LISTTPL),
 		},
 		{
 			uri:     fmt.Sprintf("%s%s?userId=1&pageNum=2", Baseuri, Listuri),
-			content: GetTemplate(list2, empty, LISTTPL),
+			content: GetTemplate(list2, &empty, LISTTPL),
 		},
 		{
 			uri:     fmt.Sprintf("%s%s?userId=1&pageNum=3", Baseuri, Listuri),
-			content: GetTemplate(list3, empty, LISTTPL),
+			content: GetTemplate(list3, &empty, LISTTPL),
 		},
 	}
 
@@ -392,7 +403,7 @@ func InitValidSources() []Adsource {
 	for _, ad := range adsrc {
 		ads = append(ads, Adsource{
 			uri:     fmt.Sprintf("%s/s-anzeige/%s/%s", Baseuri, ad.Slug, ad.ID),
-			content: GetTemplate(nil, ad, ADTPL),
+			content: GetTemplate(nil, &ad, ADTPL),
 		})
 	}
 
@@ -405,28 +416,28 @@ func InitInvalidSources() []Adsource {
 		{
 			// valid ad page but without content
 			uri:     fmt.Sprintf("%s/s-anzeige/empty/1", Baseuri),
-			content: GetTemplate(nil, empty, EMPTYPAGE),
+			content: GetTemplate(nil, &empty, EMPTYPAGE),
 		},
 		{
 			// some random foreign webpage
 			uri:     INVALIDURI,
-			content: GetTemplate(nil, empty, "<html>foo</html>"),
+			content: GetTemplate(nil, &empty, "<html>foo</html>"),
 		},
 		{
 			// some invalid page path
 			uri:     fmt.Sprintf("%s/anzeige/name/1", Baseuri),
-			content: GetTemplate(nil, empty, "<html></html>"),
+			content: GetTemplate(nil, &empty, "<html></html>"),
 		},
 		{
 			// some none-ad page
 			uri:     fmt.Sprintf("%s/anzeige/name/1/foo/bar", Baseuri),
-			content: GetTemplate(nil, empty, "<html>HTTP 404: /eine-anzeige/ does not exist!</html>"),
+			content: GetTemplate(nil, &empty, "<html>HTTP 404: /eine-anzeige/ does not exist!</html>"),
 			status:  404,
 		},
 		{
 			// valid ad page but 503
 			uri:     fmt.Sprintf("%s/s-anzeige/503/1", Baseuri),
-			content: GetTemplate(nil, empty, "<html>HTTP 503: service unavailable</html>"),
+			content: GetTemplate(nil, &empty, "<html>HTTP 503: service unavailable</html>"),
 			status:  503,
 		},
 	}
@@ -459,13 +470,13 @@ func SetIntercept(ads []Adsource) {
 	}
 
 	// we just use 2 images, put this here
-	for _, image := range []string{"t/1.jpg", "t/2.jpg"} {
+	for _, image := range []string{"t/1.jpg", "t/2.jpg", "t/1.png", "t/1.gif", "t/1.webp"} {
 		httpmock.RegisterResponder("GET", image,
 			httpmock.NewBytesResponder(200, GetImage(image)).HeaderAdd(headers))
 	}
 }
 
-func VerifyAd(advertisement AdConfig) error {
+func VerifyAd(advertisement *AdConfig) error {
 	body := advertisement.Title + advertisement.Price + advertisement.ID + "Kleinanzeigen => " +
 		advertisement.Category + advertisement.Condition + advertisement.Created
 
@@ -525,8 +536,8 @@ func TestMain(t *testing.T) {
 
 	// verify if downloaded ads match
 	for _, ad := range adsrc {
-		if err := VerifyAd(ad); err != nil {
+		if err := VerifyAd(&ad); err != nil {
-			t.Errorf(err.Error())
+			t.Error(err.Error())
 		}
 	}
 }

mkrel.sh

@@ -22,7 +22,12 @@ freebsd/amd64
 linux/amd64
 netbsd/amd64
 openbsd/amd64
-windows/amd64"
+windows/amd64
+freebsd/arm64
+linux/arm64
+netbsd/arm64
+openbsd/arm64
+windows/arm64"
 
 tool="$1"
 version="$2"

scrape.go

@@ -126,16 +126,32 @@ func ScrapeAd(fetch *Fetcher, uri string) error {
 
 	advertisement.CalculateExpire()
 
-	// write listing
+	// prepare ad dir name
-	addir, err := WriteAd(fetch.Config, advertisement)
+	addir, err := AdDirName(fetch.Config, advertisement)
 	if err != nil {
 		return err
 	}
 
+	proceed := CheckAdVisited(fetch.Config, addir)
+	if !proceed {
+		return nil
+	}
+
+	// write listing
+	err = WriteAd(fetch.Config, advertisement, addir)
+	if err != nil {
+		return err
+	}
+
+	// tell the user
 	slog.Debug("extracted ad listing", "ad", advertisement)
 
+	// stats
 	fetch.Config.IncrAds()
 
+	// register for later checks
+	DirsVisited[addir] = 1
+
 	return ScrapeImages(fetch, advertisement, addir)
 }
 
@@ -154,7 +170,9 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error {
 
 	for _, imguri := range advertisement.Images {
 		imguri := imguri
-		file := filepath.Join(adpath, fmt.Sprintf("%d.jpg", img))
+
+		// we append the suffix later in NewImage() based on image format
+		basefilename := filepath.Join(adpath, fmt.Sprintf("%d", img))
 
 		egroup.Go(func() error {
 			// wait a little
@@ -176,7 +194,11 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error {
 
 			reader := bytes.NewReader(buf.Bytes())
 
-			image := NewImage(reader, file, imguri)
+			image, err := NewImage(reader, basefilename, imguri)
+			if err != nil {
+				return err
+			}
+
 			err = image.CalcHash()
 			if err != nil {
 				return err
@@ -195,7 +217,7 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error {
 				return fmt.Errorf("failed to seek(0) on image reader: %w", err)
 			}
 
-			err = WriteImage(file, reader)
+			err = WriteImage(image.Filename, reader)
 			if err != nil {
 				return err
 			}

store.go

@@ -26,8 +26,36 @@ import (
 	"runtime"
 	"strings"
 	tpl "text/template"
+	"time"
 )
 
+type OutdirData struct {
+	Year, Day, Month string
+}
+
+func OutDirName(conf *Config) (string, error) {
+	tmpl, err := tpl.New("outdir").Parse(conf.Outdir)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse outdir template: %w", err)
+	}
+
+	buf := bytes.Buffer{}
+
+	now := time.Now()
+	data := OutdirData{
+		Year:  now.Format("2006"),
+		Month: now.Format("02"),
+		Day:   now.Format("01"),
+	}
+
+	err = tmpl.Execute(&buf, data)
+	if err != nil {
+		return "", fmt.Errorf("failed to execute outdir template: %w", err)
+	}
+
+	return buf.String(), nil
+}
+
 func AdDirName(conf *Config, advertisement *Ad) (string, error) {
 	tmpl, err := tpl.New("adname").Parse(conf.Adnametemplate)
 	if err != nil {
@@ -44,19 +72,13 @@ func AdDirName(conf *Config, advertisement *Ad) (string, error) {
 	return buf.String(), nil
 }
 
-func WriteAd(conf *Config, advertisement *Ad) (string, error) {
+func WriteAd(conf *Config, advertisement *Ad, addir string) error {
-	// prepare ad dir name
-	addir, err := AdDirName(conf, advertisement)
-	if err != nil {
-		return "", err
-	}
-
 	// prepare output dir
 	dir := filepath.Join(conf.Outdir, addir)
 
-	err = Mkdir(dir)
+	err := Mkdir(dir)
 	if err != nil {
-		return "", err
+		return err
 	}
 
 	// write ad file
@@ -64,7 +86,7 @@ func WriteAd(conf *Config, advertisement *Ad) (string, error) {
 
 	listingfd, err := os.Create(listingfile)
 	if err != nil {
-		return "", fmt.Errorf("failed to create Adlisting.txt: %w", err)
+		return fmt.Errorf("failed to create Adlisting.txt: %w", err)
 	}
 	defer listingfd.Close()
 
@@ -76,17 +98,17 @@ func WriteAd(conf *Config, advertisement *Ad) (string, error) {
 
 	tmpl, err := tpl.New("adlisting").Parse(conf.Template)
 	if err != nil {
-		return "", fmt.Errorf("failed to parse adlisting template: %w", err)
+		return fmt.Errorf("failed to parse adlisting template: %w", err)
 	}
 
 	err = tmpl.Execute(listingfd, advertisement)
 	if err != nil {
-		return "", fmt.Errorf("failed to execute adlisting template: %w", err)
+		return fmt.Errorf("failed to execute adlisting template: %w", err)
 	}
 
 	slog.Info("wrote ad listing", "listingfile", listingfile)
 
-	return addir, nil
+	return nil
 }
 
 func WriteImage(filename string, reader *bytes.Reader) error {
@@ -127,9 +149,29 @@ func ReadImage(filename string) (*bytes.Buffer, error) {
 
 func fileExists(filename string) bool {
 	info, err := os.Stat(filename)
-	if os.IsNotExist(err) {
+
+	if err != nil {
+		// return false on any error
 		return false
 	}
 
 	return !info.IsDir()
 }
+
+// check if  an addir has  already been  processed by current  run and
+// decide what to do
+func CheckAdVisited(conf *Config, adname string) bool {
+	if Exists(DirsVisited, adname) {
+		if conf.ForceDownload {
+			slog.Warn("an ad with the same name has already been downloaded, overwriting", "addir", adname)
+			return true
+		}
+
+		// don't overwrite
+		slog.Warn("an ad with the same name has already been downloaded, skipping (use -f to overwrite)", "addir", adname)
+		return false
+	}
+
+	// overwrite
+	return true
+}

t/httproot/templates/render.sh

@@ -1,5 +1,7 @@
 #!/bin/sh -x
 base="../kleinanzeigen"
+
+rm -rf $base
 mkdir -p $base
 
 echo "Generating /s-bestandsliste.html"

util.go

@@ -32,7 +32,7 @@ import (
 
 func Mkdir(dir string) error {
 	if _, err := os.Stat(dir); errors.Is(err, os.ErrNotExist) {
-		err := os.Mkdir(dir, os.ModePerm)
+		err := os.MkdirAll(dir, os.ModePerm)
 		if err != nil {
 			return fmt.Errorf("failed to create directory %s: %w", dir, err)
 		}
@@ -74,3 +74,12 @@ func IsNoTty() bool {
 func GetThrottleTime() time.Duration {
 	return time.Duration(rand.Intn(MaxThrottle-MinThrottle+1)+MinThrottle) * time.Millisecond
 }
+
+// look if a key in a map exists, generic variant
+func Exists[K comparable, V any](m map[K]V, v K) bool {
+	if _, ok := m[v]; ok {
+		return true
+	}
+
+	return false
+}