fix #71 and #73: add support for outdir template and enhance docs

.github/dependabot.yml

@@ -1,10 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: "gomod"
-    directory: "/"
-    schedule:
-      interval: "monthly"
-  - package-ecosystem: "github-actions"
-    directory: "/"
-    schedule:
-        interval: "monthly"

.github/workflows/ci.yaml

@@ -10,13 +10,13 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
     - name: Set up Go
-      uses: actions/setup-go@v5
+      uses: actions/setup-go@v3
       with:
         go-version: ${{ matrix.version }}
       id: go
 
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@v3
 
     - name: build
       run: go build
@@ -39,9 +39,9 @@ jobs:
     name: lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-go@v5
+      - uses: actions/setup-go@v3
         with:
           go-version: 1.21
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v6
+        uses: golangci/golangci-lint-action@v3

.github/workflows/pushimage.yaml

@@ -15,20 +15,20 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Log in to the Container registry
-        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567
+        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
         with:
           registry: https://ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85
+        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
         with:
           push: true
           tags: ghcr.io/tlinden/kleingebaeck:${{ github.ref_name}}
 
       - name: Build and push latest Docker image
-        uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85
+        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
         with:
           push: true
           tags: ghcr.io/tlinden/kleingebaeck:latest

Makefile

@@ -63,7 +63,6 @@ lint:
 
 lint-full:
 	golangci-lint run --enable-all --exclude-use-default --disable exhaustivestruct,exhaustruct,depguard,interfacer,deadcode,golint,structcheck,scopelint,varcheck,ifshort,maligned,nosnakecase,godot,funlen,gofumpt,cyclop,noctx,gochecknoglobals,paralleltest
-	gocritic check -enableAll *.go
 
 testfuzzy: clean
 	go test -fuzz ./... $(ARGS)

README-de.md

@@ -222,49 +222,6 @@ Sowie alle Bilder.
 Das Format kann man mit der Variable `template` in der Konfiguration
 ändern. Die `example.conf` enthält ein Beispiel für das Standard Template.
 
-## Verhalten des Tools
-
-Es gibt einige Dinge über das Verhalten von kleingebäck, über die Du
-Bescheid wissen solltest:
-
-- alle HTML Seiten und Bilder werden immer heruntergeladen
-- es wird ein (konfigurierbarer) Useragent verwendet
-- HTTP Cookies werden beachtet
-- bei Fehlern wird dreimal mit unterschiedlichem Abstand erneut
-  versucht
-- Bilder Downloads laufen parallelisiert mit leicht unterschiedlichen
-  zeitlichen Abständen ab
-- Gleich aussehende Bilder werden nicht überschrieben
-
-Der letzte Punkt muss genauer erläutert werden:
-
-Wenn man bei Kleinanzeigen.de eine Anzeige einstellt und Bilder
-postet, werden diese dort in ihrer Grösse reduziert (durch Kompression
-und Verkleinerung der Bilder usw.). Diese reduzierten Bilder werden
-dann von kleingebäck heruntergeladen. Falls Du Deine original Bilder
-behalten hast, kannst Du diese danach in das Backupverzeichnis
-kopieren. Bei einem erneuten kleingebäck-Lauf werden diese Bilder dann
-nicht überschrieben.
-
-Wir verwenden dafür einen Algorythmus namens [distance
-hashing](https://github.com/corona10/goimagehash). Dieser Algorithmus
-prüft die Ähnlichkeit von Bildern. Diese können in ihrer Auflösung,
-Kompression, Farbtiefe und vielem mehr manipuliert worden sein und
-trotzdem als das "gleiche Bild" erkannt werden (wohlgemerkt nicht "das
-selbe": die Dateien sind durchaus unterschiedlich!). Bis zu einer
-Distance von 5 überschreiben wir keine Bilder, weil wir dann davon
-ausgehen, dass das lokal Vorhandene das Original ist.
-
-Bitte beachte aber, dass dies KEIN Cachingmechanismus ist: die Bilder
-werden trotzdem immer alle heruntergeladen. Das muss so sein, da wir
-uns nicht die Dateinamen anschauen können, da kleinanzeigen.de diese
-nämlich zu Zahlen umbenennt. Und die Dateinamen können sich auch
-ändern, wenn der User in der Anzeige die Bilder umarrangiert hat.
-
-Du kannst dieses Verhalten mit der Option **--force** ausschalten. Du
-kannst ausserdem mit der Option **--ignoreerrors** auch alle Fehler
-ignorieren, die beim Bilderdownload auftreten könnten.
-
 ## Documentation
 
 Die Dokumentation kann man

README.md

@@ -207,48 +207,6 @@ variable. The supplied sample config contains the default template.
 
 All images will be stored in the same directory.
 
-## Tool Behavior
-
-There are a bunch of things you might want to know about the behavior
-of the kleingebäck tool:
-
-- all HTML pages and IMAGEs are always being downloaded
-- we use a (customizable) user agent
-- we respect HTTP cookies
-- in the case of an error, the tool does 3 retries, the time it waits
-  between tries is longer for each retry
-- image download is parallized using small time differences to look
-  more natural
-- same images are not being overwritten on subsequent download
-
-
-The latter needs to be elaborated a bit more:
-
-If you publish an ad on kleinanzeigen.de and post images, those images
-will be reduced in size by the site (by compressing and down sizing
-them). This reduced images will be downloaded by kleingebäck. However,
-you may still own the original images and may want to put them into
-that backup directory so that you have all things for one ad together.
-
-You can easily do that, because kleingebäck won't overwrite those
-original images. It uses something called a distance hash using
-[goimagehash](https://github.com/corona10/goimagehash). This
-algorithmus checks the similarity of images. If an image has been
-resized it is still very similar to the original one. We accept a
-maximum of a distance of 5, everything above leads to overwrite.
-
-This works with resizes, cropped and otherwise manipulated images as
-long as the image still shows the original contents good enough.
-
-Also note, that this is NOT a caching mechanism: the images will be
-downloaded anyway during each run. We also can't look at the file
-names because kleinanzeigen.de renames all images to numbers. And
-those might even change if the user re-arranges the images.
-
-You can override this behavior using the **--force** option. Another
-option, **--ignoreerrors**, can be used to ignore all kinds of image
-errors. 
-
 ## Documentation
 
 You can read the documentation [online](https://github.com/TLINDEN/kleingebaeck/blob/main/kleingebaeck.pod) or locally once you have installed kleingebaeck with: `kleingebaeck --manual`.

SECURITY.md

@@ -1,17 +0,0 @@
-# Security Policy
-
-## Supported Versions
-
-Only  the latest  release  is supported.  If you  find  an issue  (any
-issue!), please check with the latest release first.
-
-## Reporting a Vulnerability
-
-I don't agree with the "responsible disclosure" process most projects
-(and companies) work these days.
-
-So, if you find a vulnerability of any kind, please just open an
-[issue](https://github.com/TLINDEN/kleingebaeck/issues). Please add
-all details required to reproduce the vulnerability. You won't be chased.
-
-That's just all about it.

ad.go

@@ -73,7 +73,7 @@ func (ad *Ad) Incomplete() bool {
 }
 
 func (ad *Ad) CalculateExpire() {
-	if ad.Created != "" {
+	if len(ad.Created) > 0 {
 		ts, err := time.Parse("02.01.2006", ad.Created)
 		if err == nil {
 			ad.Expire = ts.AddDate(0, ExpireMonths, ExpireDays).Format("02.01.2006")

config.go

@@ -34,7 +34,7 @@ import (
 )
 
 const (
-	VERSION    string = "0.3.7"
+	VERSION    string = "0.3.4"
 	Baseuri    string = "https://www.kleinanzeigen.de"
 	Listuri    string = "/s-bestandsliste.html"
 	Defaultdir string = "."

fetch.go

@@ -52,7 +52,7 @@ func NewFetcher(conf *Config) (*Fetcher, error) {
 }
 
 func (f *Fetcher) Get(uri string) (io.ReadCloser, error) {
-	req, err := http.NewRequest(http.MethodGet, uri, http.NoBody)
+	req, err := http.NewRequest(http.MethodGet, uri, nil)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create a new HTTP request obj: %w", err)
 	}

go.mod

@@ -4,36 +4,36 @@ go 1.21
 
 require (
 	astuart.co/goq v1.0.0
-	github.com/corona10/goimagehash v1.1.0
-	github.com/inconshreveable/mousetrap v1.1.0
 	github.com/jarcoal/httpmock v1.3.1
 	github.com/knadh/koanf/parsers/toml v0.1.0
 	github.com/knadh/koanf/providers/confmap v0.1.0
 	github.com/knadh/koanf/providers/env v0.1.0
-	github.com/knadh/koanf/providers/file v1.1.0
+	github.com/knadh/koanf/providers/file v0.1.0
 	github.com/knadh/koanf/providers/posflag v0.1.0
-	github.com/knadh/koanf/v2 v2.1.1
+	github.com/knadh/koanf/v2 v2.0.1
-	github.com/lmittmann/tint v1.0.5
+	github.com/lmittmann/tint v1.0.4
 	github.com/mattn/go-isatty v0.0.20
 	github.com/spf13/pflag v1.0.5
-	github.com/tlinden/yadu v0.1.3
+	github.com/tlinden/yadu v0.1.2
-	golang.org/x/sync v0.8.0
+	golang.org/x/sync v0.5.0
 )
 
 require (
 	github.com/PuerkitoBio/goquery v1.5.1 // indirect
 	github.com/andybalholm/cascadia v1.1.0 // indirect
+	github.com/corona10/goimagehash v1.1.0 // indirect
 	github.com/fatih/color v1.16.0 // indirect
-	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
-	github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 // indirect
 	github.com/knadh/koanf/maps v0.1.1 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
-	golang.org/x/net v0.23.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
-	golang.org/x/sys v0.21.0 // indirect
+	golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect
+	golang.org/x/sys v0.17.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 
 )

go.sum

@@ -13,12 +13,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
-github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
+github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
-github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
+github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
-github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 h1:TQcrn6Wq+sKGkpyPvppOz99zsMBaUOKXq6HSv655U1c=
-github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
-github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
-github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww=
 github.com/jarcoal/httpmock v1.3.1/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg=
 github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NIs=
@@ -29,14 +25,14 @@ github.com/knadh/koanf/providers/confmap v0.1.0 h1:gOkxhHkemwG4LezxxN8DMOFopOPgh
 github.com/knadh/koanf/providers/confmap v0.1.0/go.mod h1:2uLhxQzJnyHKfxG927awZC7+fyHFdQkd697K4MdLnIU=
 github.com/knadh/koanf/providers/env v0.1.0 h1:LqKteXqfOWyx5Ab9VfGHmjY9BvRXi+clwyZozgVRiKg=
 github.com/knadh/koanf/providers/env v0.1.0/go.mod h1:RE8K9GbACJkeEnkl8L/Qcj8p4ZyPXZIQ191HJi44ZaQ=
-github.com/knadh/koanf/providers/file v1.1.0 h1:MTjA+gRrVl1zqgetEAIaXHqYje0XSosxSiMD4/7kz0o=
+github.com/knadh/koanf/providers/file v0.1.0 h1:fs6U7nrV58d3CFAFh8VTde8TM262ObYf3ODrc//Lp+c=
-github.com/knadh/koanf/providers/file v1.1.0/go.mod h1:/faSBcv2mxPVjFrXck95qeoyoZ5myJ6uxN8OOVNJJCI=
+github.com/knadh/koanf/providers/file v0.1.0/go.mod h1:rjJ/nHQl64iYCtAW2QQnF0eSmDEX/YZ/eNFj5yR6BvA=
 github.com/knadh/koanf/providers/posflag v0.1.0 h1:mKJlLrKPcAP7Ootf4pBZWJ6J+4wHYujwipe7Ie3qW6U=
 github.com/knadh/koanf/providers/posflag v0.1.0/go.mod h1:SYg03v/t8ISBNrMBRMlojH8OsKowbkXV7giIbBVgbz0=
-github.com/knadh/koanf/v2 v2.1.1 h1:/R8eXqasSTsmDCsAyYj+81Wteg8AqrV9CP6gvsTsOmM=
+github.com/knadh/koanf/v2 v2.0.1 h1:1dYGITt1I23x8cfx8ZnldtezdyaZtfAuRtIFOiRzK7g=
-github.com/knadh/koanf/v2 v2.1.1/go.mod h1:4mnTRbZCK+ALuBXHZMjDfG9y714L7TykVnZkXbMU3Es=
+github.com/knadh/koanf/v2 v2.0.1/go.mod h1:ZeiIlIDXTE7w1lMT6UVcNiRAS2/rCeLn/GdLNvY1Dus=
-github.com/lmittmann/tint v1.0.5 h1:NQclAutOfYsqs2F1Lenue6OoWCajs5wJcP3DfWVpePw=
+github.com/lmittmann/tint v1.0.4 h1:LeYihpJ9hyGvE0w+K2okPTGUdVLfng1+nDNVR4vWISc=
-github.com/lmittmann/tint v1.0.5/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE=
+github.com/lmittmann/tint v1.0.4/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
@@ -46,12 +42,16 @@ github.com/maxatome/go-testdeep v1.12.0 h1:Ql7Go8Tg0C1D/uMMX59LAoYK7LffeJQ6X2T04
 github.com/maxatome/go-testdeep v1.12.0/go.mod h1:lPZc/HAcJMP92l7yI6TRz1aZN5URwUBUAfUNvrclaNM=
 github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
 github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
+github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
+github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
 github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
 github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
 github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
@@ -60,22 +60,31 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/tlinden/yadu v0.1.3 h1:5cRCUmj+l5yvlM2irtpFBIJwVV2DPEgYSaWvF19FtcY=
+github.com/tlinden/yadu v0.0.0-20240118202225-ec3f0b7fc355 h1:EmgK+IGUz2m42bFKteLY5SYJLn/CyBrz6nkgS22K8Bk=
-github.com/tlinden/yadu v0.1.3/go.mod h1:l3bRmHKL9zGAR6pnBHY2HRPxBecf7L74BoBgOOpTcUA=
+github.com/tlinden/yadu v0.0.0-20240118202225-ec3f0b7fc355/go.mod h1:l3bRmHKL9zGAR6pnBHY2HRPxBecf7L74BoBgOOpTcUA=
+github.com/tlinden/yadu v0.1.0 h1:qtCi1jxg392qVRLFyrJ2LYu6/PiKSp1LT02EX+mNLME=
+github.com/tlinden/yadu v0.1.0/go.mod h1:l3bRmHKL9zGAR6pnBHY2HRPxBecf7L74BoBgOOpTcUA=
+github.com/tlinden/yadu v0.1.1 h1:116oEUy9b4PcMF5wLL2dCFA/sn/praYutOnao07MROw=
+github.com/tlinden/yadu v0.1.1/go.mod h1:l3bRmHKL9zGAR6pnBHY2HRPxBecf7L74BoBgOOpTcUA=
+github.com/tlinden/yadu v0.1.2 h1:TYYVnUJwziRJ9YPbIbRf9ikmDw0Q8Ifixm+J/kBQFh8=
+github.com/tlinden/yadu v0.1.2/go.mod h1:l3bRmHKL9zGAR6pnBHY2HRPxBecf7L74BoBgOOpTcUA=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190606173856-1492cefac77f/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0=
-golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
-golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
+golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
-golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

image.go

@@ -49,7 +49,7 @@ func (img *Image) LogValue() slog.Value {
 // holds all images of an ad
 type Cache []*goimagehash.ImageHash
 
-func NewImage(buf *bytes.Reader, filename, uri string) *Image {
+func NewImage(buf *bytes.Reader, filename string, uri string) *Image {
 	img := &Image{
 		Filename: filename,
 		URI:      uri,
@@ -134,7 +134,7 @@ func ReadImages(addir string, dont bool) (Cache, error) {
 			reader := bytes.NewReader(data.Bytes())
 
 			img := NewImage(reader, filename, "")
-			if err := img.CalcHash(); err != nil {
+			if err = img.CalcHash(); err != nil {
 				return nil, err
 			}
 

main.go

@@ -18,16 +18,13 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
 package main
 
 import (
-	"bufio"
 	"errors"
 	"fmt"
 	"io"
 	"log/slog"
 	"os"
-	"runtime"
 	"runtime/debug"
 
-	"github.com/inconshreveable/mousetrap"
 	"github.com/lmittmann/tint"
 	"github.com/tlinden/yadu"
 )
@@ -38,25 +35,6 @@ func main() {
 	os.Exit(Main(os.Stdout))
 }
 
-func init() {
-	// if we're running on Windows  AND if the user double clicked the
-	// exe  file from explorer, we  tell them and then  wait until any
-	// key has been hit, which  will make the cmd window disappear and
-	// thus give the user time to read it.
-	if runtime.GOOS == "windows" {
-		if mousetrap.StartedByExplorer() {
-			fmt.Println("Do no double click kleingebaeck.exe!")
-			fmt.Println("Please open a command shell and run it from there.")
-			fmt.Println()
-			fmt.Print("Press any key to quit: ")
-			_, err := bufio.NewReader(os.Stdin).ReadString('\n')
-			if err != nil {
-				panic(err)
-			}
-		}
-	}
-}
-
 func Main(output io.Writer) int {
 	logLevel := &slog.LevelVar{}
 	opts := &tint.Options{
@@ -138,6 +116,11 @@ func Main(output io.Writer) int {
 	if err != nil {
 		return Die(err)
 	}
+
+	err = Mkdir(outdir)
+	if err != nil {
+		return Die(err)
+	}
 	conf.Outdir = outdir
 
 	// used for all HTTP requests

main_test.go

@@ -334,14 +334,14 @@ type Adsource struct {
 }
 
 // Render a HTML template for an adlisting or an ad
-func GetTemplate(adconfigs []AdConfig, adconfig *AdConfig, htmltemplate string) string {
+func GetTemplate(adconfigs []AdConfig, adconfig AdConfig, htmltemplate string) string {
 	tmpl, err := tpl.New("template").Parse(htmltemplate)
 	if err != nil {
 		panic(err)
 	}
 
 	var out bytes.Buffer
-	if adconfig.ID == "" {
+	if len(adconfig.ID) == 0 {
 		err = tmpl.Execute(&out, adconfigs)
 	} else {
 		err = tmpl.Execute(&out, adconfig)
@@ -376,15 +376,15 @@ func InitValidSources() []Adsource {
 	ads := []Adsource{
 		{
 			uri:     fmt.Sprintf("%s%s?userId=1", Baseuri, Listuri),
-			content: GetTemplate(list1, &empty, LISTTPL),
+			content: GetTemplate(list1, empty, LISTTPL),
 		},
 		{
 			uri:     fmt.Sprintf("%s%s?userId=1&pageNum=2", Baseuri, Listuri),
-			content: GetTemplate(list2, &empty, LISTTPL),
+			content: GetTemplate(list2, empty, LISTTPL),
 		},
 		{
 			uri:     fmt.Sprintf("%s%s?userId=1&pageNum=3", Baseuri, Listuri),
-			content: GetTemplate(list3, &empty, LISTTPL),
+			content: GetTemplate(list3, empty, LISTTPL),
 		},
 	}
 
@@ -392,7 +392,7 @@ func InitValidSources() []Adsource {
 	for _, ad := range adsrc {
 		ads = append(ads, Adsource{
 			uri:     fmt.Sprintf("%s/s-anzeige/%s/%s", Baseuri, ad.Slug, ad.ID),
-			content: GetTemplate(nil, &ad, ADTPL),
+			content: GetTemplate(nil, ad, ADTPL),
 		})
 	}
 
@@ -405,28 +405,28 @@ func InitInvalidSources() []Adsource {
 		{
 			// valid ad page but without content
 			uri:     fmt.Sprintf("%s/s-anzeige/empty/1", Baseuri),
-			content: GetTemplate(nil, &empty, EMPTYPAGE),
+			content: GetTemplate(nil, empty, EMPTYPAGE),
 		},
 		{
 			// some random foreign webpage
 			uri:     INVALIDURI,
-			content: GetTemplate(nil, &empty, "<html>foo</html>"),
+			content: GetTemplate(nil, empty, "<html>foo</html>"),
 		},
 		{
 			// some invalid page path
 			uri:     fmt.Sprintf("%s/anzeige/name/1", Baseuri),
-			content: GetTemplate(nil, &empty, "<html></html>"),
+			content: GetTemplate(nil, empty, "<html></html>"),
 		},
 		{
 			// some none-ad page
 			uri:     fmt.Sprintf("%s/anzeige/name/1/foo/bar", Baseuri),
-			content: GetTemplate(nil, &empty, "<html>HTTP 404: /eine-anzeige/ does not exist!</html>"),
+			content: GetTemplate(nil, empty, "<html>HTTP 404: /eine-anzeige/ does not exist!</html>"),
 			status:  404,
 		},
 		{
 			// valid ad page but 503
 			uri:     fmt.Sprintf("%s/s-anzeige/503/1", Baseuri),
-			content: GetTemplate(nil, &empty, "<html>HTTP 503: service unavailable</html>"),
+			content: GetTemplate(nil, empty, "<html>HTTP 503: service unavailable</html>"),
 			status:  503,
 		},
 	}
@@ -465,7 +465,7 @@ func SetIntercept(ads []Adsource) {
 	}
 }
 
-func VerifyAd(advertisement *AdConfig) error {
+func VerifyAd(advertisement AdConfig) error {
 	body := advertisement.Title + advertisement.Price + advertisement.ID + "Kleinanzeigen => " +
 		advertisement.Category + advertisement.Condition + advertisement.Created
 
@@ -525,8 +525,8 @@ func TestMain(t *testing.T) {
 
 	// verify if downloaded ads match
 	for _, ad := range adsrc {
-		if err := VerifyAd(&ad); err != nil {
+		if err := VerifyAd(ad); err != nil {
-			t.Error(err.Error())
+			t.Errorf(err.Error())
 		}
 	}
 }

mkrel.sh

@@ -22,12 +22,7 @@ freebsd/amd64
 linux/amd64
 netbsd/amd64
 openbsd/amd64
-windows/amd64
+windows/amd64"
-freebsd/arm64
-linux/arm64
-netbsd/arm64
-openbsd/arm64
-windows/arm64"
 
 tool="$1"
 version="$2"

scrape.go

@@ -126,32 +126,21 @@ func ScrapeAd(fetch *Fetcher, uri string) error {
 
 	advertisement.CalculateExpire()
 
-	// prepare ad dir name
+	proceed := CheckAdVisited(fetch.Config, advertisement.Slug)
-	addir, err := AdDirName(fetch.Config, advertisement)
-	if err != nil {
-		return err
-	}
-
-	proceed := CheckAdVisited(fetch.Config, addir)
 	if !proceed {
 		return nil
 	}
 
 	// write listing
-	err = WriteAd(fetch.Config, advertisement, addir)
+	addir, err := WriteAd(fetch.Config, advertisement)
 	if err != nil {
 		return err
 	}
 
-	// tell the user
 	slog.Debug("extracted ad listing", "ad", advertisement)
 
-	// stats
 	fetch.Config.IncrAds()
 
-	// register for later checks
-	DirsVisited[addir] = 1
-
 	return ScrapeImages(fetch, advertisement, addir)
 }
 

store.go

@@ -72,13 +72,19 @@ func AdDirName(conf *Config, advertisement *Ad) (string, error) {
 	return buf.String(), nil
 }
 
-func WriteAd(conf *Config, advertisement *Ad, addir string) error {
+func WriteAd(conf *Config, advertisement *Ad) (string, error) {
+	// prepare ad dir name
+	addir, err := AdDirName(conf, advertisement)
+	if err != nil {
+		return "", err
+	}
+
 	// prepare output dir
 	dir := filepath.Join(conf.Outdir, addir)
 
-	err := Mkdir(dir)
+	err = Mkdir(dir)
 	if err != nil {
-		return err
+		return "", err
 	}
 
 	// write ad file
@@ -86,7 +92,7 @@ func WriteAd(conf *Config, advertisement *Ad, addir string) error {
 
 	listingfd, err := os.Create(listingfile)
 	if err != nil {
-		return fmt.Errorf("failed to create Adlisting.txt: %w", err)
+		return "", fmt.Errorf("failed to create Adlisting.txt: %w", err)
 	}
 	defer listingfd.Close()
 
@@ -98,17 +104,17 @@ func WriteAd(conf *Config, advertisement *Ad, addir string) error {
 
 	tmpl, err := tpl.New("adlisting").Parse(conf.Template)
 	if err != nil {
-		return fmt.Errorf("failed to parse adlisting template: %w", err)
+		return "", fmt.Errorf("failed to parse adlisting template: %w", err)
 	}
 
 	err = tmpl.Execute(listingfd, advertisement)
 	if err != nil {
-		return fmt.Errorf("failed to execute adlisting template: %w", err)
+		return "", fmt.Errorf("failed to execute adlisting template: %w", err)
 	}
 
 	slog.Info("wrote ad listing", "listingfile", listingfile)
 
-	return nil
+	return addir, nil
 }
 
 func WriteImage(filename string, reader *bytes.Reader) error {
@@ -170,6 +176,9 @@ func CheckAdVisited(conf *Config, adname string) bool {
 		return false
 	}
 
+	// register
+	DirsVisited[adname] = 1
+
 	// overwrite
 	return true
 }

util.go

@@ -32,7 +32,7 @@ import (
 
 func Mkdir(dir string) error {
 	if _, err := os.Stat(dir); errors.Is(err, os.ErrNotExist) {
-		err := os.MkdirAll(dir, os.ModePerm)
+		err := os.Mkdir(dir, os.ModePerm)
 		if err != nil {
 			return fmt.Errorf("failed to create directory %s: %w", dir, err)
 		}