fix #87: document image distance hash

Recursively create ad dir including output dir. The output dir itself
is not being created separately anymore. That way, no directory will
be created if no ads could be downloaded.

README-de.md

@@ -222,6 +222,49 @@ Sowie alle Bilder.
 Das Format kann man mit der Variable `template` in der Konfiguration
 ändern. Die `example.conf` enthält ein Beispiel für das Standard Template.
 
+## Verhalten des Tools
+
+Es gibt einige Dinge über das Verhalten von kleingebäck, über die Du
+Bescheid wissen solltest:
+
+- alle HTML Seiten und Bilder werden immer heruntergeladen
+- es wird ein (konfigurierbarer) Useragent verwendet
+- HTTP Cookies werden beachtet
+- bei Fehlern wird dreimal mit unterschiedlichem Abstand erneut
+  versucht
+- Bilder Downloads laufen parallelisiert mit leicht unterschiedlichen
+  zeitlichen Abständen ab
+- Gleich aussehende Bilder werden nicht überschrieben
+
+Der letzte Punkt muss genauer erläutert werden:
+
+Wenn man bei Kleinanzeigen.de eine Anzeige einstellt und Bilder
+postet, werden diese dort in ihrer Grösse reduziert (durch Kompression
+und Verkleinerung der Bilder usw.). Diese reduzierten Bilder werden
+dann von kleingebäck heruntergeladen. Falls Du Deine original Bilder
+behalten hast, kannst Du diese danach in das Backupverzeichnis
+kopieren. Bei einem erneuten kleingebäck-Lauf werden diese Bilder dann
+nicht überschrieben.
+
+Wir verwenden dafür einen Algorythmus namens [distance
+hashing](https://github.com/corona10/goimagehash). Dieser Algorithmus
+prüft die Ähnlichkeit von Bildern. Diese können in ihrer Auflösung,
+Kompression, Farbtiefe und vielem mehr manipuliert worden sein und
+trotzdem als das "gleiche Bild" erkannt werden (wohlgemerkt nicht "das
+selbe": die Dateien sind durchaus unterschiedlich!). Bis zu einer
+Distance von 5 überschreiben wir keine Bilder, weil wir dann davon
+ausgehen, dass das lokal Vorhandene das Original ist.
+
+Bitte beachte aber, dass dies KEIN Cachingmechanismus ist: die Bilder
+werden trotzdem immer alle heruntergeladen. Das muss so sein, da wir
+uns nicht die Dateinamen anschauen können, da kleinanzeigen.de diese
+nämlich zu Zahlen umbenennt. Und die Dateinamen können sich auch
+ändern, wenn der User in der Anzeige die Bilder umarrangiert hat.
+
+Du kannst dieses Verhalten mit der Option **--force** ausschalten. Du
+kannst ausserdem mit der Option **--ignoreerrors** auch alle Fehler
+ignorieren, die beim Bilderdownload auftreten könnten.
+
 ## Documentation
 
 Die Dokumentation kann man

README.md

@@ -207,6 +207,48 @@ variable. The supplied sample config contains the default template.
 
 All images will be stored in the same directory.
 
+## Tool Behavior
+
+There are a bunch of things you might want to know about the behavior
+of the kleingebäck tool:
+
+- all HTML pages and IMAGEs are always being downloaded
+- we use a (customizable) user agent
+- we respect HTTP cookies
+- in the case of an error, the tool does 3 retries, the time it waits
+  between tries is longer for each retry
+- image download is parallized using small time differences to look
+  more natural
+- same images are not being overwritten on subsequent download
+
+
+The latter needs to be elaborated a bit more:
+
+If you publish an ad on kleinanzeigen.de and post images, those images
+will be reduced in size by the site (by compressing and down sizing
+them). This reduced images will be downloaded by kleingebäck. However,
+you may still own the original images and may want to put them into
+that backup directory so that you have all things for one ad together.
+
+You can easily do that, because kleingebäck won't overwrite those
+original images. It uses something called a distance hash using
+[goimagehash](https://github.com/corona10/goimagehash). This
+algorithmus checks the similarity of images. If an image has been
+resized it is still very similar to the original one. We accept a
+maximum of a distance of 5, everything above leads to overwrite.
+
+This works with resizes, cropped and otherwise manipulated images as
+long as the image still shows the original contents good enough.
+
+Also note, that this is NOT a caching mechanism: the images will be
+downloaded anyway during each run. We also can't look at the file
+names because kleinanzeigen.de renames all images to numbers. And
+those might even change if the user re-arranges the images.
+
+You can override this behavior using the **--force** option. Another
+option, **--ignoreerrors**, can be used to ignore all kinds of image
+errors. 
+
 ## Documentation
 
 You can read the documentation [online](https://github.com/TLINDEN/kleingebaeck/blob/main/kleingebaeck.pod) or locally once you have installed kleingebaeck with: `kleingebaeck --manual`.

config.go

@@ -34,7 +34,7 @@ import (
 )
 
 const (
-	VERSION    string = "0.3.3"
+	VERSION    string = "0.3.5"
 	Baseuri    string = "https://www.kleinanzeigen.de"
 	Listuri    string = "/s-bestandsliste.html"
 	Defaultdir string = "."
@@ -52,6 +52,8 @@ const (
 
 	DefaultAdNameTemplate string = "{{.Slug}}"
 
+	DefaultOutdirTemplate string = "."
+
 	// for image download throttling
 	MinThrottle int = 2
 	MaxThrottle int = 20
@@ -65,6 +67,8 @@ const (
 	WIN string = "windows"
 )
 
+var DirsVisited map[string]int
+
 const Usage string = `This is kleingebaeck, the kleinanzeigen.de backup tool.
 
 Usage: kleingebaeck [-dvVhmoclu] [<ad-listing-url>,...]
@@ -77,7 +81,7 @@ Options:
 -l --limit   <num>      Limit the ads to download to <num>, default: load all.
 -c --config  <file>     Use config file <file> (default: ~/.kleingebaeck).
    --ignoreerrors       Ignore HTTP errors, may lead to incomplete ad backup.
--f --force              Download images even if they already exist.
+-f --force              Overwrite images and ads even if the already exist.
 -m --manual             Show manual.
 -h --help               Show usage.
 -V --version            Show program version.
@@ -126,7 +130,7 @@ func InitConfig(output io.Writer) (*Config, error) {
 	// Load default values using the confmap provider.
 	if err := kloader.Load(confmap.Provider(map[string]interface{}{
 		"template":       template,
-		"outdir":         ".",
+		"outdir":         DefaultOutdirTemplate,
 		"loglevel":       "notice",
 		"userid":         0,
 		"adnametemplate": DefaultAdNameTemplate,

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/lmittmann/tint v1.0.4
 	github.com/mattn/go-isatty v0.0.20
 	github.com/spf13/pflag v1.0.5
-	github.com/tlinden/yadu v0.1.1
+	github.com/tlinden/yadu v0.1.2
 	golang.org/x/sync v0.5.0
 )
 
@@ -33,7 +33,7 @@ require (
 	github.com/pelletier/go-toml v1.9.5 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect
-	golang.org/x/sys v0.14.0 // indirect
+	golang.org/x/sys v0.17.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 
 )

go.sum

@@ -66,6 +66,8 @@ github.com/tlinden/yadu v0.1.0 h1:qtCi1jxg392qVRLFyrJ2LYu6/PiKSp1LT02EX+mNLME=
 github.com/tlinden/yadu v0.1.0/go.mod h1:l3bRmHKL9zGAR6pnBHY2HRPxBecf7L74BoBgOOpTcUA=
 github.com/tlinden/yadu v0.1.1 h1:116oEUy9b4PcMF5wLL2dCFA/sn/praYutOnao07MROw=
 github.com/tlinden/yadu v0.1.1/go.mod h1:l3bRmHKL9zGAR6pnBHY2HRPxBecf7L74BoBgOOpTcUA=
+github.com/tlinden/yadu v0.1.2 h1:TYYVnUJwziRJ9YPbIbRf9ikmDw0Q8Ifixm+J/kBQFh8=
+github.com/tlinden/yadu v0.1.2/go.mod h1:l3bRmHKL9zGAR6pnBHY2HRPxBecf7L74BoBgOOpTcUA=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -81,6 +83,8 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
 golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

kleingebaeck.1

@@ -133,7 +133,7 @@
 .\" ========================================================================
 .\"
 .IX Title "KLEINGEBAECK 1"
-.TH KLEINGEBAECK 1 "2024-01-25" "1" "User Commands"
+.TH KLEINGEBAECK 1 "2024-02-10" "1" "User Commands"
 .\" For nroff, turn off justification.  Always turn off hyphenation; it makes
 .\" way too many mistakes in technical documents.
 .if n .ad l
@@ -152,7 +152,7 @@ kleingebaeck \- kleinanzeigen.de backup tool
 \&    \-l \-\-limit   <num>      Limit the ads to download to <num>, default: load all.
 \&    \-c \-\-config  <file>     Use config file <file> (default: ~/.kleingebaeck).
 \&       \-\-ignoreerrors       Ignore HTTP errors, may lead to incomplete ad backup.
-\&    \-f \-\-force              Download images even if they already exist.
+\&    \-f \-\-force              Overwrite images and ads even if the already exist.
 \&    \-m \-\-manual             Show manual.
 \&    \-h \-\-help               Show usage.
 \&    \-V \-\-version            Show program version.
@@ -195,7 +195,7 @@ Be careful  if you want  to change the  template.  The variable  is a
 multiline string surrounded  by three double quotes. You  can left out
 certain   fields  and   use  any   formatting  you   like.  Refer   to
 <https://pkg.go.dev/text/template>  for   details  how  to   write  a
-template.
+template. Also read the \s-1TEMPLATES\s0 section below.
 .PP
 If you're on  windows and want to customize the  output directory, put
 it into single  quotes to avoid the backslashes  interpreted as escape
@@ -204,6 +204,94 @@ chars like this:
 .Vb 1
 \&    outdir = \*(AqC:\eData\eAds\*(Aq
 .Ve
+.SH "TEMPLATES"
+.IX Header "TEMPLATES"
+Various parts  of the configuration  can be modified  using templates:
+the output directory, the ad directory and the ad listing itself.
+.SS "\s-1OUTPUT DIR TEMPLATE\s0"
+.IX Subsection "OUTPUT DIR TEMPLATE"
+The config varialbe \f(CW\*(C`outdir\*(C'\fR or the command line parameter \f(CW\*(C`\-o\*(C'\fR take a
+template which may contain:
+.ie n .IP """{{.Year}}""" 4
+.el .IP "\f(CW{{.Year}}\fR" 4
+.IX Item "{{.Year}}"
+.PD 0
+.ie n .IP """{{.Month}}""" 4
+.el .IP "\f(CW{{.Month}}\fR" 4
+.IX Item "{{.Month}}"
+.ie n .IP """{{.Day}}""" 4
+.el .IP "\f(CW{{.Day}}\fR" 4
+.IX Item "{{.Day}}"
+.PD
+.PP
+That way you can create a new output directory for every backup
+run. For example:
+.PP
+.Vb 1
+\&    outdir = "/home/backups/ads\-{{.Year}}\-{{.Month}}\-{{.Day}}"
+.Ve
+.PP
+Or using the command line flag:
+.PP
+.Vb 1
+\&    \-o "/home/backups/ads\-{{.Year}}\-{{.Month}}\-{{.Day}}"
+.Ve
+.PP
+The default value is \f(CW\*(C`.\*(C'\fR \- the current directory.
+.SS "\s-1AD DIRECTORY TEMPLATE\s0"
+.IX Subsection "AD DIRECTORY TEMPLATE"
+The ad directory name can be modified using the following ad values:
+.IP "{{.Price}}" 4
+.IX Item "{{.Price}}"
+.PD 0
+.IP "{{.ID}}" 4
+.IX Item "{{.ID}}"
+.IP "{{.Category}}" 4
+.IX Item "{{.Category}}"
+.IP "{{.Condition}}" 4
+.IX Item "{{.Condition}}"
+.IP "{{.Created}}" 4
+.IX Item "{{.Created}}"
+.IP "{{.Slug}}" 4
+.IX Item "{{.Slug}}"
+.IP "{{.Text}}" 4
+.IX Item "{{.Text}}"
+.PD
+.PP
+It can only be configured in the config file. By default only
+\&\f(CW\*(C`{{.Slug}}\*(C'\fR is being used, this is the title of the ad in url format.
+.SS "\s-1AD TEMPLATE\s0"
+.IX Subsection "AD TEMPLATE"
+The ad listing itself can be modified as well, using the same
+variables as the ad name template above.
+.PP
+This is the default template:
+.PP
+.Vb 7
+\&    Title: {{.Title}}
+\&    Price: {{.Price}}
+\&    Id: {{.ID}}
+\&    Category: {{.Category}}
+\&    Condition: {{.Condition}}
+\&    Created: {{.Created}}
+\&    Expire: {{.Expire}}
+\&    
+\&    {{.Text}}
+.Ve
+.PP
+The config parameter to modify is \f(CW\*(C`template\*(C'\fR. See example.conf in the
+source repository. Please take care, since this is a multiline
+string. This is how it shall look if you modify it:
+.PP
+.Vb 2
+\&    template="""
+\&    Title: {{.Title}}
+\&    
+\&    {{.Text}}
+\&    """
+.Ve
+.PP
+That is, the content between the two \f(CW"""\fR chars is the template.
 .SH "SETUP"
 .IX Header "SETUP"
 To setup the tool, you need to lookup your userid on

kleingebaeck.go

@@ -14,7 +14,7 @@ SYNOPSYS
         -l --limit   <num>      Limit the ads to download to <num>, default: load all.
         -c --config  <file>     Use config file <file> (default: ~/.kleingebaeck).
            --ignoreerrors       Ignore HTTP errors, may lead to incomplete ad backup.
-        -f --force              Download images even if they already exist.
+        -f --force              Overwrite images and ads even if the already exist.
         -m --manual             Show manual.
         -h --help               Show usage.
         -V --version            Show program version.
@@ -55,6 +55,7 @@ CONFIGURATION
     multiline string surrounded by three double quotes. You can left out
     certain fields and use any formatting you like. Refer to
     <https://pkg.go.dev/text/template> for details how to write a template.
+    Also read the TEMPLATES section below.
 
     If you're on windows and want to customize the output directory, put it
     into single quotes to avoid the backslashes interpreted as escape chars
@@ -62,6 +63,71 @@ CONFIGURATION
 
         outdir = 'C:\Data\Ads'
 
+TEMPLATES
+    Various parts of the configuration can be modified using templates: the
+    output directory, the ad directory and the ad listing itself.
+
+  OUTPUT DIR TEMPLATE
+    The config varialbe "outdir" or the command line parameter "-o" take a
+    template which may contain:
+
+    "{{.Year}}"
+    "{{.Month}}"
+    "{{.Day}}"
+
+    That way you can create a new output directory for every backup run. For
+    example:
+
+        outdir = "/home/backups/ads-{{.Year}}-{{.Month}}-{{.Day}}"
+
+    Or using the command line flag:
+
+        -o "/home/backups/ads-{{.Year}}-{{.Month}}-{{.Day}}"
+
+    The default value is "." - the current directory.
+
+  AD DIRECTORY TEMPLATE
+    The ad directory name can be modified using the following ad values:
+
+    {{.Price}}
+    {{.ID}}
+    {{.Category}}
+    {{.Condition}}
+    {{.Created}}
+    {{.Slug}}
+    {{.Text}}
+
+    It can only be configured in the config file. By default only
+    "{{.Slug}}" is being used, this is the title of the ad in url format.
+
+  AD TEMPLATE
+    The ad listing itself can be modified as well, using the same variables
+    as the ad name template above.
+
+    This is the default template:
+
+        Title: {{.Title}}
+        Price: {{.Price}}
+        Id: {{.ID}}
+        Category: {{.Category}}
+        Condition: {{.Condition}}
+        Created: {{.Created}}
+        Expire: {{.Expire}}
+    
+        {{.Text}}
+
+    The config parameter to modify is "template". See example.conf in the
+    source repository. Please take care, since this is a multiline string.
+    This is how it shall look if you modify it:
+
+        template="""
+        Title: {{.Title}}
+    
+        {{.Text}}
+        """
+
+    That is, the content between the two """ chars is the template.
+
 SETUP
     To setup the tool, you need to lookup your userid on kleinanzeigen.de.
     Go to your ad overview page while NOT being logged in:

kleingebaeck.pod

@@ -13,7 +13,7 @@ kleingebaeck - kleinanzeigen.de backup tool
     -l --limit   <num>      Limit the ads to download to <num>, default: load all.
     -c --config  <file>     Use config file <file> (default: ~/.kleingebaeck).
        --ignoreerrors       Ignore HTTP errors, may lead to incomplete ad backup.
-    -f --force              Download images even if they already exist.
+    -f --force              Overwrite images and ads even if the already exist.
     -m --manual             Show manual.
     -h --help               Show usage.
     -V --version            Show program version.
@@ -55,7 +55,7 @@ Be careful  if you want  to change the  template.  The variable  is a
 multiline string surrounded  by three double quotes. You  can left out
 certain   fields  and   use  any   formatting  you   like.  Refer   to
 L<https://pkg.go.dev/text/template>  for   details  how  to   write  a
-template.
+template. Also read the TEMPLATES section below.
 
 If you're on  windows and want to customize the  output directory, put
 it into single  quotes to avoid the backslashes  interpreted as escape
@@ -63,6 +63,91 @@ chars like this:
 
     outdir = 'C:\Data\Ads'
 
+=head1 TEMPLATES
+
+Various parts  of the configuration  can be modified  using templates:
+the output directory, the ad directory and the ad listing itself.
+
+=head2 OUTPUT DIR TEMPLATE
+
+The config varialbe C<outdir> or the command line parameter C<-o> take a
+template which may contain:
+
+=over
+
+=item C<{{.Year}}>
+
+=item C<{{.Month}}>
+
+=item C<{{.Day}}>
+
+=back
+
+That way you can create a new output directory for every backup
+run. For example:
+
+    outdir = "/home/backups/ads-{{.Year}}-{{.Month}}-{{.Day}}"
+
+Or using the command line flag:
+
+    -o "/home/backups/ads-{{.Year}}-{{.Month}}-{{.Day}}"
+
+The default value is C<.> - the current directory.
+
+=head2 AD DIRECTORY TEMPLATE
+
+The ad directory name can be modified using the following ad values:
+
+=over
+
+=item {{.Price}}
+
+=item {{.ID}}
+
+=item {{.Category}}
+
+=item {{.Condition}}
+
+=item {{.Created}}
+
+=item {{.Slug}}
+
+=item {{.Text}}
+
+=back
+
+It can only be configured in the config file. By default only
+C<{{.Slug}}> is being used, this is the title of the ad in url format.
+
+=head2 AD TEMPLATE
+
+The ad listing itself can be modified as well, using the same
+variables as the ad name template above.
+
+This is the default template:
+
+    Title: {{.Title}}
+    Price: {{.Price}}
+    Id: {{.ID}}
+    Category: {{.Category}}
+    Condition: {{.Condition}}
+    Created: {{.Created}}
+    Expire: {{.Expire}}
+    
+    {{.Text}}
+
+The config parameter to modify is C<template>. See example.conf in the
+source repository. Please take care, since this is a multiline
+string. This is how it shall look if you modify it:
+
+    template="""
+    Title: {{.Title}}
+    
+    {{.Text}}
+    """
+
+That is, the content between the two C<"""> chars is the template.
+
 =head1 SETUP
 
 To setup the tool, you need to lookup your userid on

main.go

@@ -112,10 +112,11 @@ func Main(output io.Writer) int {
 	slog.Debug("config", "conf", conf)
 
 	// prepare output dir
-	err = Mkdir(conf.Outdir)
+	outdir, err := OutDirName(conf)
 	if err != nil {
 		return Die(err)
 	}
+	conf.Outdir = outdir
 
 	// used for all HTTP requests
 	fetch, err := NewFetcher(conf)
@@ -123,6 +124,9 @@ func Main(output io.Writer) int {
 		return Die(err)
 	}
 
+	// setup ad dir registry, needed to check for duplicates
+	DirsVisited = make(map[string]int)
+
 	switch {
 	case len(conf.Adlinks) >= 1:
 		// directly backup ad listing[s]

mkrel.sh

@@ -22,7 +22,12 @@ freebsd/amd64
 linux/amd64
 netbsd/amd64
 openbsd/amd64
-windows/amd64"
+windows/amd64
+freebsd/arm64
+linux/arm64
+netbsd/arm64
+openbsd/arm64
+windows/arm64"
 
 tool="$1"
 version="$2"

scrape.go

@@ -126,16 +126,32 @@ func ScrapeAd(fetch *Fetcher, uri string) error {
 
 	advertisement.CalculateExpire()
 
-	// write listing
+	// prepare ad dir name
-	addir, err := WriteAd(fetch.Config, advertisement)
+	addir, err := AdDirName(fetch.Config, advertisement)
 	if err != nil {
 		return err
 	}
 
+	proceed := CheckAdVisited(fetch.Config, addir)
+	if !proceed {
+		return nil
+	}
+
+	// write listing
+	err = WriteAd(fetch.Config, advertisement, addir)
+	if err != nil {
+		return err
+	}
+
+	// tell the user
 	slog.Debug("extracted ad listing", "ad", advertisement)
 
+	// stats
 	fetch.Config.IncrAds()
 
+	// register for later checks
+	DirsVisited[addir] = 1
+
 	return ScrapeImages(fetch, advertisement, addir)
 }
 

store.go

@@ -26,8 +26,36 @@ import (
 	"runtime"
 	"strings"
 	tpl "text/template"
+	"time"
 )
 
+type OutdirData struct {
+	Year, Day, Month string
+}
+
+func OutDirName(conf *Config) (string, error) {
+	tmpl, err := tpl.New("outdir").Parse(conf.Outdir)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse outdir template: %w", err)
+	}
+
+	buf := bytes.Buffer{}
+
+	now := time.Now()
+	data := OutdirData{
+		Year:  now.Format("2006"),
+		Month: now.Format("02"),
+		Day:   now.Format("01"),
+	}
+
+	err = tmpl.Execute(&buf, data)
+	if err != nil {
+		return "", fmt.Errorf("failed to execute outdir template: %w", err)
+	}
+
+	return buf.String(), nil
+}
+
 func AdDirName(conf *Config, advertisement *Ad) (string, error) {
 	tmpl, err := tpl.New("adname").Parse(conf.Adnametemplate)
 	if err != nil {
@@ -44,19 +72,13 @@ func AdDirName(conf *Config, advertisement *Ad) (string, error) {
 	return buf.String(), nil
 }
 
-func WriteAd(conf *Config, advertisement *Ad) (string, error) {
+func WriteAd(conf *Config, advertisement *Ad, addir string) error {
-	// prepare ad dir name
-	addir, err := AdDirName(conf, advertisement)
-	if err != nil {
-		return "", err
-	}
-
 	// prepare output dir
 	dir := filepath.Join(conf.Outdir, addir)
 
-	err = Mkdir(dir)
+	err := Mkdir(dir)
 	if err != nil {
-		return "", err
+		return err
 	}
 
 	// write ad file
@@ -64,7 +86,7 @@ func WriteAd(conf *Config, advertisement *Ad) (string, error) {
 
 	listingfd, err := os.Create(listingfile)
 	if err != nil {
-		return "", fmt.Errorf("failed to create Adlisting.txt: %w", err)
+		return fmt.Errorf("failed to create Adlisting.txt: %w", err)
 	}
 	defer listingfd.Close()
 
@@ -76,17 +98,17 @@ func WriteAd(conf *Config, advertisement *Ad) (string, error) {
 
 	tmpl, err := tpl.New("adlisting").Parse(conf.Template)
 	if err != nil {
-		return "", fmt.Errorf("failed to parse adlisting template: %w", err)
+		return fmt.Errorf("failed to parse adlisting template: %w", err)
 	}
 
 	err = tmpl.Execute(listingfd, advertisement)
 	if err != nil {
-		return "", fmt.Errorf("failed to execute adlisting template: %w", err)
+		return fmt.Errorf("failed to execute adlisting template: %w", err)
 	}
 
 	slog.Info("wrote ad listing", "listingfile", listingfile)
 
-	return addir, nil
+	return nil
 }
 
 func WriteImage(filename string, reader *bytes.Reader) error {
@@ -133,3 +155,21 @@ func fileExists(filename string) bool {
 
 	return !info.IsDir()
 }
+
+// check if  an addir has  already been  processed by current  run and
+// decide what to do
+func CheckAdVisited(conf *Config, adname string) bool {
+	if Exists(DirsVisited, adname) {
+		if conf.ForceDownload {
+			slog.Warn("an ad with the same name has already been downloaded, overwriting", "addir", adname)
+			return true
+		}
+
+		// don't overwrite
+		slog.Warn("an ad with the same name has already been downloaded, skipping (use -f to overwrite)", "addir", adname)
+		return false
+	}
+
+	// overwrite
+	return true
+}

t/httproot/templates/render.sh

@@ -1,5 +1,7 @@
 #!/bin/sh -x
 base="../kleinanzeigen"
+
+rm -rf $base
 mkdir -p $base
 
 echo "Generating /s-bestandsliste.html"

util.go

@@ -32,7 +32,7 @@ import (
 
 func Mkdir(dir string) error {
 	if _, err := os.Stat(dir); errors.Is(err, os.ErrNotExist) {
-		err := os.Mkdir(dir, os.ModePerm)
+		err := os.MkdirAll(dir, os.ModePerm)
 		if err != nil {
 			return fmt.Errorf("failed to create directory %s: %w", dir, err)
 		}
@@ -74,3 +74,12 @@ func IsNoTty() bool {
 func GetThrottleTime() time.Duration {
 	return time.Duration(rand.Intn(MaxThrottle-MinThrottle+1)+MinThrottle) * time.Millisecond
 }
+
+// look if a key in a map exists, generic variant
+func Exists[K comparable, V any](m map[K]V, v K) bool {
+	if _, ok := m[v]; ok {
+		return true
+	}
+
+	return false
+}