Fix #47: add mock http server for testing w/o hitting prod site

.gitignore

@@ -3,3 +3,6 @@ kleingebaeck
 releases
 t/out
 .bak
+t/httproot/out
+t/httproot/kleinanzeigen
+t/httproot/favicon.ico

config.go

@@ -62,6 +62,7 @@ Options:
 -l --limit   <num>      Limit the ads to download to <num>, default: load all.
 -c --config  <file>     Use config file <file> (default: ~/.kleingebaeck).
    --ignoreerrors       Ignore HTTP errors, may lead to incomplete ad backup.
+-f --force              Download images even if they already exist.
 -m --manual             Show manual.
 -h --help               Show usage.
 -V --version            Show program version.
@@ -82,6 +83,7 @@ type Config struct {
 	Loglevel         string `koanf:"loglevel"`
 	Limit            int    `koanf:"limit"`
 	IgnoreErrors     bool   `koanf:"ignoreerrors"`
+	ForceDownload    bool   `koanf:"force"`
 	Adlinks          []string
 	StatsCountAds    int
 	StatsCountImages int
@@ -133,6 +135,7 @@ func InitConfig(w io.Writer) (*Config, error) {
 	f.BoolP("version", "V", false, "show program version")
 	f.BoolP("help", "h", false, "show usage")
 	f.BoolP("manual", "m", false, "show manual")
+	f.BoolP("force", "f", false, "force")
 
 	if err := f.Parse(os.Args[1:]); err != nil {
 		return nil, err

go.mod

@@ -21,6 +21,7 @@ require (
 require (
 	github.com/PuerkitoBio/goquery v1.5.1 // indirect
 	github.com/andybalholm/cascadia v1.1.0 // indirect
+	github.com/corona10/goimagehash v1.1.0 // indirect
 	github.com/fatih/color v1.16.0 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/knadh/koanf/maps v0.1.1 // indirect
@@ -28,6 +29,7 @@ require (
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
 	golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect
 	golang.org/x/sys v0.14.0 // indirect

go.sum

@@ -6,6 +6,8 @@ github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBK
 github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
 github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
 github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/corona10/goimagehash v1.1.0 h1:teNMX/1e+Wn/AYSbLHX8mj+mF9r60R1kBeqE9MkoYwI=
+github.com/corona10/goimagehash v1.1.0/go.mod h1:VkvE0mLn84L4aF8vCb6mafVajEb6QYMHl2ZJLn0mOGI=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -44,6 +46,8 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
 github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
+github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
+github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
 github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
 github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

image.go

@@ -0,0 +1,142 @@
+/*
+Copyright © 2023-2024 Thomas von Dein
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package main
+
+import (
+	"bytes"
+	"image/jpeg"
+	"log/slog"
+	"os"
+	"path/filepath"
+
+	"github.com/corona10/goimagehash"
+)
+
+const MaxDistance = 3
+
+type Image struct {
+	Filename string
+	Hash     *goimagehash.ImageHash
+	Data     *bytes.Buffer
+	Uri      string
+}
+
+// used for logging to avoid printing Data
+func (img *Image) LogValue() slog.Value {
+	return slog.GroupValue(
+		slog.String("filename", img.Filename),
+		slog.String("uri", img.Uri),
+		slog.String("hash", img.Hash.ToString()),
+	)
+}
+
+// holds all images of an ad
+type Cache []*goimagehash.ImageHash
+
+func NewImage(buf *bytes.Buffer, filename string, uri string) *Image {
+	img := &Image{
+		Filename: filename,
+		Uri:      uri,
+		Data:     buf,
+	}
+
+	return img
+}
+
+// Calculate diff hash of the image
+func (img *Image) CalcHash() error {
+	jpgdata, err := jpeg.Decode(img.Data)
+	if err != nil {
+		return err
+	}
+
+	hash1, err := goimagehash.DifferenceHash(jpgdata)
+	if err != nil {
+		return err
+	}
+
+	img.Hash = hash1
+
+	return nil
+}
+
+// checks if 2 images are similar enough to be considered the same
+func (img *Image) Similar(hash *goimagehash.ImageHash) bool {
+	distance, err := img.Hash.Distance(hash)
+	if err != nil {
+		slog.Debug("failed to compute diff hash distance", "error", err)
+		return false
+	}
+
+	if distance < MaxDistance {
+		slog.Debug("distance computation", "image-A", img.Hash.ToString(),
+			"image-B", hash.ToString(), "distance", distance)
+		return true
+	} else {
+		return false
+	}
+}
+
+// check current image against all known hashes.
+func (img *Image) SimilarExists(cache Cache) bool {
+	for _, otherimg := range cache {
+		if img.Similar(otherimg) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// read all  JPG images  in a  ad directory,  compute diff  hashes and
+// store the results in the slice Images
+func ReadImages(addir string, dont bool) (Cache, error) {
+	files, err := os.ReadDir(addir)
+	if err != nil {
+		return nil, err
+	}
+
+	cache := Cache{}
+
+	if dont {
+		// forced download, -f given
+		return cache, nil
+	}
+
+	for _, file := range files {
+		ext := filepath.Ext(file.Name())
+		if !file.IsDir() && (ext == ".jpg" || ext == ".jpeg" || ext == ".JPG" || ext == ".JPEG") {
+			filename := filepath.Join(addir, file.Name())
+			data, err := ReadImage(filename)
+			if err != nil {
+				return nil, err
+			}
+
+			img := NewImage(data, filename, "")
+			if err = img.CalcHash(); err != nil {
+				return nil, err
+			}
+
+			slog.Debug("Caching image from file system", "image", img, "hash", img.Hash.ToString())
+			cache = append(cache, img.Hash)
+		}
+	}
+
+	//return nil, errors.New("ende")
+	return cache, nil
+}

kleingebaeck.1

@@ -133,7 +133,7 @@
 .\" ========================================================================
 .\"
 .IX Title "KLEINGEBAECK 1"
-.TH KLEINGEBAECK 1 "2024-01-17" "1" "User Commands"
+.TH KLEINGEBAECK 1 "2024-01-22" "1" "User Commands"
 .\" For nroff, turn off justification.  Always turn off hyphenation; it makes
 .\" way too many mistakes in technical documents.
 .if n .ad l
@@ -142,7 +142,7 @@
 kleingebaeck \- kleinanzeigen.de backup tool
 .SH "SYNOPSYS"
 .IX Header "SYNOPSYS"
-.Vb 12
+.Vb 10
 \&    Usage: kleingebaeck [\-dvVhmoc] [<ad\-listing\-url>,...]
 \&    Options:
 \&    \-u \-\-user    <uid>      Backup ads from user with uid <uid>.
@@ -152,6 +152,7 @@ kleingebaeck \- kleinanzeigen.de backup tool
 \&    \-l \-\-limit   <num>      Limit the ads to download to <num>, default: load all.
 \&    \-c \-\-config  <file>     Use config file <file> (default: ~/.kleingebaeck).
 \&       \-\-ignoreerrors       Ignore HTTP errors, may lead to incomplete ad backup.
+\&    \-f \-\-force              Download images even if they already exist.
 \&    \-m \-\-manual             Show manual.
 \&    \-h \-\-help               Show usage.
 \&    \-V \-\-version            Show program version.

kleingebaeck.go

@@ -14,6 +14,7 @@ SYNOPSYS
         -l --limit   <num>      Limit the ads to download to <num>, default: load all.
         -c --config  <file>     Use config file <file> (default: ~/.kleingebaeck).
            --ignoreerrors       Ignore HTTP errors, may lead to incomplete ad backup.
+        -f --force              Download images even if they already exist.
         -m --manual             Show manual.
         -h --help               Show usage.
         -V --version            Show program version.

kleingebaeck.pod

@@ -13,6 +13,7 @@ kleingebaeck - kleinanzeigen.de backup tool
     -l --limit   <num>      Limit the ads to download to <num>, default: load all.
     -c --config  <file>     Use config file <file> (default: ~/.kleingebaeck).
        --ignoreerrors       Ignore HTTP errors, may lead to incomplete ad backup.
+    -f --force              Download images even if they already exist.
     -m --manual             Show manual.
     -h --help               Show usage.
     -V --version            Show program version.

scrape.go

@@ -18,6 +18,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
 package main
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"log/slog"
@@ -119,14 +120,14 @@ func ScrapeAd(fetch *Fetcher, uri string) error {
 
 	ad.CalculateExpire()
 
-	slog.Debug("extracted ad listing", "ad", ad)
-
 	// write listing
 	addir, err := WriteAd(fetch.Config, ad)
 	if err != nil {
 		return err
 	}
 
+	slog.Debug("extracted ad listing", "ad", ad)
+
 	fetch.Config.IncrAds()
 
 	return ScrapeImages(fetch, ad, addir)
@@ -135,22 +136,52 @@ func ScrapeAd(fetch *Fetcher, uri string) error {
 func ScrapeImages(fetch *Fetcher, ad *Ad, addir string) error {
 	// fetch images
 	img := 1
+	adpath := filepath.Join(fetch.Config.Outdir, addir)
+
+	// scan existing images, if any
+	cache, err := ReadImages(adpath, fetch.Config.ForceDownload)
+	if err != nil {
+		return err
+	}
+
 	g := new(errgroup.Group)
 
 	for _, imguri := range ad.Images {
 		imguri := imguri
-		file := filepath.Join(fetch.Config.Outdir, addir, fmt.Sprintf("%d.jpg", img))
+		file := filepath.Join(adpath, fmt.Sprintf("%d.jpg", img))
 		g.Go(func() error {
 			body, err := fetch.Getimage(imguri)
 			if err != nil {
 				return err
 			}
 
-			err = WriteImage(file, body)
+			buf := new(bytes.Buffer)
+			_, err = buf.ReadFrom(body)
 			if err != nil {
 				return err
 			}
 
+			buf2 := buf.Bytes() // needed for image writing
+
+			image := NewImage(buf, "", imguri)
+			err = image.CalcHash()
+			if err != nil {
+				return err
+			}
+
+			if !fetch.Config.ForceDownload {
+				if image.SimilarExists(cache) {
+					slog.Debug("similar image exists, not written", "uri", image.Uri)
+					return nil
+				}
+			}
+
+			err = WriteImage(file, buf2)
+			if err != nil {
+				return err
+			}
+
+			slog.Debug("wrote image", "image", image, "size", len(buf2))
 			return nil
 		})
 		img++

store.go

@@ -19,7 +19,7 @@ package main
 
 import (
 	"bytes"
-	"io"
+	"fmt"
 	"log/slog"
 	"os"
 	"path/filepath"
@@ -86,17 +86,46 @@ func WriteAd(c *Config, ad *Ad) (string, error) {
 	return addir, nil
 }
 
-func WriteImage(filename string, reader io.ReadCloser) error {
+func WriteImage(filename string, buf []byte) error {
 	file, err := os.Create(filename)
 	if err != nil {
 		return err
 	}
 	defer file.Close()
 
-	_, err = io.Copy(file, reader)
+	_, err = file.Write(buf)
+
 	if err != nil {
 		return err
 	}
 
 	return nil
 }
+
+func ReadImage(filename string) (*bytes.Buffer, error) {
+	var buf bytes.Buffer
+
+	if !fileExists(filename) {
+		return nil, fmt.Errorf("image %s does not exist", filename)
+	}
+
+	data, err := os.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = buf.Write(data)
+	if err != nil {
+		return nil, err
+	}
+
+	return &buf, nil
+}
+
+func fileExists(filename string) bool {
+	info, err := os.Stat(filename)
+	if os.IsNotExist(err) {
+		return false
+	}
+	return !info.IsDir()
+}

t/httproot/README.md

@@ -0,0 +1,13 @@
+# Mock http server
+
+Install ehfs from https://github.com/mjpclab/extra-http-file-server/.
+
+Install p2cli from https://github.com/wrouesnel/p2cli.
+
+Run `templates/render.sh` to build the file structure.
+
+Run `server.sh` to start the http server.
+
+To scrape an ad from it, use such a URL:
+
+http://localhost:8080/s-anzeige/first-ad/111-11-111

t/httproot/serve.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+ehfs -a :/s-anzeige:./kleinanzeigen \
+     -a :/api/v1/prod-ads/images/fc:./img \
+     -l localhost:8080 -I index.html

t/httproot/templates/ad.tpl

@@ -0,0 +1,50 @@
+<!DOCTYPE html>
+<html lang="de">
+  <head>
+    <title>Ad Listing</title>
+  </head>
+  <body>
+
+    <div class="l-container-row">
+        <div id="vap-brdcrmb" class="breadcrump">
+            <a class="breadcrump-link" itemprop="url" href="/" title="Kleinanzeigen ">
+                <span itemprop="title">Kleinanzeigen </span>
+            </a>
+            <a class="breadcrump-link" itemprop="url" href="/egal">
+               <span itemprop="title">{{ category }}</span></a>
+            </div>
+    </div>
+
+    {% for image in images %}
+    <div class="galleryimage-element" data-ix="3">
+      <img src="http://localhost:8080/api/v1/prod-ads/images/fc/{{ image.id }}?rule=$_59.JPG"/>
+    </div>
+    {% endfor %}
+
+    <h1 id="viewad-title" class="boxedarticle--title" itemprop="name" data-soldlabel="Verkauft">
+      {{ title }}</h1>
+    <div class="boxedarticle--flex--container">
+      <h2 class="boxedarticle--price" id="viewad-price">
+        {{ price }}</h2>
+    </div>
+
+    <div id="viewad-extra-info" class="boxedarticle--details--full">
+      <div><i class="icon icon-small icon-calendar-gray-simple"></i><span>{{ created }}</span></div>
+    </div>
+
+    <div class="splitlinebox l-container-row" id="viewad-details">
+      <ul class="addetailslist">
+        <li class="addetailslist--detail">
+          Zustand<span class="addetailslist--detail--value" >
+          {{ condition }}</span>
+        </li>
+      </ul>
+    </div>
+
+    <div class="l-container last-paragraph-no-margin-bottom">
+      <p id="viewad-description-text" class="text-force-linebreak " itemprop="description">
+        {{ text }}
+      </p>
+    </div>
+  </body>
+</html>

t/httproot/templates/index.tpl

@@ -0,0 +1,15 @@
+<!DOCTYPE html>
+<html lang="de" >
+  <head>
+    <title>Ads</title>
+  </head>
+  <body>
+    {% for ad in ads %}
+     <h2 class="text-module-begin">
+        <a class="ellipsis"
+           href="/s-anzeige/{{ ad.slug }}/{{ ad.id }}">{{ ad.title }}</a>
+     </h2>
+    {% endfor %}
+  </body>
+</html>
+

t/httproot/templates/render.sh

@@ -0,0 +1,13 @@
+#!/bin/sh -x
+base="../kleinanzeigen"
+mkdir -p $base
+
+echo "Generating /s-bestandsliste.html"
+p2cli -t index.tpl -i vars.yaml > $base/s-bestandsliste.html
+
+for idx in 0 1; do
+    slug=$(cat vars.yaml | yq ".ads[$idx].slug")
+    id=$(cat vars.yaml | yq ".ads[$idx].id")
+    mkdir -p $base/$slug/$id
+    cat vars.yaml | yq ".ads[$idx]" | p2cli -t ad.tpl -f yaml > $base/$slug/$id/index.html
+done

t/httproot/templates/vars.yaml

@@ -0,0 +1,27 @@
+ads:
+  - slug: first-ad
+    id: 111-11-111
+    title: First Ad
+    price: "19 €"
+    condition: "Sehr gut"
+    category: "Weitere Elektronik"
+    created: 21.12.2023
+    images:
+      - id: fcf6d664-5258-42c2-bf58-d1b8e9221574
+      - id: fcf6d664-5258-42c2-bf58-as43as5d43as
+    text: |
+      Zu Verkaufen.
+      Zahlung nur Paypal.
+  - slug: second-ad
+    id: 222-22-222
+    title: Second Ad
+    price: "200 €"
+    condition: "Sehr gut"
+    category: "Elektronik"
+    created: 21.12.2023
+    images:
+      - id: cdas4sd5-5258-42c2-bf58-d1b8e9221574
+      - id: cdas4sd5-5258-42c2-bf58-as43as5d43as
+    text: |
+      Zu Verkaufen.
+      Zahlung nur Überweisung.