bump version

Bumps [github.com/lmittmann/tint](https://github.com/lmittmann/tint) from 1.0.5 to 1.0.6.
- [Release notes](https://github.com/lmittmann/tint/releases)
- [Commits](https://github.com/lmittmann/tint/compare/v1.0.5...v1.0.6)

---
updated-dependencies:
- dependency-name: github.com/lmittmann/tint
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

.github/workflows/pushimage.yaml

@@ -22,13 +22,13 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75
+        uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355
         with:
           push: true
           tags: ghcr.io/tlinden/kleingebaeck:${{ github.ref_name}}
 
       - name: Build and push latest Docker image
-        uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75
+        uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355
         with:
           push: true
           tags: ghcr.io/tlinden/kleingebaeck:latest

config.go

@@ -34,7 +34,7 @@ import (
 )
 
 const (
-	VERSION    string = "0.3.9"
+	VERSION    string = "0.3.12"
 	Baseuri    string = "https://www.kleinanzeigen.de"
 	Listuri    string = "/s-bestandsliste.html"
 	Defaultdir string = "."
@@ -183,13 +183,22 @@ func InitConfig(output io.Writer) (*Config, error) {
 
 	// Load the config file[s]
 	for _, cfgfile := range configfiles {
-		if path, err := os.Stat(cfgfile); !os.IsNotExist(err) {
-			if !path.IsDir() {
-				if err := kloader.Load(file.Provider(cfgfile), toml.Parser()); err != nil {
-					return nil, fmt.Errorf("error loading config file: %w", err)
-				}
+		path, err := os.Stat(cfgfile)
+
+		if err != nil {
+			// ignore non-existent files, but bail out on any other errors
+			if !os.IsNotExist(err) {
+				return nil, fmt.Errorf("failed to stat config file: %w", err)
 			}
-		} // else: we ignore the file if it doesn't exists
+
+			continue
+		}
+
+		if !path.IsDir() {
+			if err := kloader.Load(file.Provider(cfgfile), toml.Parser()); err != nil {
+				return nil, fmt.Errorf("error loading config file: %w", err)
+			}
+		}
 	}
 
 	// env overrides config file

go.mod

@@ -12,12 +12,13 @@ require (
 	github.com/knadh/koanf/providers/env v1.0.0
 	github.com/knadh/koanf/providers/file v1.1.2
 	github.com/knadh/koanf/providers/posflag v0.1.0
-	github.com/knadh/koanf/v2 v2.1.1
-	github.com/lmittmann/tint v1.0.5
+	github.com/knadh/koanf/v2 v2.1.2
+	github.com/lmittmann/tint v1.0.6
 	github.com/mattn/go-isatty v0.0.20
 	github.com/spf13/pflag v1.0.5
 	github.com/tlinden/yadu v0.1.3
-	golang.org/x/sync v0.8.0
+	golang.org/x/image v0.23.0
+	golang.org/x/sync v0.10.0
 )
 
 require (
@@ -25,7 +26,7 @@ require (
 	github.com/andybalholm/cascadia v1.1.0 // indirect
 	github.com/fatih/color v1.16.0 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
-	github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 // indirect
+	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
 	github.com/knadh/koanf/maps v0.1.1 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect

go.sum

@@ -15,8 +15,8 @@ github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
-github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 h1:TQcrn6Wq+sKGkpyPvppOz99zsMBaUOKXq6HSv655U1c=
-github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss=
+github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww=
@@ -33,14 +33,14 @@ github.com/knadh/koanf/providers/file v1.1.2 h1:aCC36YGOgV5lTtAFz2qkgtWdeQsgfxUk
 github.com/knadh/koanf/providers/file v1.1.2/go.mod h1:/faSBcv2mxPVjFrXck95qeoyoZ5myJ6uxN8OOVNJJCI=
 github.com/knadh/koanf/providers/posflag v0.1.0 h1:mKJlLrKPcAP7Ootf4pBZWJ6J+4wHYujwipe7Ie3qW6U=
 github.com/knadh/koanf/providers/posflag v0.1.0/go.mod h1:SYg03v/t8ISBNrMBRMlojH8OsKowbkXV7giIbBVgbz0=
-github.com/knadh/koanf/v2 v2.1.1 h1:/R8eXqasSTsmDCsAyYj+81Wteg8AqrV9CP6gvsTsOmM=
-github.com/knadh/koanf/v2 v2.1.1/go.mod h1:4mnTRbZCK+ALuBXHZMjDfG9y714L7TykVnZkXbMU3Es=
+github.com/knadh/koanf/v2 v2.1.2 h1:I2rtLRqXRy1p01m/utEtpZSSA6dcJbgGVuE27kW2PzQ=
+github.com/knadh/koanf/v2 v2.1.2/go.mod h1:Gphfaen0q1Fc1HTgJgSTC4oRX9R2R5ErYMZJy8fLJBo=
 github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/lmittmann/tint v1.0.5 h1:NQclAutOfYsqs2F1Lenue6OoWCajs5wJcP3DfWVpePw=
-github.com/lmittmann/tint v1.0.5/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE=
+github.com/lmittmann/tint v1.0.6 h1:vkkuDAZXc0EFGNzYjWcV0h7eEX+uujH48f/ifSkJWgc=
+github.com/lmittmann/tint v1.0.6/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
@@ -67,14 +67,16 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl
 github.com/tlinden/yadu v0.1.3 h1:5cRCUmj+l5yvlM2irtpFBIJwVV2DPEgYSaWvF19FtcY=
 github.com/tlinden/yadu v0.1.3/go.mod h1:l3bRmHKL9zGAR6pnBHY2HRPxBecf7L74BoBgOOpTcUA=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/image v0.23.0 h1:HseQ7c2OpPKTPVzNjG5fwJsOTCiiwS4QdsYi5XU6H68=
+golang.org/x/image v0.23.0/go.mod h1:wJJBTdLfCCf3tiHa1fNxpZmUI4mmoZvwMCPP0ddoNKY=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190606173856-1492cefac77f/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
 golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
-golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
-golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

image.go

@@ -20,11 +20,16 @@ package main
 import (
 	"bytes"
 	"fmt"
-	"image/jpeg"
+	"image"
+	_ "image/gif"
+	_ "image/jpeg"
+	_ "image/png"
 	"log/slog"
 	"os"
 	"path/filepath"
 
+	_ "golang.org/x/image/webp"
+
 	"github.com/corona10/goimagehash"
 )
 
@@ -35,6 +40,7 @@ type Image struct {
 	Hash     *goimagehash.ImageHash
 	Data     *bytes.Reader
 	URI      string
+	Mime     string
 }
 
 // used for logging to avoid printing Data
@@ -49,21 +55,52 @@ func (img *Image) LogValue() slog.Value {
 // holds all images of an ad
 type Cache []*goimagehash.ImageHash
 
-func NewImage(buf *bytes.Reader, filename, uri string) *Image {
+// filename comes from the scraper, it contains directory/base w/o suffix
+func NewImage(buf *bytes.Reader, filename, uri string) (*Image, error) {
+	_, imgconfig, err := image.DecodeConfig(buf)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode image: %w", err)
+	}
+
+	_, err = buf.Seek(0, 0)
+	if err != nil {
+		return nil, fmt.Errorf("failed to seek(0) on image buffer: %w", err)
+	}
+
+	if imgconfig == "jpeg" {
+		// we're  using the  format as file  extension, but  have used
+		// "jpg" in the past, so to be backwards compatible, stay with
+		// it.
+		imgconfig = "jpg"
+	}
+
+	if imgconfig == "" {
+		return nil, fmt.Errorf("failed to process image: unknown or unsupported image format (supported: jpg,png,gif,webp)")
+	}
+
+	filename += "." + imgconfig
+
 	img := &Image{
 		Filename: filename,
 		URI:      uri,
 		Data:     buf,
+		Mime:     imgconfig,
 	}
 
-	return img
+	slog.Debug("image MIME", "mime", img.Mime)
+
+	return img, nil
 }
 
 // Calculate diff hash of the image
 func (img *Image) CalcHash() error {
-	jpgdata, err := jpeg.Decode(img.Data)
+	jpgdata, format, err := image.Decode(img.Data)
 	if err != nil {
-		return fmt.Errorf("failed to decode JPEG image: %w", err)
+		return fmt.Errorf("failed to decode image: %w", err)
+	}
+
+	if format == "" {
+		return fmt.Errorf("failed to decode image: unknown or unsupported image format (supported: jpg,png,gif,webp)")
 	}
 
 	hash1, err := goimagehash.DifferenceHash(jpgdata)
@@ -133,12 +170,19 @@ func ReadImages(addir string, dont bool) (Cache, error) {
 
 			reader := bytes.NewReader(data.Bytes())
 
-			img := NewImage(reader, filename, "")
+			img, err := NewImage(reader, filename, "")
+			if err != nil {
+				return nil, err
+			}
+
 			if err := img.CalcHash(); err != nil {
 				return nil, err
 			}
 
-			slog.Debug("Caching image from file system", "image", img, "hash", img.Hash.ToString())
+			if img.Hash != nil {
+				slog.Debug("Caching image from file system", "image", img, "hash", img.Hash.ToString())
+			}
+
 			cache = append(cache, img.Hash)
 		}
 	}

main_test.go

@@ -182,13 +182,13 @@ var tests = []Tests{
 	{
 		name:     "download-all-ads",
 		args:     base + " -o t/out -u 1",
-		expect:   "Successfully downloaded 6 ads with 12 images to t/out",
+		expect:   "Successfully downloaded 7 ads with 16 images to t/out",
 		exitcode: 0,
 	},
 	{
 		name:     "download-all-ads-using-config",
 		args:     "kleingebaeck -c t/fullconfig.conf",
-		expect:   "Successfully downloaded 6 ads with 12 images to t/out",
+		expect:   "Successfully downloaded 7 ads with 16 images to t/out",
 		exitcode: 0,
 	},
 }
@@ -321,6 +321,17 @@ var adsrc = []AdConfig{
 		Created:   "Yesterday",
 		Images:    []string{"t/1.jpg", "t/2.jpg"},
 	},
+	{
+		Title:     "Ad with multiple img formats",
+		ID:        "7",
+		Price:     "5€",
+		Category:  "Klunker",
+		Text:      "Thing to sale",
+		Slug:      "seventh-ad",
+		Condition: "works",
+		Created:   "Yesterday",
+		Images:    []string{"t/1.png", "t/1.gif", "t/1.webp", "t/1.jpg"},
+	},
 }
 
 // An  Adsource  is used  to  construct  a  httpmock responder  for  a
@@ -363,7 +374,7 @@ func InitValidSources() []Adsource {
 
 	// valid ad listing page 2
 	list2 := []AdConfig{
-		adsrc[3], adsrc[4], adsrc[5],
+		adsrc[3], adsrc[4], adsrc[5], adsrc[6],
 	}
 
 	// valid ad listing page 3, which is empty
@@ -459,7 +470,7 @@ func SetIntercept(ads []Adsource) {
 	}
 
 	// we just use 2 images, put this here
-	for _, image := range []string{"t/1.jpg", "t/2.jpg"} {
+	for _, image := range []string{"t/1.jpg", "t/2.jpg", "t/1.png", "t/1.gif", "t/1.webp"} {
 		httpmock.RegisterResponder("GET", image,
 			httpmock.NewBytesResponder(200, GetImage(image)).HeaderAdd(headers))
 	}

scrape.go

@@ -170,7 +170,9 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error {
 
 	for _, imguri := range advertisement.Images {
 		imguri := imguri
-		file := filepath.Join(adpath, fmt.Sprintf("%d.jpg", img))
+
+		// we append the suffix later in NewImage() based on image format
+		basefilename := filepath.Join(adpath, fmt.Sprintf("%d", img))
 
 		egroup.Go(func() error {
 			// wait a little
@@ -192,7 +194,11 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error {
 
 			reader := bytes.NewReader(buf.Bytes())
 
-			image := NewImage(reader, file, imguri)
+			image, err := NewImage(reader, basefilename, imguri)
+			if err != nil {
+				return err
+			}
+
 			err = image.CalcHash()
 			if err != nil {
 				return err
@@ -211,7 +217,7 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error {
 				return fmt.Errorf("failed to seek(0) on image reader: %w", err)
 			}
 
-			err = WriteImage(file, reader)
+			err = WriteImage(image.Filename, reader)
 			if err != nil {
 				return err
 			}

store.go

@@ -149,7 +149,9 @@ func ReadImage(filename string) (*bytes.Buffer, error) {
 
 func fileExists(filename string) bool {
 	info, err := os.Stat(filename)
-	if os.IsNotExist(err) {
+
+	if err != nil {
+		// return false on any error
 		return false
 	}