Compare commits

..

13 Commits

Author SHA1 Message Date
16b47aa7b0 fix docker 2026-10-12 21:27:27 +02:00
562292de2b dump version 2026-10-12 21:27:22 +02:00
7f2f61ca9e fix tests 2026-10-12 21:23:28 +02:00
4e77573223 satisfy linter 2026-10-12 14:30:28 +02:00
79892bc55b upd linter 2026-10-12 14:16:04 +02:00
75087b47bf fix #125: could not extract data from subsequent pages 2026-10-12 14:08:02 +02:00
3c9124a3e6 reorder for better readability 2026-10-12 14:07:51 +02:00
d20a499519 update go 2026-10-12 14:07:35 +02:00
0d5ee0b485 pin to older release, see woodpecker-plugins/docker-buildx#306 2026-01-07 10:39:38 +01:00
T. von Dein
b44c1ba1dc update x/image (#124) 2025-11-05 10:13:52 +01:00
b6350344be fix image links 2025-11-05 08:56:20 +01:00
e5ff2bbc5c fix changelog link 2025-11-05 08:45:12 +01:00
T. von Dein
230bbf3d53 migrate to codeberg (#123) 2025-11-05 08:35:00 +01:00
19 changed files with 275 additions and 95 deletions

106
.golangci.yml Normal file
View File

@@ -0,0 +1,106 @@
version: "2"
linters:
enable:
- errcheck
- govet
- ineffassign
- staticcheck
- unused
- arangolint
- asasalint
- asciicheck
- bidichk
- bodyclose
- canonicalheader
- clickhouselint
- containedctx
#- contextcheck
- copyloopvar
#- cyclop
- decorder
- dogsled
- dupword
- durationcheck
- embeddedstructfieldcheck
- errchkjson
- errname
#- errorlint
- exhaustive
- exptostd
- fatcontext
- funcorder
#- funlen
- ginkgolinter
- gocheckcompilerdirectives
#- gochecknoinits
- gochecksumtype
#- gocognit
- gocritic
#- gocyclo
- goheader
- gomoddirectives
- gomodguard_v2
- goprintffuncname
#- gosec
- gosmopolitan
- grouper
- iface
- importas
- inamedparam
- interfacebloat
- intrange
- iotamixing
- lll
- loggercheck
- makezero
- misspell
- modernize
- nakedret
#- nestif
- nilerr
- nilnesserr
- nlreturn
- nonamedreturns
- nosprintfhostport
- perfsprint
- prealloc
- promlinter
- protogetter
- reassign
- recvcheck
- rowserrcheck
- sloglint
- spancheck
- sqlclosecheck
- tagalign
- testableexamples
- testifylint
- testpackage
- thelper
- tparallel
- unconvert
- unparam
- unqueryvet
- usestdlibvars
- usetesting
- varnamelen
- wastedassign
- whitespace
- wsl_v5
- zerologlint
settings:
varnamelen:
ignore-names:
- err
- wg
- mu
- ts
- to
- es
- op
- fd
- id
- fn

View File

@@ -66,4 +66,4 @@ release:
--- ---
Full Changelog: [{{ .PreviousTag }}...{{ .Tag }}](https://codeberg.org/scip/epuppy/compare/{{ .PreviousTag }}...{{ .Tag }}) Full Changelog: [{{ .PreviousTag }}...{{ .Tag }}](https://codeberg.org/scip/kleingebaeck/compare/{{ .PreviousTag }}...{{ .Tag }})

View File

@@ -2,7 +2,9 @@ matrix:
platform: platform:
- linux/amd64 - linux/amd64
goversion: goversion:
- 1.24 - 1.26
lintversion:
- v2.12.2
labels: labels:
platform: ${platform} platform: ${platform}
@@ -21,7 +23,7 @@ steps:
event: [push] event: [push]
image: golang:${goversion} image: golang:${goversion}
commands: commands:
- curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.5.0 - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin ${lintversion}
- golangci-lint --version - golangci-lint --version
- golangci-lint run ./... - golangci-lint run ./...
depends_on: [build] depends_on: [build]

View File

@@ -6,7 +6,7 @@ variables:
steps: steps:
dryrun: dryrun:
image: docker.io/woodpeckerci/plugin-docker-buildx:latest image: docker.io/woodpeckerci/plugin-docker-buildx:6.0.3
settings: settings:
dockerfile: Dockerfile dockerfile: Dockerfile
platforms: linux/amd64 platforms: linux/amd64
@@ -17,7 +17,7 @@ steps:
event: [pull_request] event: [pull_request]
publish: publish:
image: docker.io/woodpeckerci/plugin-docker-buildx:latest image: docker.io/woodpeckerci/plugin-docker-buildx:6.0.3
settings: settings:
dockerfile: Dockerfile dockerfile: Dockerfile
platforms: linux/amd64 platforms: linux/amd64

View File

@@ -1,4 +1,4 @@
FROM golang:1.24-alpine as builder FROM golang:1.26-alpine as builder
RUN apk update RUN apk update
RUN apk upgrade RUN apk upgrade

View File

@@ -1,13 +1,12 @@
## Kleingebäck - kleinanzeigen.de Backup ## Kleingebäck - kleinanzeigen.de Backup
![Kleingebaeck Logo](https://codeberg.org/scip/kleingebaeck/raw/branch/.github/assets/kleingebaecklogo-small.png) ![Kleingebaeck Logo](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/kleingebaecklogo-small.png)
[![Go Report Card](https://goreportcard.com/badge/codeberg.org/scip/kleingebaeck)](https://goreportcard.com/report/codeberg.org/scip/kleingebaeck) [![Go Report Card](https://goreportcard.com/badge/codeberg.org/scip/kleingebaeck)](https://goreportcard.com/report/codeberg.org/scip/kleingebaeck)
[![status-badge](https://ci.codeberg.org/api/badges/15530/status.svg)](https://ci.codeberg.org/repos/15530) [![status-badge](https://ci.codeberg.org/api/badges/15530/status.svg)](https://ci.codeberg.org/repos/15530)
[![Go Coverage](https://codeberg.org/scip/kleingebaeck/wiki/coverage.svg)](https://raw.githack.com/wiki/tlinden/kleingebaeck/coverage.html)
![GitHub License](https://img.shields.io/github/license/tlinden/kleingebaeck) ![GitHub License](https://img.shields.io/github/license/tlinden/kleingebaeck)
[![GitHub release](https://img.shields.io/github/v/release/tlinden/kleingebaeck?color=%2300a719)](https://codeberg.org/scip/kleingebaeck/releases) [![GitHub release](https://img.shields.io/github/v/release/tlinden/kleingebaeck?color=%2300a719)](https://codeberg.org/scip/kleingebaeck/releases)
[![English](https://codeberg.org/scip/kleingebaeck/raw/branch/.github/assets/english.png)](https://codeberg.org/scip/kleingebaeck/raw/branch/README.md) [![English](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/english.png)](https://codeberg.org/scip/kleingebaeck/raw/branch/main/README.md)
Mit diesem Tool kann man seine Anzeigen bei https://kleinanzeigen.de sichern. Mit diesem Tool kann man seine Anzeigen bei https://kleinanzeigen.de sichern.
@@ -30,33 +29,33 @@ für mehr Details.
Das ist die Hauptseite meines kleinanzeigen.de Accounts: Das ist die Hauptseite meines kleinanzeigen.de Accounts:
![Index](https://codeberg.org/scip/kleingebaeck/raw/branch/.github/assets/kleinanzeigen-index.png) ![Index](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/kleinanzeigen-index.png)
Sichern ich meine Anzeigen: Sichern ich meine Anzeigen:
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/.github/assets/kleinanzeigen-download.png) ![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/kleinanzeigen-download.png)
Backupverzeichnis nach dem Download: Backupverzeichnis nach dem Download:
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/.github/assets/kleinanzeigen-backup.png) ![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/kleinanzeigen-backup.png)
Verzeichnis einer Anzeige: Verzeichnis einer Anzeige:
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/.github/assets/kleinanzeigen-ad.png) ![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/kleinanzeigen-ad.png)
**Das gleiche unter Windows:** **Das gleiche unter Windows:**
Anzeigen Sichern: Anzeigen Sichern:
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/.github/assets/cmd-windows.jpg) ![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/cmd-windows.jpg)
Backupverzeichnis nach dem Download Backupverzeichnis nach dem Download
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/.github/assets/liste-windows.jpg) ![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/liste-windows.jpg)
Und eine Anzeige: Und eine Anzeige:
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/.github/assets/adlisting-windows.jpg) ![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/adlisting-windows.jpg)
## Installation ## Installation
@@ -277,7 +276,7 @@ ignorieren, die beim Bilderdownload auftreten könnten.
## Documentation ## Documentation
Die Dokumentation kann man Die Dokumentation kann man
[online](https://codeberg.org/scip/kleingebaeck/raw/branch/kleingebaeck.pod) [online](https://codeberg.org/scip/kleingebaeck/raw/branch/main/kleingebaeck.pod)
oder lokal lesen mit: `kleingebaeck --manual`. Hat man das Tool mit oder lokal lesen mit: `kleingebaeck --manual`. Hat man das Tool mit
dem Tarball installiert, funktioniert auch `man kleingebaeck`. dem Tarball installiert, funktioniert auch `man kleingebaeck`.

View File

@@ -1,13 +1,12 @@
## Kleingebäck - kleinanzeigen.de Backup ## Kleingebäck - kleinanzeigen.de Backup
![Kleingebaeck Logo](https://codeberg.org/scip/kleingebaeck/raw/branch/.github/assets/kleingebaecklogo-small.png) ![Kleingebaeck Logo](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/kleingebaecklogo-small.png)
[![Go Report Card](https://goreportcard.com/badge/codeberg.org/scip/kleingebaeck)](https://goreportcard.com/report/codeberg.org/scip/kleingebaeck) [![Go Report Card](https://goreportcard.com/badge/codeberg.org/scip/kleingebaeck)](https://goreportcard.com/report/codeberg.org/scip/kleingebaeck)
[![status-badge](https://ci.codeberg.org/api/badges/15530/status.svg)](https://ci.codeberg.org/repos/15530) [![status-badge](https://ci.codeberg.org/api/badges/15530/status.svg)](https://ci.codeberg.org/repos/15530)
[![Go Coverage](https://codeberg.org/scip/kleingebaeck/wiki/coverage.svg)](https://raw.githack.com/wiki/tlinden/kleingebaeck/coverage.html)
![GitHub License](https://img.shields.io/github/license/tlinden/kleingebaeck) ![GitHub License](https://img.shields.io/github/license/tlinden/kleingebaeck)
[![GitHub release](https://img.shields.io/github/v/release/tlinden/kleingebaeck?color=%2300a719)](https://codeberg.org/scip/kleingebaeck/releases) [![GitHub release](https://img.shields.io/github/v/release/tlinden/kleingebaeck?color=%2300a719)](https://codeberg.org/scip/kleingebaeck/releases)
[![German](https://codeberg.org/scip/kleingebaeck/raw/branch/.github/assets/german.png)](https://codeberg.org/scip/kleingebaeck/raw/branch/README-de.md) [![German](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/german.png)](https://codeberg.org/scip/kleingebaeck/raw/branch/main/README-de.md)
[Die deutsche Version des READMEs findet Ihr hier](README-de.md). [Die deutsche Version des READMEs findet Ihr hier](README-de.md).
@@ -31,33 +30,33 @@ for more details.
This is the index of my kleinanzeigen.de Account: This is the index of my kleinanzeigen.de Account:
![Index](https://codeberg.org/scip/kleingebaeck/raw/branch/.github/assets/kleinanzeigen-index.png) ![Index](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/kleinanzeigen-index.png)
Here I download my ads on the commandline: Here I download my ads on the commandline:
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/.github/assets/kleinanzeigen-download.png) ![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/kleinanzeigen-download.png)
And this is the backup directory after download: And this is the backup directory after download:
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/.github/assets/kleinanzeigen-backup.png) ![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/kleinanzeigen-backup.png)
Here's a directory for one ad: Here's a directory for one ad:
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/.github/assets/kleinanzeigen-ad.png) ![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/kleinanzeigen-ad.png)
**The same thing under windows:** **The same thing under windows:**
Downloading ads: Downloading ads:
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/.github/assets/cmd-windows.jpg) ![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/cmd-windows.jpg)
Backup directory after download: Backup directory after download:
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/.github/assets/liste-windows.jpg) ![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/liste-windows.jpg)
And one ad listing directory: And one ad listing directory:
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/.github/assets/adlisting-windows.jpg) ![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/adlisting-windows.jpg)
## Installation ## Installation
@@ -262,7 +261,7 @@ errors.
## Documentation ## Documentation
You can read the documentation [online](https://codeberg.org/scip/kleingebaeck/raw/branch/kleingebaeck.pod) or locally once you have installed kleingebaeck with: `kleingebaeck --manual`. You can read the documentation [online](https://codeberg.org/scip/kleingebaeck/raw/branch/main/kleingebaeck.pod) or locally once you have installed kleingebaeck with: `kleingebaeck --manual`.
## Kleingebäck? ## Kleingebäck?

20
ad.go
View File

@@ -29,16 +29,8 @@ type Index struct {
} }
type Ad struct { type Ad struct {
Title string `goquery:"h1"` Title string `goquery:"h1"`
Slug string Details string `goquery:".addetailslist--detail,text"`
ID string
Details string `goquery:".addetailslist--detail,text"`
Attributes map[string]string // processed afterwards
Condition string // post processed from details for backward compatibility
Type string // post processed from details for backward compatibility
Color string // post processed from details for backward compatibility
Material string // post processed from details for backward compatibility
Category string
CategoryTree []string `goquery:".breadcrump-link,text"` CategoryTree []string `goquery:".breadcrump-link,text"`
Price string `goquery:"h2#viewad-price"` Price string `goquery:"h2#viewad-price"`
Created string `goquery:"#viewad-extra-info,text"` Created string `goquery:"#viewad-extra-info,text"`
@@ -46,6 +38,14 @@ type Ad struct {
Images []string `goquery:".galleryimage-element img,[src]"` Images []string `goquery:".galleryimage-element img,[src]"`
Shipping string `goquery:".boxedarticle--details--shipping,text"` // not always filled Shipping string `goquery:".boxedarticle--details--shipping,text"` // not always filled
Expire string Expire string
Slug string
ID string
Attributes map[string]string // processed afterwards
Condition string // post processed from details for backward compatibility
Type string // post processed from details for backward compatibility
Color string // post processed from details for backward compatibility
Material string // post processed from details for backward compatibility
Category string
// runtime computed // runtime computed
Year, Day, Month string Year, Day, Month string

View File

@@ -34,7 +34,7 @@ import (
) )
const ( const (
VERSION string = "0.3.23" VERSION string = "0.3.25"
Baseuri string = "https://www.kleinanzeigen.de" Baseuri string = "https://www.kleinanzeigen.de"
Listuri string = "/s-bestandsliste.html" Listuri string = "/s-bestandsliste.html"
Defaultdir string = "." Defaultdir string = "."
@@ -140,7 +140,7 @@ func InitConfig(output io.Writer) (*Config, error) {
} }
// Load default values using the confmap provider. // Load default values using the confmap provider.
if err := kloader.Load(confmap.Provider(map[string]interface{}{ if err := kloader.Load(confmap.Provider(map[string]any{
"template": template, "template": template,
"outdir": DefaultOutdirTemplate, "outdir": DefaultOutdirTemplate,
"loglevel": "notice", "loglevel": "notice",
@@ -200,7 +200,6 @@ func InitConfig(output io.Writer) (*Config, error) {
// Load the config file[s] // Load the config file[s]
for _, cfgfile := range configfiles { for _, cfgfile := range configfiles {
path, err := os.Stat(cfgfile) path, err := os.Stat(cfgfile)
if err != nil { if err != nil {
// ignore non-existent files, but bail out on any other errors // ignore non-existent files, but bail out on any other errors
if !os.IsNotExist(err) { if !os.IsNotExist(err) {

View File

@@ -24,7 +24,11 @@ import (
"log/slog" "log/slog"
"net/http" "net/http"
"net/http/cookiejar" "net/http/cookiejar"
"net/http/httputil"
"net/url" "net/url"
"os"
"strings"
"sync"
) )
// convenient wrapper to fetch some web content // convenient wrapper to fetch some web content
@@ -34,12 +38,25 @@ type Fetcher struct {
Cookies []*http.Cookie Cookies []*http.Cookie
} }
// Pos is a counter for debug output file names
type Pos struct {
sync.Mutex
pos int
}
var pos *Pos
func NewFetcher(conf *Config) (*Fetcher, error) { func NewFetcher(conf *Config) (*Fetcher, error) {
jar, err := cookiejar.New(nil) jar, err := cookiejar.New(nil)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to create a cookie jar obj: %w", err) return nil, fmt.Errorf("failed to create a cookie jar obj: %w", err)
} }
if pos == nil {
pos = &Pos{}
}
return &Fetcher{ return &Fetcher{
Client: &http.Client{ Client: &http.Client{
Transport: &loggingTransport{}, // implemented in http.go Transport: &loggingTransport{}, // implemented in http.go
@@ -82,9 +99,43 @@ func (f *Fetcher) Get(uri string) (io.ReadCloser, error) {
slog.Debug("got cookies?", "cookies", res.Cookies()) slog.Debug("got cookies?", "cookies", res.Cookies())
f.Cookies = res.Cookies() f.Cookies = res.Cookies()
if err := f.Dump(res); err != nil {
return nil, err
}
return res.Body, nil return res.Body, nil
} }
func (f *Fetcher) Dump(res *http.Response) error {
if f.Config.Debug && strings.Contains(res.Header.Get("Content-Type"), "text/html") {
// => dump content
pos.Lock()
defer pos.Unlock()
pos.pos++
dump, err := httputil.DumpResponse(res, true)
if err != nil {
return err
}
file := fmt.Sprintf("out%d.html", pos.pos)
fd, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
return err
}
if _, err := fd.Write(dump); err != nil {
return err
}
slog.Debug("dump content", "file", file)
}
return nil
}
// fetch an image // fetch an image
func (f *Fetcher) Getimage(uri string) (io.ReadCloser, error) { func (f *Fetcher) Getimage(uri string) (io.ReadCloser, error) {
slog.Debug("fetching ad image", "uri", uri) slog.Debug("fetching ad image", "uri", uri)

6
go.mod
View File

@@ -1,8 +1,6 @@
module kleingebaeck module kleingebaeck
go 1.24.0 go 1.26.0
toolchain go1.24.5
require ( require (
astuart.co/goq v1.0.0 astuart.co/goq v1.0.0
@@ -19,7 +17,7 @@ require (
github.com/mattn/go-isatty v0.0.20 github.com/mattn/go-isatty v0.0.20
github.com/spf13/pflag v1.0.10 github.com/spf13/pflag v1.0.10
github.com/tlinden/yadu v0.1.3 github.com/tlinden/yadu v0.1.3
golang.org/x/image v0.31.0 golang.org/x/image v0.32.0
golang.org/x/sync v0.17.0 golang.org/x/sync v0.17.0
) )

2
go.sum
View File

@@ -68,6 +68,8 @@ github.com/tlinden/yadu v0.1.3/go.mod h1:l3bRmHKL9zGAR6pnBHY2HRPxBecf7L74BoBgOOp
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/image v0.31.0 h1:mLChjE2MV6g1S7oqbXC0/UcKijjm5fnJLUYKIYrLESA= golang.org/x/image v0.31.0 h1:mLChjE2MV6g1S7oqbXC0/UcKijjm5fnJLUYKIYrLESA=
golang.org/x/image v0.31.0/go.mod h1:R9ec5Lcp96v9FTF+ajwaH3uGxPH4fKfHHAVbUILxghA= golang.org/x/image v0.31.0/go.mod h1:R9ec5Lcp96v9FTF+ajwaH3uGxPH4fKfHHAVbUILxghA=
golang.org/x/image v0.32.0 h1:6lZQWq75h7L5IWNk0r+SCpUJ6tUVd3v4ZHnbRKLkUDQ=
golang.org/x/image v0.32.0/go.mod h1:/R37rrQmKXtO6tYXAjtDLwQgFLHmhW+V6ayXlxzP2Pc=
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190606173856-1492cefac77f/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190606173856-1492cefac77f/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=

View File

@@ -123,7 +123,6 @@ func (t *loggingTransport) RoundTrip(req *http.Request) (*http.Response, error)
// actual retry // actual retry
resp, err = http.DefaultTransport.RoundTrip(req) resp, err = http.DefaultTransport.RoundTrip(req)
if err == nil { if err == nil {
slog.Debug("RESPONSE", "id", requestid, "status", resp.StatusCode, slog.Debug("RESPONSE", "id", requestid, "status", resp.StatusCode,
"contentlength", resp.ContentLength, "retry", retries) "contentlength", resp.ContentLength, "retry", retries)

View File

@@ -19,6 +19,7 @@ package main
import ( import (
"bytes" "bytes"
"errors"
"fmt" "fmt"
"image" "image"
_ "image/gif" _ "image/gif"
@@ -27,6 +28,7 @@ import (
"log/slog" "log/slog"
"os" "os"
"path/filepath" "path/filepath"
"slices"
_ "golang.org/x/image/webp" _ "golang.org/x/image/webp"
@@ -43,18 +45,6 @@ type Image struct {
Mime string Mime string
} }
// used for logging to avoid printing Data
func (img *Image) LogValue() slog.Value {
return slog.GroupValue(
slog.String("filename", img.Filename),
slog.String("uri", img.URI),
slog.String("hash", img.Hash.ToString()),
)
}
// holds all images of an ad
type Cache []*goimagehash.ImageHash
// filename comes from the scraper, it contains directory/base w/o suffix // filename comes from the scraper, it contains directory/base w/o suffix
func NewImage(buf *bytes.Reader, filename, uri string) (*Image, error) { func NewImage(buf *bytes.Reader, filename, uri string) (*Image, error) {
_, imgconfig, err := image.DecodeConfig(buf) _, imgconfig, err := image.DecodeConfig(buf)
@@ -75,7 +65,7 @@ func NewImage(buf *bytes.Reader, filename, uri string) (*Image, error) {
} }
if imgconfig == "" { if imgconfig == "" {
return nil, fmt.Errorf("failed to process image: unknown or unsupported image format (supported: jpg,png,gif,webp)") return nil, errors.New("failed to process image: unknown or unsupported image format (supported: jpg,png,gif,webp)")
} }
filename += "." + imgconfig filename += "." + imgconfig
@@ -92,6 +82,18 @@ func NewImage(buf *bytes.Reader, filename, uri string) (*Image, error) {
return img, nil return img, nil
} }
// used for logging to avoid printing Data
func (img *Image) LogValue() slog.Value {
return slog.GroupValue(
slog.String("filename", img.Filename),
slog.String("uri", img.URI),
slog.String("hash", img.Hash.ToString()),
)
}
// holds all images of an ad
type Cache []*goimagehash.ImageHash
// Calculate diff hash of the image // Calculate diff hash of the image
func (img *Image) CalcHash() error { func (img *Image) CalcHash() error {
jpgdata, format, err := image.Decode(img.Data) jpgdata, format, err := image.Decode(img.Data)
@@ -100,7 +102,7 @@ func (img *Image) CalcHash() error {
} }
if format == "" { if format == "" {
return fmt.Errorf("failed to decode image: unknown or unsupported image format (supported: jpg,png,gif,webp)") return errors.New("failed to decode image: unknown or unsupported image format (supported: jpg,png,gif,webp)")
} }
hash1, err := goimagehash.DifferenceHash(jpgdata) hash1, err := goimagehash.DifferenceHash(jpgdata)
@@ -134,13 +136,9 @@ func (img *Image) Similar(hash *goimagehash.ImageHash) bool {
// check current image against all known hashes. // check current image against all known hashes.
func (img *Image) SimilarExists(cache Cache) bool { func (img *Image) SimilarExists(cache Cache) bool {
for _, otherimg := range cache { return slices.ContainsFunc(cache, func(otherimg *goimagehash.ImageHash) bool {
if img.Similar(otherimg) { return img.Similar(otherimg)
return true })
}
}
return false
} }
// read all JPG images in a ad directory, compute diff hashes and // read all JPG images in a ad directory, compute diff hashes and

12
main.go
View File

@@ -49,6 +49,7 @@ func init() {
fmt.Println("Please open a command shell and run it from there.") fmt.Println("Please open a command shell and run it from there.")
fmt.Println() fmt.Println()
fmt.Print("Press any key to quit: ") fmt.Print("Press any key to quit: ")
_, err := bufio.NewReader(os.Stdin).ReadString('\n') _, err := bufio.NewReader(os.Stdin).ReadString('\n')
if err != nil { if err != nil {
panic(err) panic(err)
@@ -144,13 +145,8 @@ func Main(output io.Writer) int {
if err != nil { if err != nil {
return Die(err) return Die(err)
} }
conf.Outdir = outdir
// used for all HTTP requests conf.Outdir = outdir
fetch, err := NewFetcher(conf)
if err != nil {
return Die(err)
}
// setup ad dir registry, needed to check for duplicates // setup ad dir registry, needed to check for duplicates
DirsVisited = make(map[string]int) DirsVisited = make(map[string]int)
@@ -159,14 +155,14 @@ func Main(output io.Writer) int {
case len(conf.Adlinks) >= 1: case len(conf.Adlinks) >= 1:
// directly backup ad listing[s] // directly backup ad listing[s]
for _, uri := range conf.Adlinks { for _, uri := range conf.Adlinks {
err := ScrapeAd(fetch, uri) err = ScrapeAd(conf, uri)
if err != nil { if err != nil {
return Die(err) return Die(err)
} }
} }
case conf.User > 0: case conf.User > 0:
// backup all ads of the given user (via config or cmdline) // backup all ads of the given user (via config or cmdline)
err := ScrapeUser(fetch) err := ScrapeUser(conf)
if err != nil { if err != nil {
return Die(err) return Die(err)
} }

View File

@@ -405,6 +405,7 @@ func InitValidSources() []Adsource {
empty := AdConfig{} empty := AdConfig{}
// prepare urls for the listing pages // prepare urls for the listing pages
// nolint:prealloc
ads := []Adsource{ ads := []Adsource{
{ {
uri: fmt.Sprintf("%s%s?userId=1", Baseuri, Listuri), uri: fmt.Sprintf("%s%s?userId=1", Baseuri, Listuri),
@@ -436,7 +437,7 @@ func InitInvalidSources() []Adsource {
ads := []Adsource{ ads := []Adsource{
{ {
// valid ad page but without content // valid ad page but without content
uri: fmt.Sprintf("%s/s-anzeige/empty/1", Baseuri), uri: Baseuri + "/s-anzeige/empty/1",
content: GetTemplate(nil, &empty, EMPTYPAGE), content: GetTemplate(nil, &empty, EMPTYPAGE),
}, },
{ {
@@ -446,18 +447,18 @@ func InitInvalidSources() []Adsource {
}, },
{ {
// some invalid page path // some invalid page path
uri: fmt.Sprintf("%s/anzeige/name/1", Baseuri), uri: Baseuri + "/anzeige/name/1",
content: GetTemplate(nil, &empty, "<html></html>"), content: GetTemplate(nil, &empty, "<html></html>"),
}, },
{ {
// some none-ad page // some none-ad page
uri: fmt.Sprintf("%s/anzeige/name/1/foo/bar", Baseuri), uri: Baseuri + "/anzeige/name/1/foo/bar",
content: GetTemplate(nil, &empty, "<html>HTTP 404: /eine-anzeige/ does not exist!</html>"), content: GetTemplate(nil, &empty, "<html>HTTP 404: /eine-anzeige/ does not exist!</html>"),
status: 404, status: 404,
}, },
{ {
// valid ad page but 503 // valid ad page but 503
uri: fmt.Sprintf("%s/s-anzeige/503/1", Baseuri), uri: Baseuri + "/s-anzeige/503/1",
content: GetTemplate(nil, &empty, "<html>HTTP 503: service unavailable</html>"), content: GetTemplate(nil, &empty, "<html>HTTP 503: service unavailable</html>"),
status: 503, status: 503,
}, },
@@ -531,6 +532,7 @@ func TestMain(t *testing.T) {
defer func() { os.Args = oldargs }() defer func() { os.Args = oldargs }()
httpmock.Activate() httpmock.Activate()
defer httpmock.DeactivateAndReset() defer httpmock.DeactivateAndReset()
// prepare httpmock responders // prepare httpmock responders
@@ -568,6 +570,7 @@ func TestMainInvalids(t *testing.T) {
defer func() { os.Args = oldargs }() defer func() { os.Args = oldargs }()
httpmock.Activate() httpmock.Activate()
defer httpmock.DeactivateAndReset() defer httpmock.DeactivateAndReset()
// prepare httpmock responders // prepare httpmock responders

View File

@@ -19,7 +19,9 @@ package main
import ( import (
"bytes" "bytes"
"errors"
"fmt" "fmt"
"io"
"log/slog" "log/slog"
"path/filepath" "path/filepath"
"strconv" "strconv"
@@ -32,30 +34,44 @@ import (
// extract links from all ad listing pages (that is: use pagination) // extract links from all ad listing pages (that is: use pagination)
// and scrape every page // and scrape every page
func ScrapeUser(fetch *Fetcher) error { func ScrapeUser(conf *Config) error {
adlinks := []string{} adlinks := []string{}
baseuri := fmt.Sprintf("%s%s?userId=%d", Baseuri, Listuri, fetch.Config.User) baseuri := fmt.Sprintf("%s%s?userId=%d", Baseuri, Listuri, conf.User)
page := 1 page := 1
uri := baseuri uri := baseuri
slog.Info("fetching ad pages", "user", fetch.Config.User) slog.Info("fetching ad pages", "user", conf.User)
for { for {
var index Index var index Index
slog.Debug("fetching page", "uri", uri) slog.Debug("fetching page", "uri", uri)
body, err := fetch.Get(uri) fetch, err := NewFetcher(conf)
if err != nil { if err != nil {
return err return err
} }
body, err := fetch.Get(uri)
defer func() { defer func() {
if body == nil {
return
}
// make sure body is completely flushed
if _, err := io.Copy(io.Discard, body); err != nil {
panic(err)
}
if err := body.Close(); err != nil { if err := body.Close(); err != nil {
panic(err) panic(err)
} }
}() }()
if err != nil {
return err
}
err = goq.NewDecoder(body).Decode(&index) err = goq.NewDecoder(body).Decode(&index)
if err != nil { if err != nil {
return fmt.Errorf("failed to goquery decode HTML index body: %w", err) return fmt.Errorf("failed to goquery decode HTML index body: %w", err)
@@ -77,12 +93,12 @@ func ScrapeUser(fetch *Fetcher) error {
} }
for index, adlink := range adlinks { for index, adlink := range adlinks {
err := ScrapeAd(fetch, Baseuri+adlink) err := ScrapeAd(conf, Baseuri+adlink)
if err != nil { if err != nil {
return err return err
} }
if fetch.Config.Limit > 0 && index == fetch.Config.Limit-1 { if conf.Limit > 0 && index == conf.Limit-1 {
break break
} }
} }
@@ -91,7 +107,7 @@ func ScrapeUser(fetch *Fetcher) error {
} }
// scrape an ad. uri is the full uri of the ad, dir is the basedir // scrape an ad. uri is the full uri of the ad, dir is the basedir
func ScrapeAd(fetch *Fetcher, uri string) error { func ScrapeAd(conf *Config, uri string) error {
now := time.Now() now := time.Now()
advertisement := &Ad{ advertisement := &Ad{
@@ -112,16 +128,31 @@ func ScrapeAd(fetch *Fetcher, uri string) error {
// get the ad // get the ad
slog.Debug("fetching ad page", "uri", uri) slog.Debug("fetching ad page", "uri", uri)
body, err := fetch.Get(uri) fetch, err := NewFetcher(conf)
if err != nil { if err != nil {
return err return err
} }
body, err := fetch.Get(uri)
defer func() { defer func() {
if body == nil {
return
}
// make sure body is completely flushed
if _, err := io.Copy(io.Discard, body); err != nil {
panic(err)
}
if err := body.Close(); err != nil { if err := body.Close(); err != nil {
panic(err) panic(err)
} }
}() }()
if err != nil {
return err
}
// extract ad contents with goquery/goq // extract ad contents with goquery/goq
err = goq.NewDecoder(body).Decode(&advertisement) err = goq.NewDecoder(body).Decode(&advertisement)
if err != nil { if err != nil {
@@ -135,7 +166,7 @@ func ScrapeAd(fetch *Fetcher, uri string) error {
if advertisement.Incomplete() { if advertisement.Incomplete() {
slog.Debug("got ad", "ad", advertisement) slog.Debug("got ad", "ad", advertisement)
return fmt.Errorf("could not extract ad data from page, got empty struct") return errors.New("could not extract ad data from page, got empty struct")
} }
advertisement.DecodeAttributes() advertisement.DecodeAttributes()
@@ -184,14 +215,11 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error {
egroup := new(errgroup.Group) egroup := new(errgroup.Group)
for _, imguri := range advertisement.Images { for _, imguri := range advertisement.Images {
imguri := imguri
// we append the suffix later in NewImage() based on image format // we append the suffix later in NewImage() based on image format
basefilename := filepath.Join(adpath, fmt.Sprintf("%d", img)) basefilename := filepath.Join(adpath, strconv.Itoa(img))
egroup.Go(func() error { egroup.Go(func() error {
// wait a little // wait a little
throttle := GetThrottleTime() throttle := GetThrottleTime()
time.Sleep(throttle) time.Sleep(throttle)
@@ -241,6 +269,7 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error {
return nil return nil
}) })
img++ img++
} }

View File

@@ -127,7 +127,6 @@ func WriteImage(filename string, reader *bytes.Reader) error {
}() }()
_, err = reader.WriteTo(file) _, err = reader.WriteTo(file)
if err != nil { if err != nil {
return fmt.Errorf("failed to write to image file: %w", err) return fmt.Errorf("failed to write to image file: %w", err)
} }
@@ -157,7 +156,6 @@ func ReadImage(filename string) (*bytes.Buffer, error) {
func fileExists(filename string) bool { func fileExists(filename string) bool {
info, err := os.Stat(filename) info, err := os.Stat(filename)
if err != nil { if err != nil {
// return false on any error // return false on any error
return false return false
@@ -172,11 +170,13 @@ func CheckAdVisited(conf *Config, adname string) bool {
if Exists(DirsVisited, adname) { if Exists(DirsVisited, adname) {
if conf.ForceDownload { if conf.ForceDownload {
slog.Warn("an ad with the same name has already been downloaded, overwriting", "addir", adname) slog.Warn("an ad with the same name has already been downloaded, overwriting", "addir", adname)
return true return true
} }
// don't overwrite // don't overwrite
slog.Warn("an ad with the same name has already been downloaded, skipping (use -f to overwrite)", "addir", adname) slog.Warn("an ad with the same name has already been downloaded, skipping (use -f to overwrite)", "addir", adname)
return false return false
} }

View File

@@ -53,7 +53,6 @@ func man() error {
man.Stderr = os.Stderr man.Stderr = os.Stderr
err := man.Run() err := man.Run()
if err != nil { if err != nil {
return fmt.Errorf("failed to execute 'less': %w", err) return fmt.Errorf("failed to execute 'less': %w", err)
} }