Compare commits

...

6 Commits

Author SHA1 Message Date
7f2f61ca9e fix tests 2026-10-12 21:23:28 +02:00
4e77573223 satisfy linter 2026-10-12 14:30:28 +02:00
79892bc55b upd linter 2026-10-12 14:16:04 +02:00
75087b47bf fix #125: could not extract data from subsequent pages 2026-10-12 14:08:02 +02:00
3c9124a3e6 reorder for better readability 2026-10-12 14:07:51 +02:00
d20a499519 update go 2026-10-12 14:07:35 +02:00
14 changed files with 249 additions and 67 deletions

106
.golangci.yml Normal file
View File

@@ -0,0 +1,106 @@
version: "2"
linters:
enable:
- errcheck
- govet
- ineffassign
- staticcheck
- unused
- arangolint
- asasalint
- asciicheck
- bidichk
- bodyclose
- canonicalheader
- clickhouselint
- containedctx
#- contextcheck
- copyloopvar
#- cyclop
- decorder
- dogsled
- dupword
- durationcheck
- embeddedstructfieldcheck
- errchkjson
- errname
#- errorlint
- exhaustive
- exptostd
- fatcontext
- funcorder
#- funlen
- ginkgolinter
- gocheckcompilerdirectives
#- gochecknoinits
- gochecksumtype
#- gocognit
- gocritic
#- gocyclo
- goheader
- gomoddirectives
- gomodguard_v2
- goprintffuncname
#- gosec
- gosmopolitan
- grouper
- iface
- importas
- inamedparam
- interfacebloat
- intrange
- iotamixing
- lll
- loggercheck
- makezero
- misspell
- modernize
- nakedret
#- nestif
- nilerr
- nilnesserr
- nlreturn
- nonamedreturns
- nosprintfhostport
- perfsprint
- prealloc
- promlinter
- protogetter
- reassign
- recvcheck
- rowserrcheck
- sloglint
- spancheck
- sqlclosecheck
- tagalign
- testableexamples
- testifylint
- testpackage
- thelper
- tparallel
- unconvert
- unparam
- unqueryvet
- usestdlibvars
- usetesting
- varnamelen
- wastedassign
- whitespace
- wsl_v5
- zerologlint
settings:
varnamelen:
ignore-names:
- err
- wg
- mu
- ts
- to
- es
- op
- fd
- id
- fn

View File

@@ -2,7 +2,9 @@ matrix:
platform:
- linux/amd64
goversion:
- 1.24
- 1.26
lintversion:
- v2.12.2
labels:
platform: ${platform}
@@ -21,7 +23,7 @@ steps:
event: [push]
image: golang:${goversion}
commands:
- curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.5.0
- curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin ${lintversion}
- golangci-lint --version
- golangci-lint run ./...
depends_on: [build]

20
ad.go
View File

@@ -29,16 +29,8 @@ type Index struct {
}
type Ad struct {
Title string `goquery:"h1"`
Slug string
ID string
Details string `goquery:".addetailslist--detail,text"`
Attributes map[string]string // processed afterwards
Condition string // post processed from details for backward compatibility
Type string // post processed from details for backward compatibility
Color string // post processed from details for backward compatibility
Material string // post processed from details for backward compatibility
Category string
Title string `goquery:"h1"`
Details string `goquery:".addetailslist--detail,text"`
CategoryTree []string `goquery:".breadcrump-link,text"`
Price string `goquery:"h2#viewad-price"`
Created string `goquery:"#viewad-extra-info,text"`
@@ -46,6 +38,14 @@ type Ad struct {
Images []string `goquery:".galleryimage-element img,[src]"`
Shipping string `goquery:".boxedarticle--details--shipping,text"` // not always filled
Expire string
Slug string
ID string
Attributes map[string]string // processed afterwards
Condition string // post processed from details for backward compatibility
Type string // post processed from details for backward compatibility
Color string // post processed from details for backward compatibility
Material string // post processed from details for backward compatibility
Category string
// runtime computed
Year, Day, Month string

View File

@@ -140,7 +140,7 @@ func InitConfig(output io.Writer) (*Config, error) {
}
// Load default values using the confmap provider.
if err := kloader.Load(confmap.Provider(map[string]interface{}{
if err := kloader.Load(confmap.Provider(map[string]any{
"template": template,
"outdir": DefaultOutdirTemplate,
"loglevel": "notice",
@@ -200,7 +200,6 @@ func InitConfig(output io.Writer) (*Config, error) {
// Load the config file[s]
for _, cfgfile := range configfiles {
path, err := os.Stat(cfgfile)
if err != nil {
// ignore non-existent files, but bail out on any other errors
if !os.IsNotExist(err) {

View File

@@ -24,7 +24,11 @@ import (
"log/slog"
"net/http"
"net/http/cookiejar"
"net/http/httputil"
"net/url"
"os"
"strings"
"sync"
)
// convenient wrapper to fetch some web content
@@ -34,12 +38,25 @@ type Fetcher struct {
Cookies []*http.Cookie
}
// Pos is a counter for debug output file names
type Pos struct {
sync.Mutex
pos int
}
var pos *Pos
func NewFetcher(conf *Config) (*Fetcher, error) {
jar, err := cookiejar.New(nil)
if err != nil {
return nil, fmt.Errorf("failed to create a cookie jar obj: %w", err)
}
if pos == nil {
pos = &Pos{}
}
return &Fetcher{
Client: &http.Client{
Transport: &loggingTransport{}, // implemented in http.go
@@ -82,9 +99,43 @@ func (f *Fetcher) Get(uri string) (io.ReadCloser, error) {
slog.Debug("got cookies?", "cookies", res.Cookies())
f.Cookies = res.Cookies()
if err := f.Dump(res); err != nil {
return nil, err
}
return res.Body, nil
}
func (f *Fetcher) Dump(res *http.Response) error {
if f.Config.Debug && strings.Contains(res.Header.Get("Content-Type"), "text/html") {
// => dump content
pos.Lock()
defer pos.Unlock()
pos.pos++
dump, err := httputil.DumpResponse(res, true)
if err != nil {
return err
}
file := fmt.Sprintf("out%d.html", pos.pos)
fd, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
return err
}
if _, err := fd.Write(dump); err != nil {
return err
}
slog.Debug("dump content", "file", file)
}
return nil
}
// fetch an image
func (f *Fetcher) Getimage(uri string) (io.ReadCloser, error) {
slog.Debug("fetching ad image", "uri", uri)

4
go.mod
View File

@@ -1,8 +1,6 @@
module kleingebaeck
go 1.24.0
toolchain go1.24.5
go 1.26.0
require (
astuart.co/goq v1.0.0

2
go.sum
View File

@@ -66,6 +66,8 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl
github.com/tlinden/yadu v0.1.3 h1:5cRCUmj+l5yvlM2irtpFBIJwVV2DPEgYSaWvF19FtcY=
github.com/tlinden/yadu v0.1.3/go.mod h1:l3bRmHKL9zGAR6pnBHY2HRPxBecf7L74BoBgOOpTcUA=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/image v0.31.0 h1:mLChjE2MV6g1S7oqbXC0/UcKijjm5fnJLUYKIYrLESA=
golang.org/x/image v0.31.0/go.mod h1:R9ec5Lcp96v9FTF+ajwaH3uGxPH4fKfHHAVbUILxghA=
golang.org/x/image v0.32.0 h1:6lZQWq75h7L5IWNk0r+SCpUJ6tUVd3v4ZHnbRKLkUDQ=
golang.org/x/image v0.32.0/go.mod h1:/R37rrQmKXtO6tYXAjtDLwQgFLHmhW+V6ayXlxzP2Pc=
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=

View File

@@ -123,7 +123,6 @@ func (t *loggingTransport) RoundTrip(req *http.Request) (*http.Response, error)
// actual retry
resp, err = http.DefaultTransport.RoundTrip(req)
if err == nil {
slog.Debug("RESPONSE", "id", requestid, "status", resp.StatusCode,
"contentlength", resp.ContentLength, "retry", retries)

View File

@@ -19,6 +19,7 @@ package main
import (
"bytes"
"errors"
"fmt"
"image"
_ "image/gif"
@@ -27,6 +28,7 @@ import (
"log/slog"
"os"
"path/filepath"
"slices"
_ "golang.org/x/image/webp"
@@ -43,18 +45,6 @@ type Image struct {
Mime string
}
// used for logging to avoid printing Data
func (img *Image) LogValue() slog.Value {
return slog.GroupValue(
slog.String("filename", img.Filename),
slog.String("uri", img.URI),
slog.String("hash", img.Hash.ToString()),
)
}
// holds all images of an ad
type Cache []*goimagehash.ImageHash
// filename comes from the scraper, it contains directory/base w/o suffix
func NewImage(buf *bytes.Reader, filename, uri string) (*Image, error) {
_, imgconfig, err := image.DecodeConfig(buf)
@@ -75,7 +65,7 @@ func NewImage(buf *bytes.Reader, filename, uri string) (*Image, error) {
}
if imgconfig == "" {
return nil, fmt.Errorf("failed to process image: unknown or unsupported image format (supported: jpg,png,gif,webp)")
return nil, errors.New("failed to process image: unknown or unsupported image format (supported: jpg,png,gif,webp)")
}
filename += "." + imgconfig
@@ -92,6 +82,18 @@ func NewImage(buf *bytes.Reader, filename, uri string) (*Image, error) {
return img, nil
}
// used for logging to avoid printing Data
func (img *Image) LogValue() slog.Value {
return slog.GroupValue(
slog.String("filename", img.Filename),
slog.String("uri", img.URI),
slog.String("hash", img.Hash.ToString()),
)
}
// holds all images of an ad
type Cache []*goimagehash.ImageHash
// Calculate diff hash of the image
func (img *Image) CalcHash() error {
jpgdata, format, err := image.Decode(img.Data)
@@ -100,7 +102,7 @@ func (img *Image) CalcHash() error {
}
if format == "" {
return fmt.Errorf("failed to decode image: unknown or unsupported image format (supported: jpg,png,gif,webp)")
return errors.New("failed to decode image: unknown or unsupported image format (supported: jpg,png,gif,webp)")
}
hash1, err := goimagehash.DifferenceHash(jpgdata)
@@ -134,13 +136,9 @@ func (img *Image) Similar(hash *goimagehash.ImageHash) bool {
// check current image against all known hashes.
func (img *Image) SimilarExists(cache Cache) bool {
for _, otherimg := range cache {
if img.Similar(otherimg) {
return true
}
}
return false
return slices.ContainsFunc(cache, func(otherimg *goimagehash.ImageHash) bool {
return img.Similar(otherimg)
})
}
// read all JPG images in a ad directory, compute diff hashes and

12
main.go
View File

@@ -49,6 +49,7 @@ func init() {
fmt.Println("Please open a command shell and run it from there.")
fmt.Println()
fmt.Print("Press any key to quit: ")
_, err := bufio.NewReader(os.Stdin).ReadString('\n')
if err != nil {
panic(err)
@@ -144,13 +145,8 @@ func Main(output io.Writer) int {
if err != nil {
return Die(err)
}
conf.Outdir = outdir
// used for all HTTP requests
fetch, err := NewFetcher(conf)
if err != nil {
return Die(err)
}
conf.Outdir = outdir
// setup ad dir registry, needed to check for duplicates
DirsVisited = make(map[string]int)
@@ -159,14 +155,14 @@ func Main(output io.Writer) int {
case len(conf.Adlinks) >= 1:
// directly backup ad listing[s]
for _, uri := range conf.Adlinks {
err := ScrapeAd(fetch, uri)
err = ScrapeAd(conf, uri)
if err != nil {
return Die(err)
}
}
case conf.User > 0:
// backup all ads of the given user (via config or cmdline)
err := ScrapeUser(fetch)
err := ScrapeUser(conf)
if err != nil {
return Die(err)
}

View File

@@ -405,6 +405,7 @@ func InitValidSources() []Adsource {
empty := AdConfig{}
// prepare urls for the listing pages
// nolint:prealloc
ads := []Adsource{
{
uri: fmt.Sprintf("%s%s?userId=1", Baseuri, Listuri),
@@ -436,7 +437,7 @@ func InitInvalidSources() []Adsource {
ads := []Adsource{
{
// valid ad page but without content
uri: fmt.Sprintf("%s/s-anzeige/empty/1", Baseuri),
uri: Baseuri + "/s-anzeige/empty/1",
content: GetTemplate(nil, &empty, EMPTYPAGE),
},
{
@@ -446,18 +447,18 @@ func InitInvalidSources() []Adsource {
},
{
// some invalid page path
uri: fmt.Sprintf("%s/anzeige/name/1", Baseuri),
uri: Baseuri + "/anzeige/name/1",
content: GetTemplate(nil, &empty, "<html></html>"),
},
{
// some none-ad page
uri: fmt.Sprintf("%s/anzeige/name/1/foo/bar", Baseuri),
uri: Baseuri + "/anzeige/name/1/foo/bar",
content: GetTemplate(nil, &empty, "<html>HTTP 404: /eine-anzeige/ does not exist!</html>"),
status: 404,
},
{
// valid ad page but 503
uri: fmt.Sprintf("%s/s-anzeige/503/1", Baseuri),
uri: Baseuri + "/s-anzeige/503/1",
content: GetTemplate(nil, &empty, "<html>HTTP 503: service unavailable</html>"),
status: 503,
},
@@ -531,6 +532,7 @@ func TestMain(t *testing.T) {
defer func() { os.Args = oldargs }()
httpmock.Activate()
defer httpmock.DeactivateAndReset()
// prepare httpmock responders
@@ -568,6 +570,7 @@ func TestMainInvalids(t *testing.T) {
defer func() { os.Args = oldargs }()
httpmock.Activate()
defer httpmock.DeactivateAndReset()
// prepare httpmock responders

View File

@@ -19,7 +19,9 @@ package main
import (
"bytes"
"errors"
"fmt"
"io"
"log/slog"
"path/filepath"
"strconv"
@@ -32,30 +34,44 @@ import (
// extract links from all ad listing pages (that is: use pagination)
// and scrape every page
func ScrapeUser(fetch *Fetcher) error {
func ScrapeUser(conf *Config) error {
adlinks := []string{}
baseuri := fmt.Sprintf("%s%s?userId=%d", Baseuri, Listuri, fetch.Config.User)
baseuri := fmt.Sprintf("%s%s?userId=%d", Baseuri, Listuri, conf.User)
page := 1
uri := baseuri
slog.Info("fetching ad pages", "user", fetch.Config.User)
slog.Info("fetching ad pages", "user", conf.User)
for {
var index Index
slog.Debug("fetching page", "uri", uri)
body, err := fetch.Get(uri)
fetch, err := NewFetcher(conf)
if err != nil {
return err
}
body, err := fetch.Get(uri)
defer func() {
if body == nil {
return
}
// make sure body is completely flushed
if _, err := io.Copy(io.Discard, body); err != nil {
panic(err)
}
if err := body.Close(); err != nil {
panic(err)
}
}()
if err != nil {
return err
}
err = goq.NewDecoder(body).Decode(&index)
if err != nil {
return fmt.Errorf("failed to goquery decode HTML index body: %w", err)
@@ -77,12 +93,12 @@ func ScrapeUser(fetch *Fetcher) error {
}
for index, adlink := range adlinks {
err := ScrapeAd(fetch, Baseuri+adlink)
err := ScrapeAd(conf, Baseuri+adlink)
if err != nil {
return err
}
if fetch.Config.Limit > 0 && index == fetch.Config.Limit-1 {
if conf.Limit > 0 && index == conf.Limit-1 {
break
}
}
@@ -91,7 +107,7 @@ func ScrapeUser(fetch *Fetcher) error {
}
// scrape an ad. uri is the full uri of the ad, dir is the basedir
func ScrapeAd(fetch *Fetcher, uri string) error {
func ScrapeAd(conf *Config, uri string) error {
now := time.Now()
advertisement := &Ad{
@@ -112,16 +128,31 @@ func ScrapeAd(fetch *Fetcher, uri string) error {
// get the ad
slog.Debug("fetching ad page", "uri", uri)
body, err := fetch.Get(uri)
fetch, err := NewFetcher(conf)
if err != nil {
return err
}
body, err := fetch.Get(uri)
defer func() {
if body == nil {
return
}
// make sure body is completely flushed
if _, err := io.Copy(io.Discard, body); err != nil {
panic(err)
}
if err := body.Close(); err != nil {
panic(err)
}
}()
if err != nil {
return err
}
// extract ad contents with goquery/goq
err = goq.NewDecoder(body).Decode(&advertisement)
if err != nil {
@@ -135,7 +166,7 @@ func ScrapeAd(fetch *Fetcher, uri string) error {
if advertisement.Incomplete() {
slog.Debug("got ad", "ad", advertisement)
return fmt.Errorf("could not extract ad data from page, got empty struct")
return errors.New("could not extract ad data from page, got empty struct")
}
advertisement.DecodeAttributes()
@@ -184,14 +215,11 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error {
egroup := new(errgroup.Group)
for _, imguri := range advertisement.Images {
imguri := imguri
// we append the suffix later in NewImage() based on image format
basefilename := filepath.Join(adpath, fmt.Sprintf("%d", img))
basefilename := filepath.Join(adpath, strconv.Itoa(img))
egroup.Go(func() error {
// wait a little
throttle := GetThrottleTime()
time.Sleep(throttle)
@@ -241,6 +269,7 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error {
return nil
})
img++
}

View File

@@ -127,7 +127,6 @@ func WriteImage(filename string, reader *bytes.Reader) error {
}()
_, err = reader.WriteTo(file)
if err != nil {
return fmt.Errorf("failed to write to image file: %w", err)
}
@@ -157,7 +156,6 @@ func ReadImage(filename string) (*bytes.Buffer, error) {
func fileExists(filename string) bool {
info, err := os.Stat(filename)
if err != nil {
// return false on any error
return false
@@ -172,11 +170,13 @@ func CheckAdVisited(conf *Config, adname string) bool {
if Exists(DirsVisited, adname) {
if conf.ForceDownload {
slog.Warn("an ad with the same name has already been downloaded, overwriting", "addir", adname)
return true
}
// don't overwrite
slog.Warn("an ad with the same name has already been downloaded, skipping (use -f to overwrite)", "addir", adname)
return false
}

View File

@@ -53,7 +53,6 @@ func man() error {
man.Stderr = os.Stderr
err := man.Run()
if err != nil {
return fmt.Errorf("failed to execute 'less': %w", err)
}