mirror of
https://codeberg.org/scip/kleingebaeck.git
synced 2026-11-22 22:48:59 +01:00
Compare commits
12 Commits
v0.3.23-co
...
fix/scrapi
| Author | SHA1 | Date | |
|---|---|---|---|
| 16b47aa7b0 | |||
| 562292de2b | |||
| 7f2f61ca9e | |||
| 4e77573223 | |||
| 79892bc55b | |||
| 75087b47bf | |||
| 3c9124a3e6 | |||
| d20a499519 | |||
| 0d5ee0b485 | |||
|
|
b44c1ba1dc | ||
| b6350344be | |||
| e5ff2bbc5c |
106
.golangci.yml
Normal file
106
.golangci.yml
Normal file
@@ -0,0 +1,106 @@
|
||||
version: "2"
|
||||
|
||||
linters:
|
||||
enable:
|
||||
- errcheck
|
||||
- govet
|
||||
- ineffassign
|
||||
- staticcheck
|
||||
- unused
|
||||
- arangolint
|
||||
- asasalint
|
||||
- asciicheck
|
||||
- bidichk
|
||||
- bodyclose
|
||||
- canonicalheader
|
||||
- clickhouselint
|
||||
- containedctx
|
||||
#- contextcheck
|
||||
- copyloopvar
|
||||
#- cyclop
|
||||
- decorder
|
||||
- dogsled
|
||||
- dupword
|
||||
- durationcheck
|
||||
- embeddedstructfieldcheck
|
||||
- errchkjson
|
||||
- errname
|
||||
#- errorlint
|
||||
- exhaustive
|
||||
- exptostd
|
||||
- fatcontext
|
||||
- funcorder
|
||||
#- funlen
|
||||
- ginkgolinter
|
||||
- gocheckcompilerdirectives
|
||||
#- gochecknoinits
|
||||
- gochecksumtype
|
||||
#- gocognit
|
||||
- gocritic
|
||||
#- gocyclo
|
||||
- goheader
|
||||
- gomoddirectives
|
||||
- gomodguard_v2
|
||||
- goprintffuncname
|
||||
#- gosec
|
||||
- gosmopolitan
|
||||
- grouper
|
||||
- iface
|
||||
- importas
|
||||
- inamedparam
|
||||
- interfacebloat
|
||||
- intrange
|
||||
- iotamixing
|
||||
- lll
|
||||
- loggercheck
|
||||
- makezero
|
||||
- misspell
|
||||
- modernize
|
||||
- nakedret
|
||||
#- nestif
|
||||
- nilerr
|
||||
- nilnesserr
|
||||
- nlreturn
|
||||
- nonamedreturns
|
||||
- nosprintfhostport
|
||||
- perfsprint
|
||||
- prealloc
|
||||
- promlinter
|
||||
- protogetter
|
||||
- reassign
|
||||
- recvcheck
|
||||
- rowserrcheck
|
||||
- sloglint
|
||||
- spancheck
|
||||
- sqlclosecheck
|
||||
- tagalign
|
||||
- testableexamples
|
||||
- testifylint
|
||||
- testpackage
|
||||
- thelper
|
||||
- tparallel
|
||||
- unconvert
|
||||
- unparam
|
||||
- unqueryvet
|
||||
- usestdlibvars
|
||||
- usetesting
|
||||
- varnamelen
|
||||
- wastedassign
|
||||
- whitespace
|
||||
- wsl_v5
|
||||
- zerologlint
|
||||
|
||||
settings:
|
||||
varnamelen:
|
||||
ignore-names:
|
||||
- err
|
||||
- wg
|
||||
- mu
|
||||
- ts
|
||||
- to
|
||||
- es
|
||||
- op
|
||||
- fd
|
||||
- id
|
||||
- fn
|
||||
|
||||
@@ -66,4 +66,4 @@ release:
|
||||
|
||||
---
|
||||
|
||||
Full Changelog: [{{ .PreviousTag }}...{{ .Tag }}](https://codeberg.org/scip/epuppy/compare/{{ .PreviousTag }}...{{ .Tag }})
|
||||
Full Changelog: [{{ .PreviousTag }}...{{ .Tag }}](https://codeberg.org/scip/kleingebaeck/compare/{{ .PreviousTag }}...{{ .Tag }})
|
||||
|
||||
@@ -2,7 +2,9 @@ matrix:
|
||||
platform:
|
||||
- linux/amd64
|
||||
goversion:
|
||||
- 1.24
|
||||
- 1.26
|
||||
lintversion:
|
||||
- v2.12.2
|
||||
|
||||
labels:
|
||||
platform: ${platform}
|
||||
@@ -21,7 +23,7 @@ steps:
|
||||
event: [push]
|
||||
image: golang:${goversion}
|
||||
commands:
|
||||
- curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.5.0
|
||||
- curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin ${lintversion}
|
||||
- golangci-lint --version
|
||||
- golangci-lint run ./...
|
||||
depends_on: [build]
|
||||
|
||||
@@ -6,7 +6,7 @@ variables:
|
||||
|
||||
steps:
|
||||
dryrun:
|
||||
image: docker.io/woodpeckerci/plugin-docker-buildx:latest
|
||||
image: docker.io/woodpeckerci/plugin-docker-buildx:6.0.3
|
||||
settings:
|
||||
dockerfile: Dockerfile
|
||||
platforms: linux/amd64
|
||||
@@ -17,7 +17,7 @@ steps:
|
||||
event: [pull_request]
|
||||
|
||||
publish:
|
||||
image: docker.io/woodpeckerci/plugin-docker-buildx:latest
|
||||
image: docker.io/woodpeckerci/plugin-docker-buildx:6.0.3
|
||||
settings:
|
||||
dockerfile: Dockerfile
|
||||
platforms: linux/amd64
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.24-alpine as builder
|
||||
FROM golang:1.26-alpine as builder
|
||||
|
||||
RUN apk update
|
||||
RUN apk upgrade
|
||||
|
||||
21
README-de.md
21
README-de.md
@@ -1,13 +1,12 @@
|
||||
## Kleingebäck - kleinanzeigen.de Backup
|
||||
|
||||

|
||||

|
||||
|
||||
[](https://goreportcard.com/report/codeberg.org/scip/kleingebaeck)
|
||||
[](https://ci.codeberg.org/repos/15530)
|
||||
[](https://raw.githack.com/wiki/tlinden/kleingebaeck/coverage.html)
|
||||

|
||||
[](https://codeberg.org/scip/kleingebaeck/releases)
|
||||
[](https://codeberg.org/scip/kleingebaeck/raw/branch/README.md)
|
||||
[](https://codeberg.org/scip/kleingebaeck/raw/branch/main/README.md)
|
||||
|
||||
Mit diesem Tool kann man seine Anzeigen bei https://kleinanzeigen.de sichern.
|
||||
|
||||
@@ -30,33 +29,33 @@ für mehr Details.
|
||||
|
||||
Das ist die Hauptseite meines kleinanzeigen.de Accounts:
|
||||
|
||||

|
||||

|
||||
|
||||
Sichern ich meine Anzeigen:
|
||||
|
||||

|
||||

|
||||
|
||||
Backupverzeichnis nach dem Download:
|
||||
|
||||

|
||||

|
||||
|
||||
Verzeichnis einer Anzeige:
|
||||
|
||||

|
||||

|
||||
|
||||
**Das gleiche unter Windows:**
|
||||
|
||||
Anzeigen Sichern:
|
||||
|
||||

|
||||

|
||||
|
||||
Backupverzeichnis nach dem Download
|
||||
|
||||

|
||||

|
||||
|
||||
Und eine Anzeige:
|
||||
|
||||

|
||||

|
||||
|
||||
## Installation
|
||||
|
||||
@@ -277,7 +276,7 @@ ignorieren, die beim Bilderdownload auftreten könnten.
|
||||
## Documentation
|
||||
|
||||
Die Dokumentation kann man
|
||||
[online](https://codeberg.org/scip/kleingebaeck/raw/branch/kleingebaeck.pod)
|
||||
[online](https://codeberg.org/scip/kleingebaeck/raw/branch/main/kleingebaeck.pod)
|
||||
oder lokal lesen mit: `kleingebaeck --manual`. Hat man das Tool mit
|
||||
dem Tarball installiert, funktioniert auch `man kleingebaeck`.
|
||||
|
||||
|
||||
21
README.md
21
README.md
@@ -1,13 +1,12 @@
|
||||
## Kleingebäck - kleinanzeigen.de Backup
|
||||
|
||||

|
||||

|
||||
|
||||
[](https://goreportcard.com/report/codeberg.org/scip/kleingebaeck)
|
||||
[](https://ci.codeberg.org/repos/15530)
|
||||
[](https://raw.githack.com/wiki/tlinden/kleingebaeck/coverage.html)
|
||||

|
||||
[](https://codeberg.org/scip/kleingebaeck/releases)
|
||||
[](https://codeberg.org/scip/kleingebaeck/raw/branch/README-de.md)
|
||||
[](https://codeberg.org/scip/kleingebaeck/raw/branch/main/README-de.md)
|
||||
|
||||
[Die deutsche Version des READMEs findet Ihr hier](README-de.md).
|
||||
|
||||
@@ -31,33 +30,33 @@ for more details.
|
||||
|
||||
This is the index of my kleinanzeigen.de Account:
|
||||
|
||||

|
||||

|
||||
|
||||
Here I download my ads on the commandline:
|
||||
|
||||

|
||||

|
||||
|
||||
And this is the backup directory after download:
|
||||
|
||||

|
||||

|
||||
|
||||
Here's a directory for one ad:
|
||||
|
||||

|
||||

|
||||
|
||||
**The same thing under windows:**
|
||||
|
||||
Downloading ads:
|
||||
|
||||

|
||||

|
||||
|
||||
Backup directory after download:
|
||||
|
||||

|
||||

|
||||
|
||||
And one ad listing directory:
|
||||
|
||||

|
||||

|
||||
|
||||
## Installation
|
||||
|
||||
@@ -262,7 +261,7 @@ errors.
|
||||
|
||||
## Documentation
|
||||
|
||||
You can read the documentation [online](https://codeberg.org/scip/kleingebaeck/raw/branch/kleingebaeck.pod) or locally once you have installed kleingebaeck with: `kleingebaeck --manual`.
|
||||
You can read the documentation [online](https://codeberg.org/scip/kleingebaeck/raw/branch/main/kleingebaeck.pod) or locally once you have installed kleingebaeck with: `kleingebaeck --manual`.
|
||||
|
||||
## Kleingebäck?
|
||||
|
||||
|
||||
20
ad.go
20
ad.go
@@ -29,16 +29,8 @@ type Index struct {
|
||||
}
|
||||
|
||||
type Ad struct {
|
||||
Title string `goquery:"h1"`
|
||||
Slug string
|
||||
ID string
|
||||
Details string `goquery:".addetailslist--detail,text"`
|
||||
Attributes map[string]string // processed afterwards
|
||||
Condition string // post processed from details for backward compatibility
|
||||
Type string // post processed from details for backward compatibility
|
||||
Color string // post processed from details for backward compatibility
|
||||
Material string // post processed from details for backward compatibility
|
||||
Category string
|
||||
Title string `goquery:"h1"`
|
||||
Details string `goquery:".addetailslist--detail,text"`
|
||||
CategoryTree []string `goquery:".breadcrump-link,text"`
|
||||
Price string `goquery:"h2#viewad-price"`
|
||||
Created string `goquery:"#viewad-extra-info,text"`
|
||||
@@ -46,6 +38,14 @@ type Ad struct {
|
||||
Images []string `goquery:".galleryimage-element img,[src]"`
|
||||
Shipping string `goquery:".boxedarticle--details--shipping,text"` // not always filled
|
||||
Expire string
|
||||
Slug string
|
||||
ID string
|
||||
Attributes map[string]string // processed afterwards
|
||||
Condition string // post processed from details for backward compatibility
|
||||
Type string // post processed from details for backward compatibility
|
||||
Color string // post processed from details for backward compatibility
|
||||
Material string // post processed from details for backward compatibility
|
||||
Category string
|
||||
|
||||
// runtime computed
|
||||
Year, Day, Month string
|
||||
|
||||
@@ -34,7 +34,7 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
VERSION string = "0.3.23"
|
||||
VERSION string = "0.3.25"
|
||||
Baseuri string = "https://www.kleinanzeigen.de"
|
||||
Listuri string = "/s-bestandsliste.html"
|
||||
Defaultdir string = "."
|
||||
@@ -140,7 +140,7 @@ func InitConfig(output io.Writer) (*Config, error) {
|
||||
}
|
||||
|
||||
// Load default values using the confmap provider.
|
||||
if err := kloader.Load(confmap.Provider(map[string]interface{}{
|
||||
if err := kloader.Load(confmap.Provider(map[string]any{
|
||||
"template": template,
|
||||
"outdir": DefaultOutdirTemplate,
|
||||
"loglevel": "notice",
|
||||
@@ -200,7 +200,6 @@ func InitConfig(output io.Writer) (*Config, error) {
|
||||
// Load the config file[s]
|
||||
for _, cfgfile := range configfiles {
|
||||
path, err := os.Stat(cfgfile)
|
||||
|
||||
if err != nil {
|
||||
// ignore non-existent files, but bail out on any other errors
|
||||
if !os.IsNotExist(err) {
|
||||
|
||||
51
fetch.go
51
fetch.go
@@ -24,7 +24,11 @@ import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/cookiejar"
|
||||
"net/http/httputil"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// convenient wrapper to fetch some web content
|
||||
@@ -34,12 +38,25 @@ type Fetcher struct {
|
||||
Cookies []*http.Cookie
|
||||
}
|
||||
|
||||
// Pos is a counter for debug output file names
|
||||
type Pos struct {
|
||||
sync.Mutex
|
||||
|
||||
pos int
|
||||
}
|
||||
|
||||
var pos *Pos
|
||||
|
||||
func NewFetcher(conf *Config) (*Fetcher, error) {
|
||||
jar, err := cookiejar.New(nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create a cookie jar obj: %w", err)
|
||||
}
|
||||
|
||||
if pos == nil {
|
||||
pos = &Pos{}
|
||||
}
|
||||
|
||||
return &Fetcher{
|
||||
Client: &http.Client{
|
||||
Transport: &loggingTransport{}, // implemented in http.go
|
||||
@@ -82,9 +99,43 @@ func (f *Fetcher) Get(uri string) (io.ReadCloser, error) {
|
||||
slog.Debug("got cookies?", "cookies", res.Cookies())
|
||||
f.Cookies = res.Cookies()
|
||||
|
||||
if err := f.Dump(res); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return res.Body, nil
|
||||
}
|
||||
|
||||
func (f *Fetcher) Dump(res *http.Response) error {
|
||||
if f.Config.Debug && strings.Contains(res.Header.Get("Content-Type"), "text/html") {
|
||||
// => dump content
|
||||
pos.Lock()
|
||||
defer pos.Unlock()
|
||||
|
||||
pos.pos++
|
||||
|
||||
dump, err := httputil.DumpResponse(res, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
file := fmt.Sprintf("out%d.html", pos.pos)
|
||||
|
||||
fd, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := fd.Write(dump); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
slog.Debug("dump content", "file", file)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// fetch an image
|
||||
func (f *Fetcher) Getimage(uri string) (io.ReadCloser, error) {
|
||||
slog.Debug("fetching ad image", "uri", uri)
|
||||
|
||||
6
go.mod
6
go.mod
@@ -1,8 +1,6 @@
|
||||
module kleingebaeck
|
||||
|
||||
go 1.24.0
|
||||
|
||||
toolchain go1.24.5
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
astuart.co/goq v1.0.0
|
||||
@@ -19,7 +17,7 @@ require (
|
||||
github.com/mattn/go-isatty v0.0.20
|
||||
github.com/spf13/pflag v1.0.10
|
||||
github.com/tlinden/yadu v0.1.3
|
||||
golang.org/x/image v0.31.0
|
||||
golang.org/x/image v0.32.0
|
||||
golang.org/x/sync v0.17.0
|
||||
)
|
||||
|
||||
|
||||
2
go.sum
2
go.sum
@@ -68,6 +68,8 @@ github.com/tlinden/yadu v0.1.3/go.mod h1:l3bRmHKL9zGAR6pnBHY2HRPxBecf7L74BoBgOOp
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/image v0.31.0 h1:mLChjE2MV6g1S7oqbXC0/UcKijjm5fnJLUYKIYrLESA=
|
||||
golang.org/x/image v0.31.0/go.mod h1:R9ec5Lcp96v9FTF+ajwaH3uGxPH4fKfHHAVbUILxghA=
|
||||
golang.org/x/image v0.32.0 h1:6lZQWq75h7L5IWNk0r+SCpUJ6tUVd3v4ZHnbRKLkUDQ=
|
||||
golang.org/x/image v0.32.0/go.mod h1:/R37rrQmKXtO6tYXAjtDLwQgFLHmhW+V6ayXlxzP2Pc=
|
||||
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190606173856-1492cefac77f/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
|
||||
1
http.go
1
http.go
@@ -123,7 +123,6 @@ func (t *loggingTransport) RoundTrip(req *http.Request) (*http.Response, error)
|
||||
|
||||
// actual retry
|
||||
resp, err = http.DefaultTransport.RoundTrip(req)
|
||||
|
||||
if err == nil {
|
||||
slog.Debug("RESPONSE", "id", requestid, "status", resp.StatusCode,
|
||||
"contentlength", resp.ContentLength, "retry", retries)
|
||||
|
||||
40
image.go
40
image.go
@@ -19,6 +19,7 @@ package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
_ "image/gif"
|
||||
@@ -27,6 +28,7 @@ import (
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
|
||||
_ "golang.org/x/image/webp"
|
||||
|
||||
@@ -43,18 +45,6 @@ type Image struct {
|
||||
Mime string
|
||||
}
|
||||
|
||||
// used for logging to avoid printing Data
|
||||
func (img *Image) LogValue() slog.Value {
|
||||
return slog.GroupValue(
|
||||
slog.String("filename", img.Filename),
|
||||
slog.String("uri", img.URI),
|
||||
slog.String("hash", img.Hash.ToString()),
|
||||
)
|
||||
}
|
||||
|
||||
// holds all images of an ad
|
||||
type Cache []*goimagehash.ImageHash
|
||||
|
||||
// filename comes from the scraper, it contains directory/base w/o suffix
|
||||
func NewImage(buf *bytes.Reader, filename, uri string) (*Image, error) {
|
||||
_, imgconfig, err := image.DecodeConfig(buf)
|
||||
@@ -75,7 +65,7 @@ func NewImage(buf *bytes.Reader, filename, uri string) (*Image, error) {
|
||||
}
|
||||
|
||||
if imgconfig == "" {
|
||||
return nil, fmt.Errorf("failed to process image: unknown or unsupported image format (supported: jpg,png,gif,webp)")
|
||||
return nil, errors.New("failed to process image: unknown or unsupported image format (supported: jpg,png,gif,webp)")
|
||||
}
|
||||
|
||||
filename += "." + imgconfig
|
||||
@@ -92,6 +82,18 @@ func NewImage(buf *bytes.Reader, filename, uri string) (*Image, error) {
|
||||
return img, nil
|
||||
}
|
||||
|
||||
// used for logging to avoid printing Data
|
||||
func (img *Image) LogValue() slog.Value {
|
||||
return slog.GroupValue(
|
||||
slog.String("filename", img.Filename),
|
||||
slog.String("uri", img.URI),
|
||||
slog.String("hash", img.Hash.ToString()),
|
||||
)
|
||||
}
|
||||
|
||||
// holds all images of an ad
|
||||
type Cache []*goimagehash.ImageHash
|
||||
|
||||
// Calculate diff hash of the image
|
||||
func (img *Image) CalcHash() error {
|
||||
jpgdata, format, err := image.Decode(img.Data)
|
||||
@@ -100,7 +102,7 @@ func (img *Image) CalcHash() error {
|
||||
}
|
||||
|
||||
if format == "" {
|
||||
return fmt.Errorf("failed to decode image: unknown or unsupported image format (supported: jpg,png,gif,webp)")
|
||||
return errors.New("failed to decode image: unknown or unsupported image format (supported: jpg,png,gif,webp)")
|
||||
}
|
||||
|
||||
hash1, err := goimagehash.DifferenceHash(jpgdata)
|
||||
@@ -134,13 +136,9 @@ func (img *Image) Similar(hash *goimagehash.ImageHash) bool {
|
||||
|
||||
// check current image against all known hashes.
|
||||
func (img *Image) SimilarExists(cache Cache) bool {
|
||||
for _, otherimg := range cache {
|
||||
if img.Similar(otherimg) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
return slices.ContainsFunc(cache, func(otherimg *goimagehash.ImageHash) bool {
|
||||
return img.Similar(otherimg)
|
||||
})
|
||||
}
|
||||
|
||||
// read all JPG images in a ad directory, compute diff hashes and
|
||||
|
||||
12
main.go
12
main.go
@@ -49,6 +49,7 @@ func init() {
|
||||
fmt.Println("Please open a command shell and run it from there.")
|
||||
fmt.Println()
|
||||
fmt.Print("Press any key to quit: ")
|
||||
|
||||
_, err := bufio.NewReader(os.Stdin).ReadString('\n')
|
||||
if err != nil {
|
||||
panic(err)
|
||||
@@ -144,13 +145,8 @@ func Main(output io.Writer) int {
|
||||
if err != nil {
|
||||
return Die(err)
|
||||
}
|
||||
conf.Outdir = outdir
|
||||
|
||||
// used for all HTTP requests
|
||||
fetch, err := NewFetcher(conf)
|
||||
if err != nil {
|
||||
return Die(err)
|
||||
}
|
||||
conf.Outdir = outdir
|
||||
|
||||
// setup ad dir registry, needed to check for duplicates
|
||||
DirsVisited = make(map[string]int)
|
||||
@@ -159,14 +155,14 @@ func Main(output io.Writer) int {
|
||||
case len(conf.Adlinks) >= 1:
|
||||
// directly backup ad listing[s]
|
||||
for _, uri := range conf.Adlinks {
|
||||
err := ScrapeAd(fetch, uri)
|
||||
err = ScrapeAd(conf, uri)
|
||||
if err != nil {
|
||||
return Die(err)
|
||||
}
|
||||
}
|
||||
case conf.User > 0:
|
||||
// backup all ads of the given user (via config or cmdline)
|
||||
err := ScrapeUser(fetch)
|
||||
err := ScrapeUser(conf)
|
||||
if err != nil {
|
||||
return Die(err)
|
||||
}
|
||||
|
||||
11
main_test.go
11
main_test.go
@@ -405,6 +405,7 @@ func InitValidSources() []Adsource {
|
||||
empty := AdConfig{}
|
||||
|
||||
// prepare urls for the listing pages
|
||||
// nolint:prealloc
|
||||
ads := []Adsource{
|
||||
{
|
||||
uri: fmt.Sprintf("%s%s?userId=1", Baseuri, Listuri),
|
||||
@@ -436,7 +437,7 @@ func InitInvalidSources() []Adsource {
|
||||
ads := []Adsource{
|
||||
{
|
||||
// valid ad page but without content
|
||||
uri: fmt.Sprintf("%s/s-anzeige/empty/1", Baseuri),
|
||||
uri: Baseuri + "/s-anzeige/empty/1",
|
||||
content: GetTemplate(nil, &empty, EMPTYPAGE),
|
||||
},
|
||||
{
|
||||
@@ -446,18 +447,18 @@ func InitInvalidSources() []Adsource {
|
||||
},
|
||||
{
|
||||
// some invalid page path
|
||||
uri: fmt.Sprintf("%s/anzeige/name/1", Baseuri),
|
||||
uri: Baseuri + "/anzeige/name/1",
|
||||
content: GetTemplate(nil, &empty, "<html></html>"),
|
||||
},
|
||||
{
|
||||
// some none-ad page
|
||||
uri: fmt.Sprintf("%s/anzeige/name/1/foo/bar", Baseuri),
|
||||
uri: Baseuri + "/anzeige/name/1/foo/bar",
|
||||
content: GetTemplate(nil, &empty, "<html>HTTP 404: /eine-anzeige/ does not exist!</html>"),
|
||||
status: 404,
|
||||
},
|
||||
{
|
||||
// valid ad page but 503
|
||||
uri: fmt.Sprintf("%s/s-anzeige/503/1", Baseuri),
|
||||
uri: Baseuri + "/s-anzeige/503/1",
|
||||
content: GetTemplate(nil, &empty, "<html>HTTP 503: service unavailable</html>"),
|
||||
status: 503,
|
||||
},
|
||||
@@ -531,6 +532,7 @@ func TestMain(t *testing.T) {
|
||||
defer func() { os.Args = oldargs }()
|
||||
|
||||
httpmock.Activate()
|
||||
|
||||
defer httpmock.DeactivateAndReset()
|
||||
|
||||
// prepare httpmock responders
|
||||
@@ -568,6 +570,7 @@ func TestMainInvalids(t *testing.T) {
|
||||
defer func() { os.Args = oldargs }()
|
||||
|
||||
httpmock.Activate()
|
||||
|
||||
defer httpmock.DeactivateAndReset()
|
||||
|
||||
// prepare httpmock responders
|
||||
|
||||
55
scrape.go
55
scrape.go
@@ -19,7 +19,9 @@ package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
@@ -32,30 +34,44 @@ import (
|
||||
|
||||
// extract links from all ad listing pages (that is: use pagination)
|
||||
// and scrape every page
|
||||
func ScrapeUser(fetch *Fetcher) error {
|
||||
func ScrapeUser(conf *Config) error {
|
||||
adlinks := []string{}
|
||||
|
||||
baseuri := fmt.Sprintf("%s%s?userId=%d", Baseuri, Listuri, fetch.Config.User)
|
||||
baseuri := fmt.Sprintf("%s%s?userId=%d", Baseuri, Listuri, conf.User)
|
||||
page := 1
|
||||
uri := baseuri
|
||||
|
||||
slog.Info("fetching ad pages", "user", fetch.Config.User)
|
||||
slog.Info("fetching ad pages", "user", conf.User)
|
||||
|
||||
for {
|
||||
var index Index
|
||||
|
||||
slog.Debug("fetching page", "uri", uri)
|
||||
|
||||
body, err := fetch.Get(uri)
|
||||
fetch, err := NewFetcher(conf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
body, err := fetch.Get(uri)
|
||||
defer func() {
|
||||
if body == nil {
|
||||
return
|
||||
}
|
||||
// make sure body is completely flushed
|
||||
if _, err := io.Copy(io.Discard, body); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
if err := body.Close(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}()
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = goq.NewDecoder(body).Decode(&index)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to goquery decode HTML index body: %w", err)
|
||||
@@ -77,12 +93,12 @@ func ScrapeUser(fetch *Fetcher) error {
|
||||
}
|
||||
|
||||
for index, adlink := range adlinks {
|
||||
err := ScrapeAd(fetch, Baseuri+adlink)
|
||||
err := ScrapeAd(conf, Baseuri+adlink)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if fetch.Config.Limit > 0 && index == fetch.Config.Limit-1 {
|
||||
if conf.Limit > 0 && index == conf.Limit-1 {
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -91,7 +107,7 @@ func ScrapeUser(fetch *Fetcher) error {
|
||||
}
|
||||
|
||||
// scrape an ad. uri is the full uri of the ad, dir is the basedir
|
||||
func ScrapeAd(fetch *Fetcher, uri string) error {
|
||||
func ScrapeAd(conf *Config, uri string) error {
|
||||
now := time.Now()
|
||||
|
||||
advertisement := &Ad{
|
||||
@@ -112,16 +128,31 @@ func ScrapeAd(fetch *Fetcher, uri string) error {
|
||||
// get the ad
|
||||
slog.Debug("fetching ad page", "uri", uri)
|
||||
|
||||
body, err := fetch.Get(uri)
|
||||
fetch, err := NewFetcher(conf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
body, err := fetch.Get(uri)
|
||||
defer func() {
|
||||
if body == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// make sure body is completely flushed
|
||||
if _, err := io.Copy(io.Discard, body); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
if err := body.Close(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}()
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// extract ad contents with goquery/goq
|
||||
err = goq.NewDecoder(body).Decode(&advertisement)
|
||||
if err != nil {
|
||||
@@ -135,7 +166,7 @@ func ScrapeAd(fetch *Fetcher, uri string) error {
|
||||
if advertisement.Incomplete() {
|
||||
slog.Debug("got ad", "ad", advertisement)
|
||||
|
||||
return fmt.Errorf("could not extract ad data from page, got empty struct")
|
||||
return errors.New("could not extract ad data from page, got empty struct")
|
||||
}
|
||||
|
||||
advertisement.DecodeAttributes()
|
||||
@@ -184,14 +215,11 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error {
|
||||
egroup := new(errgroup.Group)
|
||||
|
||||
for _, imguri := range advertisement.Images {
|
||||
imguri := imguri
|
||||
|
||||
// we append the suffix later in NewImage() based on image format
|
||||
basefilename := filepath.Join(adpath, fmt.Sprintf("%d", img))
|
||||
basefilename := filepath.Join(adpath, strconv.Itoa(img))
|
||||
|
||||
egroup.Go(func() error {
|
||||
// wait a little
|
||||
|
||||
throttle := GetThrottleTime()
|
||||
time.Sleep(throttle)
|
||||
|
||||
@@ -241,6 +269,7 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error {
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
img++
|
||||
}
|
||||
|
||||
|
||||
4
store.go
4
store.go
@@ -127,7 +127,6 @@ func WriteImage(filename string, reader *bytes.Reader) error {
|
||||
}()
|
||||
|
||||
_, err = reader.WriteTo(file)
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to write to image file: %w", err)
|
||||
}
|
||||
@@ -157,7 +156,6 @@ func ReadImage(filename string) (*bytes.Buffer, error) {
|
||||
|
||||
func fileExists(filename string) bool {
|
||||
info, err := os.Stat(filename)
|
||||
|
||||
if err != nil {
|
||||
// return false on any error
|
||||
return false
|
||||
@@ -172,11 +170,13 @@ func CheckAdVisited(conf *Config, adname string) bool {
|
||||
if Exists(DirsVisited, adname) {
|
||||
if conf.ForceDownload {
|
||||
slog.Warn("an ad with the same name has already been downloaded, overwriting", "addir", adname)
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// don't overwrite
|
||||
slog.Warn("an ad with the same name has already been downloaded, skipping (use -f to overwrite)", "addir", adname)
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user