diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..435cc42 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,106 @@ +version: "2" + +linters: + enable: + - errcheck + - govet + - ineffassign + - staticcheck + - unused + - arangolint + - asasalint + - asciicheck + - bidichk + - bodyclose + - canonicalheader + - clickhouselint + - containedctx + #- contextcheck + - copyloopvar + #- cyclop + - decorder + - dogsled + - dupword + - durationcheck + - embeddedstructfieldcheck + - errchkjson + - errname + #- errorlint + - exhaustive + - exptostd + - fatcontext + - funcorder + #- funlen + - ginkgolinter + - gocheckcompilerdirectives + #- gochecknoinits + - gochecksumtype + #- gocognit + - gocritic + #- gocyclo + - goheader + - gomoddirectives + - gomodguard_v2 + - goprintffuncname + #- gosec + - gosmopolitan + - grouper + - iface + - importas + - inamedparam + - interfacebloat + - intrange + - iotamixing + - lll + - loggercheck + - makezero + - misspell + - modernize + - nakedret + #- nestif + - nilerr + - nilnesserr + - nlreturn + - nonamedreturns + - nosprintfhostport + - perfsprint + - prealloc + - promlinter + - protogetter + - reassign + - recvcheck + - rowserrcheck + - sloglint + - spancheck + - sqlclosecheck + - tagalign + - testableexamples + - testifylint + - testpackage + - thelper + - tparallel + - unconvert + - unparam + - unqueryvet + - usestdlibvars + - usetesting + - varnamelen + - wastedassign + - whitespace + - wsl_v5 + - zerologlint + + settings: + varnamelen: + ignore-names: + - err + - wg + - mu + - ts + - to + - es + - op + - fd + - id + - fn + diff --git a/.woodpecker/build.yaml b/.woodpecker/build.yaml index e20cc6f..9b8393b 100644 --- a/.woodpecker/build.yaml +++ b/.woodpecker/build.yaml @@ -2,7 +2,9 @@ matrix: platform: - linux/amd64 goversion: - - 1.24 + - 1.26 + lintversion: + - v2.12.2 labels: platform: ${platform} @@ -21,7 +23,7 @@ steps: event: [push] image: golang:${goversion} commands: - - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.5.0 + - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin ${lintversion} - golangci-lint --version - golangci-lint run ./... depends_on: [build] diff --git a/Dockerfile b/Dockerfile index 2439c45..497b201 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.24-alpine as builder +FROM golang:1.26-alpine as builder RUN apk update RUN apk upgrade diff --git a/ad.go b/ad.go index c32e268..d277839 100644 --- a/ad.go +++ b/ad.go @@ -29,16 +29,8 @@ type Index struct { } type Ad struct { - Title string `goquery:"h1"` - Slug string - ID string - Details string `goquery:".addetailslist--detail,text"` - Attributes map[string]string // processed afterwards - Condition string // post processed from details for backward compatibility - Type string // post processed from details for backward compatibility - Color string // post processed from details for backward compatibility - Material string // post processed from details for backward compatibility - Category string + Title string `goquery:"h1"` + Details string `goquery:".addetailslist--detail,text"` CategoryTree []string `goquery:".breadcrump-link,text"` Price string `goquery:"h2#viewad-price"` Created string `goquery:"#viewad-extra-info,text"` @@ -46,6 +38,14 @@ type Ad struct { Images []string `goquery:".galleryimage-element img,[src]"` Shipping string `goquery:".boxedarticle--details--shipping,text"` // not always filled Expire string + Slug string + ID string + Attributes map[string]string // processed afterwards + Condition string // post processed from details for backward compatibility + Type string // post processed from details for backward compatibility + Color string // post processed from details for backward compatibility + Material string // post processed from details for backward compatibility + Category string // runtime computed Year, Day, Month string diff --git a/config.go b/config.go index ee78d4f..3dc9941 100644 --- a/config.go +++ b/config.go @@ -34,7 +34,7 @@ import ( ) const ( - VERSION string = "0.3.24" + VERSION string = "0.3.25" Baseuri string = "https://www.kleinanzeigen.de" Listuri string = "/s-bestandsliste.html" Defaultdir string = "." @@ -140,7 +140,7 @@ func InitConfig(output io.Writer) (*Config, error) { } // Load default values using the confmap provider. - if err := kloader.Load(confmap.Provider(map[string]interface{}{ + if err := kloader.Load(confmap.Provider(map[string]any{ "template": template, "outdir": DefaultOutdirTemplate, "loglevel": "notice", @@ -200,7 +200,6 @@ func InitConfig(output io.Writer) (*Config, error) { // Load the config file[s] for _, cfgfile := range configfiles { path, err := os.Stat(cfgfile) - if err != nil { // ignore non-existent files, but bail out on any other errors if !os.IsNotExist(err) { diff --git a/fetch.go b/fetch.go index bfdac24..61ab98e 100644 --- a/fetch.go +++ b/fetch.go @@ -24,7 +24,11 @@ import ( "log/slog" "net/http" "net/http/cookiejar" + "net/http/httputil" "net/url" + "os" + "strings" + "sync" ) // convenient wrapper to fetch some web content @@ -34,12 +38,25 @@ type Fetcher struct { Cookies []*http.Cookie } +// Pos is a counter for debug output file names +type Pos struct { + sync.Mutex + + pos int +} + +var pos *Pos + func NewFetcher(conf *Config) (*Fetcher, error) { jar, err := cookiejar.New(nil) if err != nil { return nil, fmt.Errorf("failed to create a cookie jar obj: %w", err) } + if pos == nil { + pos = &Pos{} + } + return &Fetcher{ Client: &http.Client{ Transport: &loggingTransport{}, // implemented in http.go @@ -82,9 +99,43 @@ func (f *Fetcher) Get(uri string) (io.ReadCloser, error) { slog.Debug("got cookies?", "cookies", res.Cookies()) f.Cookies = res.Cookies() + if err := f.Dump(res); err != nil { + return nil, err + } + return res.Body, nil } +func (f *Fetcher) Dump(res *http.Response) error { + if f.Config.Debug && strings.Contains(res.Header.Get("Content-Type"), "text/html") { + // => dump content + pos.Lock() + defer pos.Unlock() + + pos.pos++ + + dump, err := httputil.DumpResponse(res, true) + if err != nil { + return err + } + + file := fmt.Sprintf("out%d.html", pos.pos) + + fd, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE, 0600) + if err != nil { + return err + } + + if _, err := fd.Write(dump); err != nil { + return err + } + + slog.Debug("dump content", "file", file) + } + + return nil +} + // fetch an image func (f *Fetcher) Getimage(uri string) (io.ReadCloser, error) { slog.Debug("fetching ad image", "uri", uri) diff --git a/go.mod b/go.mod index 0dd7796..809354e 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module kleingebaeck -go 1.24.0 - -toolchain go1.24.5 +go 1.26.0 require ( astuart.co/goq v1.0.0 diff --git a/go.sum b/go.sum index 66a3faf..ed3aca6 100644 --- a/go.sum +++ b/go.sum @@ -66,6 +66,8 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/tlinden/yadu v0.1.3 h1:5cRCUmj+l5yvlM2irtpFBIJwVV2DPEgYSaWvF19FtcY= github.com/tlinden/yadu v0.1.3/go.mod h1:l3bRmHKL9zGAR6pnBHY2HRPxBecf7L74BoBgOOpTcUA= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/image v0.31.0 h1:mLChjE2MV6g1S7oqbXC0/UcKijjm5fnJLUYKIYrLESA= +golang.org/x/image v0.31.0/go.mod h1:R9ec5Lcp96v9FTF+ajwaH3uGxPH4fKfHHAVbUILxghA= golang.org/x/image v0.32.0 h1:6lZQWq75h7L5IWNk0r+SCpUJ6tUVd3v4ZHnbRKLkUDQ= golang.org/x/image v0.32.0/go.mod h1:/R37rrQmKXtO6tYXAjtDLwQgFLHmhW+V6ayXlxzP2Pc= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/http.go b/http.go index 2c57758..0e1ce92 100644 --- a/http.go +++ b/http.go @@ -123,7 +123,6 @@ func (t *loggingTransport) RoundTrip(req *http.Request) (*http.Response, error) // actual retry resp, err = http.DefaultTransport.RoundTrip(req) - if err == nil { slog.Debug("RESPONSE", "id", requestid, "status", resp.StatusCode, "contentlength", resp.ContentLength, "retry", retries) diff --git a/image.go b/image.go index 2e56d24..c161119 100644 --- a/image.go +++ b/image.go @@ -19,6 +19,7 @@ package main import ( "bytes" + "errors" "fmt" "image" _ "image/gif" @@ -27,6 +28,7 @@ import ( "log/slog" "os" "path/filepath" + "slices" _ "golang.org/x/image/webp" @@ -43,18 +45,6 @@ type Image struct { Mime string } -// used for logging to avoid printing Data -func (img *Image) LogValue() slog.Value { - return slog.GroupValue( - slog.String("filename", img.Filename), - slog.String("uri", img.URI), - slog.String("hash", img.Hash.ToString()), - ) -} - -// holds all images of an ad -type Cache []*goimagehash.ImageHash - // filename comes from the scraper, it contains directory/base w/o suffix func NewImage(buf *bytes.Reader, filename, uri string) (*Image, error) { _, imgconfig, err := image.DecodeConfig(buf) @@ -75,7 +65,7 @@ func NewImage(buf *bytes.Reader, filename, uri string) (*Image, error) { } if imgconfig == "" { - return nil, fmt.Errorf("failed to process image: unknown or unsupported image format (supported: jpg,png,gif,webp)") + return nil, errors.New("failed to process image: unknown or unsupported image format (supported: jpg,png,gif,webp)") } filename += "." + imgconfig @@ -92,6 +82,18 @@ func NewImage(buf *bytes.Reader, filename, uri string) (*Image, error) { return img, nil } +// used for logging to avoid printing Data +func (img *Image) LogValue() slog.Value { + return slog.GroupValue( + slog.String("filename", img.Filename), + slog.String("uri", img.URI), + slog.String("hash", img.Hash.ToString()), + ) +} + +// holds all images of an ad +type Cache []*goimagehash.ImageHash + // Calculate diff hash of the image func (img *Image) CalcHash() error { jpgdata, format, err := image.Decode(img.Data) @@ -100,7 +102,7 @@ func (img *Image) CalcHash() error { } if format == "" { - return fmt.Errorf("failed to decode image: unknown or unsupported image format (supported: jpg,png,gif,webp)") + return errors.New("failed to decode image: unknown or unsupported image format (supported: jpg,png,gif,webp)") } hash1, err := goimagehash.DifferenceHash(jpgdata) @@ -134,13 +136,9 @@ func (img *Image) Similar(hash *goimagehash.ImageHash) bool { // check current image against all known hashes. func (img *Image) SimilarExists(cache Cache) bool { - for _, otherimg := range cache { - if img.Similar(otherimg) { - return true - } - } - - return false + return slices.ContainsFunc(cache, func(otherimg *goimagehash.ImageHash) bool { + return img.Similar(otherimg) + }) } // read all JPG images in a ad directory, compute diff hashes and diff --git a/main.go b/main.go index b161607..d3a6db4 100644 --- a/main.go +++ b/main.go @@ -49,6 +49,7 @@ func init() { fmt.Println("Please open a command shell and run it from there.") fmt.Println() fmt.Print("Press any key to quit: ") + _, err := bufio.NewReader(os.Stdin).ReadString('\n') if err != nil { panic(err) @@ -144,13 +145,8 @@ func Main(output io.Writer) int { if err != nil { return Die(err) } - conf.Outdir = outdir - // used for all HTTP requests - fetch, err := NewFetcher(conf) - if err != nil { - return Die(err) - } + conf.Outdir = outdir // setup ad dir registry, needed to check for duplicates DirsVisited = make(map[string]int) @@ -159,14 +155,14 @@ func Main(output io.Writer) int { case len(conf.Adlinks) >= 1: // directly backup ad listing[s] for _, uri := range conf.Adlinks { - err := ScrapeAd(fetch, uri) + err = ScrapeAd(conf, uri) if err != nil { return Die(err) } } case conf.User > 0: // backup all ads of the given user (via config or cmdline) - err := ScrapeUser(fetch) + err := ScrapeUser(conf) if err != nil { return Die(err) } diff --git a/main_test.go b/main_test.go index 37c9da6..5c5378c 100644 --- a/main_test.go +++ b/main_test.go @@ -405,6 +405,7 @@ func InitValidSources() []Adsource { empty := AdConfig{} // prepare urls for the listing pages + // nolint:prealloc ads := []Adsource{ { uri: fmt.Sprintf("%s%s?userId=1", Baseuri, Listuri), @@ -436,7 +437,7 @@ func InitInvalidSources() []Adsource { ads := []Adsource{ { // valid ad page but without content - uri: fmt.Sprintf("%s/s-anzeige/empty/1", Baseuri), + uri: Baseuri + "/s-anzeige/empty/1", content: GetTemplate(nil, &empty, EMPTYPAGE), }, { @@ -446,18 +447,18 @@ func InitInvalidSources() []Adsource { }, { // some invalid page path - uri: fmt.Sprintf("%s/anzeige/name/1", Baseuri), + uri: Baseuri + "/anzeige/name/1", content: GetTemplate(nil, &empty, ""), }, { // some none-ad page - uri: fmt.Sprintf("%s/anzeige/name/1/foo/bar", Baseuri), + uri: Baseuri + "/anzeige/name/1/foo/bar", content: GetTemplate(nil, &empty, "HTTP 404: /eine-anzeige/ does not exist!"), status: 404, }, { // valid ad page but 503 - uri: fmt.Sprintf("%s/s-anzeige/503/1", Baseuri), + uri: Baseuri + "/s-anzeige/503/1", content: GetTemplate(nil, &empty, "HTTP 503: service unavailable"), status: 503, }, @@ -531,6 +532,7 @@ func TestMain(t *testing.T) { defer func() { os.Args = oldargs }() httpmock.Activate() + defer httpmock.DeactivateAndReset() // prepare httpmock responders @@ -568,6 +570,7 @@ func TestMainInvalids(t *testing.T) { defer func() { os.Args = oldargs }() httpmock.Activate() + defer httpmock.DeactivateAndReset() // prepare httpmock responders diff --git a/scrape.go b/scrape.go index 7af3261..82d0ac4 100644 --- a/scrape.go +++ b/scrape.go @@ -19,7 +19,9 @@ package main import ( "bytes" + "errors" "fmt" + "io" "log/slog" "path/filepath" "strconv" @@ -32,30 +34,44 @@ import ( // extract links from all ad listing pages (that is: use pagination) // and scrape every page -func ScrapeUser(fetch *Fetcher) error { +func ScrapeUser(conf *Config) error { adlinks := []string{} - baseuri := fmt.Sprintf("%s%s?userId=%d", Baseuri, Listuri, fetch.Config.User) + baseuri := fmt.Sprintf("%s%s?userId=%d", Baseuri, Listuri, conf.User) page := 1 uri := baseuri - slog.Info("fetching ad pages", "user", fetch.Config.User) + slog.Info("fetching ad pages", "user", conf.User) for { var index Index slog.Debug("fetching page", "uri", uri) - body, err := fetch.Get(uri) + fetch, err := NewFetcher(conf) if err != nil { return err } + + body, err := fetch.Get(uri) defer func() { + if body == nil { + return + } + // make sure body is completely flushed + if _, err := io.Copy(io.Discard, body); err != nil { + panic(err) + } + if err := body.Close(); err != nil { panic(err) } }() + if err != nil { + return err + } + err = goq.NewDecoder(body).Decode(&index) if err != nil { return fmt.Errorf("failed to goquery decode HTML index body: %w", err) @@ -77,12 +93,12 @@ func ScrapeUser(fetch *Fetcher) error { } for index, adlink := range adlinks { - err := ScrapeAd(fetch, Baseuri+adlink) + err := ScrapeAd(conf, Baseuri+adlink) if err != nil { return err } - if fetch.Config.Limit > 0 && index == fetch.Config.Limit-1 { + if conf.Limit > 0 && index == conf.Limit-1 { break } } @@ -91,7 +107,7 @@ func ScrapeUser(fetch *Fetcher) error { } // scrape an ad. uri is the full uri of the ad, dir is the basedir -func ScrapeAd(fetch *Fetcher, uri string) error { +func ScrapeAd(conf *Config, uri string) error { now := time.Now() advertisement := &Ad{ @@ -112,16 +128,31 @@ func ScrapeAd(fetch *Fetcher, uri string) error { // get the ad slog.Debug("fetching ad page", "uri", uri) - body, err := fetch.Get(uri) + fetch, err := NewFetcher(conf) if err != nil { return err } + + body, err := fetch.Get(uri) defer func() { + if body == nil { + return + } + + // make sure body is completely flushed + if _, err := io.Copy(io.Discard, body); err != nil { + panic(err) + } + if err := body.Close(); err != nil { panic(err) } }() + if err != nil { + return err + } + // extract ad contents with goquery/goq err = goq.NewDecoder(body).Decode(&advertisement) if err != nil { @@ -135,7 +166,7 @@ func ScrapeAd(fetch *Fetcher, uri string) error { if advertisement.Incomplete() { slog.Debug("got ad", "ad", advertisement) - return fmt.Errorf("could not extract ad data from page, got empty struct") + return errors.New("could not extract ad data from page, got empty struct") } advertisement.DecodeAttributes() @@ -184,14 +215,11 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error { egroup := new(errgroup.Group) for _, imguri := range advertisement.Images { - imguri := imguri - // we append the suffix later in NewImage() based on image format - basefilename := filepath.Join(adpath, fmt.Sprintf("%d", img)) + basefilename := filepath.Join(adpath, strconv.Itoa(img)) egroup.Go(func() error { // wait a little - throttle := GetThrottleTime() time.Sleep(throttle) @@ -241,6 +269,7 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error { return nil }) + img++ } diff --git a/store.go b/store.go index 4e8fa78..669eb59 100644 --- a/store.go +++ b/store.go @@ -127,7 +127,6 @@ func WriteImage(filename string, reader *bytes.Reader) error { }() _, err = reader.WriteTo(file) - if err != nil { return fmt.Errorf("failed to write to image file: %w", err) } @@ -157,7 +156,6 @@ func ReadImage(filename string) (*bytes.Buffer, error) { func fileExists(filename string) bool { info, err := os.Stat(filename) - if err != nil { // return false on any error return false @@ -172,11 +170,13 @@ func CheckAdVisited(conf *Config, adname string) bool { if Exists(DirsVisited, adname) { if conf.ForceDownload { slog.Warn("an ad with the same name has already been downloaded, overwriting", "addir", adname) + return true } // don't overwrite slog.Warn("an ad with the same name has already been downloaded, skipping (use -f to overwrite)", "addir", adname) + return false } diff --git a/util.go b/util.go index 2c157cf..c7bc439 100644 --- a/util.go +++ b/util.go @@ -53,7 +53,6 @@ func man() error { man.Stderr = os.Stderr err := man.Run() - if err != nil { return fmt.Errorf("failed to execute 'less': %w", err) }