diff --git a/.golangci.yml b/.golangci.yml index 76d801c..435cc42 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -33,7 +33,7 @@ linters: #- funlen - ginkgolinter - gocheckcompilerdirectives - - gochecknoinits + #- gochecknoinits - gochecksumtype #- gocognit - gocritic diff --git a/config.go b/config.go index ee78d4f..5e04f99 100644 --- a/config.go +++ b/config.go @@ -140,7 +140,7 @@ func InitConfig(output io.Writer) (*Config, error) { } // Load default values using the confmap provider. - if err := kloader.Load(confmap.Provider(map[string]interface{}{ + if err := kloader.Load(confmap.Provider(map[string]any{ "template": template, "outdir": DefaultOutdirTemplate, "loglevel": "notice", @@ -200,7 +200,6 @@ func InitConfig(output io.Writer) (*Config, error) { // Load the config file[s] for _, cfgfile := range configfiles { path, err := os.Stat(cfgfile) - if err != nil { // ignore non-existent files, but bail out on any other errors if !os.IsNotExist(err) { diff --git a/fetch.go b/fetch.go index b4d4a9c..61ab98e 100644 --- a/fetch.go +++ b/fetch.go @@ -41,6 +41,7 @@ type Fetcher struct { // Pos is a counter for debug output file names type Pos struct { sync.Mutex + pos int } @@ -98,7 +99,9 @@ func (f *Fetcher) Get(uri string) (io.ReadCloser, error) { slog.Debug("got cookies?", "cookies", res.Cookies()) f.Cookies = res.Cookies() - f.Dump(res) + if err := f.Dump(res); err != nil { + return nil, err + } return res.Body, nil } @@ -110,18 +113,22 @@ func (f *Fetcher) Dump(res *http.Response) error { defer pos.Unlock() pos.pos++ + dump, err := httputil.DumpResponse(res, true) if err != nil { return err } file := fmt.Sprintf("out%d.html", pos.pos) + fd, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE, 0600) if err != nil { return err } - fd.Write(dump) + if _, err := fd.Write(dump); err != nil { + return err + } slog.Debug("dump content", "file", file) } diff --git a/http.go b/http.go index 2c57758..0e1ce92 100644 --- a/http.go +++ b/http.go @@ -123,7 +123,6 @@ func (t *loggingTransport) RoundTrip(req *http.Request) (*http.Response, error) // actual retry resp, err = http.DefaultTransport.RoundTrip(req) - if err == nil { slog.Debug("RESPONSE", "id", requestid, "status", resp.StatusCode, "contentlength", resp.ContentLength, "retry", retries) diff --git a/image.go b/image.go index 2e56d24..c161119 100644 --- a/image.go +++ b/image.go @@ -19,6 +19,7 @@ package main import ( "bytes" + "errors" "fmt" "image" _ "image/gif" @@ -27,6 +28,7 @@ import ( "log/slog" "os" "path/filepath" + "slices" _ "golang.org/x/image/webp" @@ -43,18 +45,6 @@ type Image struct { Mime string } -// used for logging to avoid printing Data -func (img *Image) LogValue() slog.Value { - return slog.GroupValue( - slog.String("filename", img.Filename), - slog.String("uri", img.URI), - slog.String("hash", img.Hash.ToString()), - ) -} - -// holds all images of an ad -type Cache []*goimagehash.ImageHash - // filename comes from the scraper, it contains directory/base w/o suffix func NewImage(buf *bytes.Reader, filename, uri string) (*Image, error) { _, imgconfig, err := image.DecodeConfig(buf) @@ -75,7 +65,7 @@ func NewImage(buf *bytes.Reader, filename, uri string) (*Image, error) { } if imgconfig == "" { - return nil, fmt.Errorf("failed to process image: unknown or unsupported image format (supported: jpg,png,gif,webp)") + return nil, errors.New("failed to process image: unknown or unsupported image format (supported: jpg,png,gif,webp)") } filename += "." + imgconfig @@ -92,6 +82,18 @@ func NewImage(buf *bytes.Reader, filename, uri string) (*Image, error) { return img, nil } +// used for logging to avoid printing Data +func (img *Image) LogValue() slog.Value { + return slog.GroupValue( + slog.String("filename", img.Filename), + slog.String("uri", img.URI), + slog.String("hash", img.Hash.ToString()), + ) +} + +// holds all images of an ad +type Cache []*goimagehash.ImageHash + // Calculate diff hash of the image func (img *Image) CalcHash() error { jpgdata, format, err := image.Decode(img.Data) @@ -100,7 +102,7 @@ func (img *Image) CalcHash() error { } if format == "" { - return fmt.Errorf("failed to decode image: unknown or unsupported image format (supported: jpg,png,gif,webp)") + return errors.New("failed to decode image: unknown or unsupported image format (supported: jpg,png,gif,webp)") } hash1, err := goimagehash.DifferenceHash(jpgdata) @@ -134,13 +136,9 @@ func (img *Image) Similar(hash *goimagehash.ImageHash) bool { // check current image against all known hashes. func (img *Image) SimilarExists(cache Cache) bool { - for _, otherimg := range cache { - if img.Similar(otherimg) { - return true - } - } - - return false + return slices.ContainsFunc(cache, func(otherimg *goimagehash.ImageHash) bool { + return img.Similar(otherimg) + }) } // read all JPG images in a ad directory, compute diff hashes and diff --git a/main.go b/main.go index c7c100c..d3a6db4 100644 --- a/main.go +++ b/main.go @@ -49,6 +49,7 @@ func init() { fmt.Println("Please open a command shell and run it from there.") fmt.Println() fmt.Print("Press any key to quit: ") + _, err := bufio.NewReader(os.Stdin).ReadString('\n') if err != nil { panic(err) @@ -144,6 +145,7 @@ func Main(output io.Writer) int { if err != nil { return Die(err) } + conf.Outdir = outdir // setup ad dir registry, needed to check for duplicates diff --git a/main_test.go b/main_test.go index 37c9da6..c708043 100644 --- a/main_test.go +++ b/main_test.go @@ -405,27 +405,26 @@ func InitValidSources() []Adsource { empty := AdConfig{} // prepare urls for the listing pages - ads := []Adsource{ - { - uri: fmt.Sprintf("%s%s?userId=1", Baseuri, Listuri), - content: GetTemplate(list1, &empty, LISTTPL), - }, - { - uri: fmt.Sprintf("%s%s?userId=1&pageNum=2", Baseuri, Listuri), - content: GetTemplate(list2, &empty, LISTTPL), - }, - { - uri: fmt.Sprintf("%s%s?userId=1&pageNum=3", Baseuri, Listuri), - content: GetTemplate(list3, &empty, LISTTPL), - }, + ads := make([]Adsource, 3+len(adsrc)) + ads[0] = Adsource{ + uri: fmt.Sprintf("%s%s?userId=1", Baseuri, Listuri), + content: GetTemplate(list1, &empty, LISTTPL), + } + ads[1] = Adsource{ + uri: fmt.Sprintf("%s%s?userId=1&pageNum=2", Baseuri, Listuri), + content: GetTemplate(list2, &empty, LISTTPL), + } + ads[2] = Adsource{ + uri: fmt.Sprintf("%s%s?userId=1&pageNum=3", Baseuri, Listuri), + content: GetTemplate(list3, &empty, LISTTPL), } // prepare urls for the ads - for _, ad := range adsrc { - ads = append(ads, Adsource{ + for idx, ad := range adsrc { + ads[idx] = Adsource{ uri: fmt.Sprintf("%s/s-anzeige/%s/%s", Baseuri, ad.Slug, ad.ID), content: GetTemplate(nil, &ad, ADTPL), - }) + } } return ads @@ -436,7 +435,7 @@ func InitInvalidSources() []Adsource { ads := []Adsource{ { // valid ad page but without content - uri: fmt.Sprintf("%s/s-anzeige/empty/1", Baseuri), + uri: Baseuri + "/s-anzeige/empty/1", content: GetTemplate(nil, &empty, EMPTYPAGE), }, { @@ -446,18 +445,18 @@ func InitInvalidSources() []Adsource { }, { // some invalid page path - uri: fmt.Sprintf("%s/anzeige/name/1", Baseuri), + uri: Baseuri + "/anzeige/name/1", content: GetTemplate(nil, &empty, ""), }, { // some none-ad page - uri: fmt.Sprintf("%s/anzeige/name/1/foo/bar", Baseuri), + uri: Baseuri + "/anzeige/name/1/foo/bar", content: GetTemplate(nil, &empty, "HTTP 404: /eine-anzeige/ does not exist!"), status: 404, }, { // valid ad page but 503 - uri: fmt.Sprintf("%s/s-anzeige/503/1", Baseuri), + uri: Baseuri + "/s-anzeige/503/1", content: GetTemplate(nil, &empty, "HTTP 503: service unavailable"), status: 503, }, @@ -531,6 +530,7 @@ func TestMain(t *testing.T) { defer func() { os.Args = oldargs }() httpmock.Activate() + defer httpmock.DeactivateAndReset() // prepare httpmock responders @@ -568,6 +568,7 @@ func TestMainInvalids(t *testing.T) { defer func() { os.Args = oldargs }() httpmock.Activate() + defer httpmock.DeactivateAndReset() // prepare httpmock responders diff --git a/scrape.go b/scrape.go index d904f05..fe356fe 100644 --- a/scrape.go +++ b/scrape.go @@ -19,6 +19,7 @@ package main import ( "bytes" + "errors" "fmt" "io" "log/slog" @@ -63,6 +64,7 @@ func ScrapeUser(conf *Config) error { panic(err) } }() + if err != nil { return err } @@ -139,6 +141,7 @@ func ScrapeAd(conf *Config, uri string) error { panic(err) } }() + if err != nil { return err } @@ -156,7 +159,7 @@ func ScrapeAd(conf *Config, uri string) error { if advertisement.Incomplete() { slog.Debug("got ad", "ad", advertisement) - return fmt.Errorf("could not extract ad data from page, got empty struct") + return errors.New("could not extract ad data from page, got empty struct") } advertisement.DecodeAttributes() @@ -206,7 +209,7 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error { for _, imguri := range advertisement.Images { // we append the suffix later in NewImage() based on image format - basefilename := filepath.Join(adpath, fmt.Sprintf("%d", img)) + basefilename := filepath.Join(adpath, strconv.Itoa(img)) egroup.Go(func() error { // wait a little @@ -259,6 +262,7 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error { return nil }) + img++ } diff --git a/store.go b/store.go index 4e8fa78..669eb59 100644 --- a/store.go +++ b/store.go @@ -127,7 +127,6 @@ func WriteImage(filename string, reader *bytes.Reader) error { }() _, err = reader.WriteTo(file) - if err != nil { return fmt.Errorf("failed to write to image file: %w", err) } @@ -157,7 +156,6 @@ func ReadImage(filename string) (*bytes.Buffer, error) { func fileExists(filename string) bool { info, err := os.Stat(filename) - if err != nil { // return false on any error return false @@ -172,11 +170,13 @@ func CheckAdVisited(conf *Config, adname string) bool { if Exists(DirsVisited, adname) { if conf.ForceDownload { slog.Warn("an ad with the same name has already been downloaded, overwriting", "addir", adname) + return true } // don't overwrite slog.Warn("an ad with the same name has already been downloaded, skipping (use -f to overwrite)", "addir", adname) + return false } diff --git a/util.go b/util.go index 2c157cf..c7bc439 100644 --- a/util.go +++ b/util.go @@ -53,7 +53,6 @@ func man() error { man.Stderr = os.Stderr err := man.Run() - if err != nil { return fmt.Errorf("failed to execute 'less': %w", err) }