mirror of
https://codeberg.org/scip/kleingebaeck.git
synced 2026-11-22 22:48:59 +01:00
satisfy linter
This commit is contained in:
@@ -33,7 +33,7 @@ linters:
|
||||
#- funlen
|
||||
- ginkgolinter
|
||||
- gocheckcompilerdirectives
|
||||
- gochecknoinits
|
||||
#- gochecknoinits
|
||||
- gochecksumtype
|
||||
#- gocognit
|
||||
- gocritic
|
||||
|
||||
@@ -140,7 +140,7 @@ func InitConfig(output io.Writer) (*Config, error) {
|
||||
}
|
||||
|
||||
// Load default values using the confmap provider.
|
||||
if err := kloader.Load(confmap.Provider(map[string]interface{}{
|
||||
if err := kloader.Load(confmap.Provider(map[string]any{
|
||||
"template": template,
|
||||
"outdir": DefaultOutdirTemplate,
|
||||
"loglevel": "notice",
|
||||
@@ -200,7 +200,6 @@ func InitConfig(output io.Writer) (*Config, error) {
|
||||
// Load the config file[s]
|
||||
for _, cfgfile := range configfiles {
|
||||
path, err := os.Stat(cfgfile)
|
||||
|
||||
if err != nil {
|
||||
// ignore non-existent files, but bail out on any other errors
|
||||
if !os.IsNotExist(err) {
|
||||
|
||||
11
fetch.go
11
fetch.go
@@ -41,6 +41,7 @@ type Fetcher struct {
|
||||
// Pos is a counter for debug output file names
|
||||
type Pos struct {
|
||||
sync.Mutex
|
||||
|
||||
pos int
|
||||
}
|
||||
|
||||
@@ -98,7 +99,9 @@ func (f *Fetcher) Get(uri string) (io.ReadCloser, error) {
|
||||
slog.Debug("got cookies?", "cookies", res.Cookies())
|
||||
f.Cookies = res.Cookies()
|
||||
|
||||
f.Dump(res)
|
||||
if err := f.Dump(res); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return res.Body, nil
|
||||
}
|
||||
@@ -110,18 +113,22 @@ func (f *Fetcher) Dump(res *http.Response) error {
|
||||
defer pos.Unlock()
|
||||
|
||||
pos.pos++
|
||||
|
||||
dump, err := httputil.DumpResponse(res, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
file := fmt.Sprintf("out%d.html", pos.pos)
|
||||
|
||||
fd, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fd.Write(dump)
|
||||
if _, err := fd.Write(dump); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
slog.Debug("dump content", "file", file)
|
||||
}
|
||||
|
||||
1
http.go
1
http.go
@@ -123,7 +123,6 @@ func (t *loggingTransport) RoundTrip(req *http.Request) (*http.Response, error)
|
||||
|
||||
// actual retry
|
||||
resp, err = http.DefaultTransport.RoundTrip(req)
|
||||
|
||||
if err == nil {
|
||||
slog.Debug("RESPONSE", "id", requestid, "status", resp.StatusCode,
|
||||
"contentlength", resp.ContentLength, "retry", retries)
|
||||
|
||||
40
image.go
40
image.go
@@ -19,6 +19,7 @@ package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
_ "image/gif"
|
||||
@@ -27,6 +28,7 @@ import (
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
|
||||
_ "golang.org/x/image/webp"
|
||||
|
||||
@@ -43,18 +45,6 @@ type Image struct {
|
||||
Mime string
|
||||
}
|
||||
|
||||
// used for logging to avoid printing Data
|
||||
func (img *Image) LogValue() slog.Value {
|
||||
return slog.GroupValue(
|
||||
slog.String("filename", img.Filename),
|
||||
slog.String("uri", img.URI),
|
||||
slog.String("hash", img.Hash.ToString()),
|
||||
)
|
||||
}
|
||||
|
||||
// holds all images of an ad
|
||||
type Cache []*goimagehash.ImageHash
|
||||
|
||||
// filename comes from the scraper, it contains directory/base w/o suffix
|
||||
func NewImage(buf *bytes.Reader, filename, uri string) (*Image, error) {
|
||||
_, imgconfig, err := image.DecodeConfig(buf)
|
||||
@@ -75,7 +65,7 @@ func NewImage(buf *bytes.Reader, filename, uri string) (*Image, error) {
|
||||
}
|
||||
|
||||
if imgconfig == "" {
|
||||
return nil, fmt.Errorf("failed to process image: unknown or unsupported image format (supported: jpg,png,gif,webp)")
|
||||
return nil, errors.New("failed to process image: unknown or unsupported image format (supported: jpg,png,gif,webp)")
|
||||
}
|
||||
|
||||
filename += "." + imgconfig
|
||||
@@ -92,6 +82,18 @@ func NewImage(buf *bytes.Reader, filename, uri string) (*Image, error) {
|
||||
return img, nil
|
||||
}
|
||||
|
||||
// used for logging to avoid printing Data
|
||||
func (img *Image) LogValue() slog.Value {
|
||||
return slog.GroupValue(
|
||||
slog.String("filename", img.Filename),
|
||||
slog.String("uri", img.URI),
|
||||
slog.String("hash", img.Hash.ToString()),
|
||||
)
|
||||
}
|
||||
|
||||
// holds all images of an ad
|
||||
type Cache []*goimagehash.ImageHash
|
||||
|
||||
// Calculate diff hash of the image
|
||||
func (img *Image) CalcHash() error {
|
||||
jpgdata, format, err := image.Decode(img.Data)
|
||||
@@ -100,7 +102,7 @@ func (img *Image) CalcHash() error {
|
||||
}
|
||||
|
||||
if format == "" {
|
||||
return fmt.Errorf("failed to decode image: unknown or unsupported image format (supported: jpg,png,gif,webp)")
|
||||
return errors.New("failed to decode image: unknown or unsupported image format (supported: jpg,png,gif,webp)")
|
||||
}
|
||||
|
||||
hash1, err := goimagehash.DifferenceHash(jpgdata)
|
||||
@@ -134,13 +136,9 @@ func (img *Image) Similar(hash *goimagehash.ImageHash) bool {
|
||||
|
||||
// check current image against all known hashes.
|
||||
func (img *Image) SimilarExists(cache Cache) bool {
|
||||
for _, otherimg := range cache {
|
||||
if img.Similar(otherimg) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
return slices.ContainsFunc(cache, func(otherimg *goimagehash.ImageHash) bool {
|
||||
return img.Similar(otherimg)
|
||||
})
|
||||
}
|
||||
|
||||
// read all JPG images in a ad directory, compute diff hashes and
|
||||
|
||||
2
main.go
2
main.go
@@ -49,6 +49,7 @@ func init() {
|
||||
fmt.Println("Please open a command shell and run it from there.")
|
||||
fmt.Println()
|
||||
fmt.Print("Press any key to quit: ")
|
||||
|
||||
_, err := bufio.NewReader(os.Stdin).ReadString('\n')
|
||||
if err != nil {
|
||||
panic(err)
|
||||
@@ -144,6 +145,7 @@ func Main(output io.Writer) int {
|
||||
if err != nil {
|
||||
return Die(err)
|
||||
}
|
||||
|
||||
conf.Outdir = outdir
|
||||
|
||||
// setup ad dir registry, needed to check for duplicates
|
||||
|
||||
41
main_test.go
41
main_test.go
@@ -405,27 +405,26 @@ func InitValidSources() []Adsource {
|
||||
empty := AdConfig{}
|
||||
|
||||
// prepare urls for the listing pages
|
||||
ads := []Adsource{
|
||||
{
|
||||
uri: fmt.Sprintf("%s%s?userId=1", Baseuri, Listuri),
|
||||
content: GetTemplate(list1, &empty, LISTTPL),
|
||||
},
|
||||
{
|
||||
uri: fmt.Sprintf("%s%s?userId=1&pageNum=2", Baseuri, Listuri),
|
||||
content: GetTemplate(list2, &empty, LISTTPL),
|
||||
},
|
||||
{
|
||||
uri: fmt.Sprintf("%s%s?userId=1&pageNum=3", Baseuri, Listuri),
|
||||
content: GetTemplate(list3, &empty, LISTTPL),
|
||||
},
|
||||
ads := make([]Adsource, 3+len(adsrc))
|
||||
ads[0] = Adsource{
|
||||
uri: fmt.Sprintf("%s%s?userId=1", Baseuri, Listuri),
|
||||
content: GetTemplate(list1, &empty, LISTTPL),
|
||||
}
|
||||
ads[1] = Adsource{
|
||||
uri: fmt.Sprintf("%s%s?userId=1&pageNum=2", Baseuri, Listuri),
|
||||
content: GetTemplate(list2, &empty, LISTTPL),
|
||||
}
|
||||
ads[2] = Adsource{
|
||||
uri: fmt.Sprintf("%s%s?userId=1&pageNum=3", Baseuri, Listuri),
|
||||
content: GetTemplate(list3, &empty, LISTTPL),
|
||||
}
|
||||
|
||||
// prepare urls for the ads
|
||||
for _, ad := range adsrc {
|
||||
ads = append(ads, Adsource{
|
||||
for idx, ad := range adsrc {
|
||||
ads[idx] = Adsource{
|
||||
uri: fmt.Sprintf("%s/s-anzeige/%s/%s", Baseuri, ad.Slug, ad.ID),
|
||||
content: GetTemplate(nil, &ad, ADTPL),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return ads
|
||||
@@ -436,7 +435,7 @@ func InitInvalidSources() []Adsource {
|
||||
ads := []Adsource{
|
||||
{
|
||||
// valid ad page but without content
|
||||
uri: fmt.Sprintf("%s/s-anzeige/empty/1", Baseuri),
|
||||
uri: Baseuri + "/s-anzeige/empty/1",
|
||||
content: GetTemplate(nil, &empty, EMPTYPAGE),
|
||||
},
|
||||
{
|
||||
@@ -446,18 +445,18 @@ func InitInvalidSources() []Adsource {
|
||||
},
|
||||
{
|
||||
// some invalid page path
|
||||
uri: fmt.Sprintf("%s/anzeige/name/1", Baseuri),
|
||||
uri: Baseuri + "/anzeige/name/1",
|
||||
content: GetTemplate(nil, &empty, "<html></html>"),
|
||||
},
|
||||
{
|
||||
// some none-ad page
|
||||
uri: fmt.Sprintf("%s/anzeige/name/1/foo/bar", Baseuri),
|
||||
uri: Baseuri + "/anzeige/name/1/foo/bar",
|
||||
content: GetTemplate(nil, &empty, "<html>HTTP 404: /eine-anzeige/ does not exist!</html>"),
|
||||
status: 404,
|
||||
},
|
||||
{
|
||||
// valid ad page but 503
|
||||
uri: fmt.Sprintf("%s/s-anzeige/503/1", Baseuri),
|
||||
uri: Baseuri + "/s-anzeige/503/1",
|
||||
content: GetTemplate(nil, &empty, "<html>HTTP 503: service unavailable</html>"),
|
||||
status: 503,
|
||||
},
|
||||
@@ -531,6 +530,7 @@ func TestMain(t *testing.T) {
|
||||
defer func() { os.Args = oldargs }()
|
||||
|
||||
httpmock.Activate()
|
||||
|
||||
defer httpmock.DeactivateAndReset()
|
||||
|
||||
// prepare httpmock responders
|
||||
@@ -568,6 +568,7 @@ func TestMainInvalids(t *testing.T) {
|
||||
defer func() { os.Args = oldargs }()
|
||||
|
||||
httpmock.Activate()
|
||||
|
||||
defer httpmock.DeactivateAndReset()
|
||||
|
||||
// prepare httpmock responders
|
||||
|
||||
@@ -19,6 +19,7 @@ package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
@@ -63,6 +64,7 @@ func ScrapeUser(conf *Config) error {
|
||||
panic(err)
|
||||
}
|
||||
}()
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -139,6 +141,7 @@ func ScrapeAd(conf *Config, uri string) error {
|
||||
panic(err)
|
||||
}
|
||||
}()
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -156,7 +159,7 @@ func ScrapeAd(conf *Config, uri string) error {
|
||||
if advertisement.Incomplete() {
|
||||
slog.Debug("got ad", "ad", advertisement)
|
||||
|
||||
return fmt.Errorf("could not extract ad data from page, got empty struct")
|
||||
return errors.New("could not extract ad data from page, got empty struct")
|
||||
}
|
||||
|
||||
advertisement.DecodeAttributes()
|
||||
@@ -206,7 +209,7 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error {
|
||||
|
||||
for _, imguri := range advertisement.Images {
|
||||
// we append the suffix later in NewImage() based on image format
|
||||
basefilename := filepath.Join(adpath, fmt.Sprintf("%d", img))
|
||||
basefilename := filepath.Join(adpath, strconv.Itoa(img))
|
||||
|
||||
egroup.Go(func() error {
|
||||
// wait a little
|
||||
@@ -259,6 +262,7 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error {
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
img++
|
||||
}
|
||||
|
||||
|
||||
4
store.go
4
store.go
@@ -127,7 +127,6 @@ func WriteImage(filename string, reader *bytes.Reader) error {
|
||||
}()
|
||||
|
||||
_, err = reader.WriteTo(file)
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to write to image file: %w", err)
|
||||
}
|
||||
@@ -157,7 +156,6 @@ func ReadImage(filename string) (*bytes.Buffer, error) {
|
||||
|
||||
func fileExists(filename string) bool {
|
||||
info, err := os.Stat(filename)
|
||||
|
||||
if err != nil {
|
||||
// return false on any error
|
||||
return false
|
||||
@@ -172,11 +170,13 @@ func CheckAdVisited(conf *Config, adname string) bool {
|
||||
if Exists(DirsVisited, adname) {
|
||||
if conf.ForceDownload {
|
||||
slog.Warn("an ad with the same name has already been downloaded, overwriting", "addir", adname)
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// don't overwrite
|
||||
slog.Warn("an ad with the same name has already been downloaded, skipping (use -f to overwrite)", "addir", adname)
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user