scip/kleingebaeck
1
0
Fork 0
mirror of https://codeberg.org/scip/kleingebaeck.git synced 2025-12-16 12:01:00 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add HTTP retries and the possibility to ignore image download errors (#33)

Browse Source 
added HTTP retry and --ignoreerrors which ignores image download errors, fix #30
This commit is contained in:
T.v.Dein
2024-01-16 13:18:15 +01:00
committed by Thomas von Dein
parent cca3211023
commit 5fa46ff106
9 changed files with 151 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
scrape.go
View File

@@ -44,9 +44,6 @@ func Get(uri string, client *http.Client) (io.ReadCloser, error) {
return nil, err
}
slog.Debug("response", "code", res.StatusCode, "status",
res.Status, "size", res.ContentLength)
if res.StatusCode != 200 {
return nil, errors.New("could not get page via HTTP")
}
@@ -165,11 +162,10 @@ func ScrapeImages(c *Config, ad *Ad, addir string, client *http.Client) error {
imguri := imguri
file := filepath.Join(c.Outdir, addir, fmt.Sprintf("%d.jpg", img))
g.Go(func() error {
err := Getimage(imguri, file, client)
err := Getimage(c, imguri, file, client)
if err != nil {
return err
}
slog.Info("wrote ad image", "image", file)
return nil
})
@@ -186,10 +182,13 @@ func ScrapeImages(c *Config, ad *Ad, addir string, client *http.Client) error {
}
// fetch an image
func Getimage(uri, fileName string, client *http.Client) error {
func Getimage(c *Config, uri, fileName string, client *http.Client) error {
slog.Debug("fetching ad image", "uri", uri)
req, err := http.NewRequest("GET", uri, nil)
if err != nil {
if c.IgnoreErrors {
slog.Info("Failed to download image, error ignored", "error", err.Error())
}
return err
}
@@ -210,5 +209,6 @@ func Getimage(uri, fileName string, client *http.Client) error {
return err
}
slog.Info("wrote ad image", "image", fileName)
return nil
}