fix linter errors, enhance error handling, rename Id to ID in tpls

This commit is contained in:
2024-01-25 18:59:20 +01:00
parent bebcd15ada
commit 39269d3790
16 changed files with 143 additions and 106 deletions

View File

@@ -62,7 +62,7 @@ lint:
golangci-lint run golangci-lint run
lint-full: lint-full:
golangci-lint run --enable-all --exclude-use-default --disable exhaustivestruct,exhaustruct,depguard,interfacer,deadcode,golint,structcheck,scopelint,varcheck,ifshort,maligned,nosnakecase,godot,funlen,gofumpt,cyclop golangci-lint run --enable-all --exclude-use-default --disable exhaustivestruct,exhaustruct,depguard,interfacer,deadcode,golint,structcheck,scopelint,varcheck,ifshort,maligned,nosnakecase,godot,funlen,gofumpt,cyclop,noctx,gochecknoglobals,paralleltest
testfuzzy: clean testfuzzy: clean
go test -fuzz ./... $(ARGS) go test -fuzz ./... $(ARGS)

6
ad.go
View File

@@ -30,7 +30,7 @@ type Index struct {
type Ad struct { type Ad struct {
Title string `goquery:"h1"` Title string `goquery:"h1"`
Slug string Slug string
Id string ID string
Condition string `goquery:".addetailslist--detail--value,text"` Condition string `goquery:".addetailslist--detail--value,text"`
Category string Category string
CategoryTree []string `goquery:".breadcrump-link,text"` CategoryTree []string `goquery:".breadcrump-link,text"`
@@ -46,7 +46,7 @@ func (ad *Ad) LogValue() slog.Value {
return slog.GroupValue( return slog.GroupValue(
slog.String("title", ad.Title), slog.String("title", ad.Title),
slog.String("price", ad.Price), slog.String("price", ad.Price),
slog.String("id", ad.Id), slog.String("id", ad.ID),
slog.Int("imagecount", len(ad.Images)), slog.Int("imagecount", len(ad.Images)),
slog.Int("bodysize", len(ad.Text)), slog.Int("bodysize", len(ad.Text)),
slog.String("categorytree", strings.Join(ad.CategoryTree, "+")), slog.String("categorytree", strings.Join(ad.CategoryTree, "+")),
@@ -76,7 +76,7 @@ func (ad *Ad) CalculateExpire() {
if len(ad.Created) > 0 { if len(ad.Created) > 0 {
ts, err := time.Parse("02.01.2006", ad.Created) ts, err := time.Parse("02.01.2006", ad.Created)
if err == nil { if err == nil {
ad.Expire = ts.AddDate(0, 2, 1).Format("02.01.2006") ad.Expire = ts.AddDate(0, ExpireMonths, ExpireDays).Format("02.01.2006")
} }
} }
} }

View File

@@ -34,16 +34,16 @@ import (
) )
const ( const (
VERSION string = "0.3.1" VERSION string = "0.3.2"
Baseuri string = "https://www.kleinanzeigen.de" Baseuri string = "https://www.kleinanzeigen.de"
Listuri string = "/s-bestandsliste.html" Listuri string = "/s-bestandsliste.html"
Defaultdir string = "." Defaultdir string = "."
DefaultTemplate string = "Title: {{.Title}}\nPrice: {{.Price}}\nId: {{.Id}}\n" + DefaultTemplate string = "Title: {{.Title}}\nPrice: {{.Price}}\nId: {{.ID}}\n" +
"Category: {{.Category}}\nCondition: {{.Condition}}\n" + "Category: {{.Category}}\nCondition: {{.Condition}}\n" +
"Created: {{.Created}}\nExpire: {{.Expire}}\n\n{{.Text}}\n" "Created: {{.Created}}\nExpire: {{.Expire}}\n\n{{.Text}}\n"
DefaultTemplateWin string = "Title: {{.Title}}\r\nPrice: {{.Price}}\r\nId: {{.Id}}\r\n" + DefaultTemplateWin string = "Title: {{.Title}}\r\nPrice: {{.Price}}\r\nId: {{.ID}}\r\n" +
"Category: {{.Category}}\r\nCondition: {{.Condition}}\r\n" + "Category: {{.Category}}\r\nCondition: {{.Condition}}\r\n" +
"Created: {{.Created}}\r\nExpires: {{.Expire}}\r\n\r\n{{.Text}}\r\n" "Created: {{.Created}}\r\nExpires: {{.Expire}}\r\n\r\n{{.Text}}\r\n"
@@ -57,7 +57,12 @@ const (
MaxThrottle int = 20 MaxThrottle int = 20
// we extract the slug from the uri // we extract the slug from the uri
SlugUriPartNum int = 6 SlugURIPartNum int = 6
ExpireMonths int = 2
ExpireDays int = 1
WIN string = "windows"
) )
const Usage string = `This is kleingebaeck, the kleinanzeigen.de backup tool. const Usage string = `This is kleingebaeck, the kleinanzeigen.de backup tool.
@@ -114,7 +119,7 @@ func InitConfig(output io.Writer) (*Config, error) {
// determine template based on os // determine template based on os
template := DefaultTemplate template := DefaultTemplate
if runtime.GOOS == "windows" { if runtime.GOOS == WIN {
template = DefaultTemplateWin template = DefaultTemplateWin
} }

View File

@@ -52,7 +52,7 @@ func NewFetcher(conf *Config) (*Fetcher, error) {
} }
func (f *Fetcher) Get(uri string) (io.ReadCloser, error) { func (f *Fetcher) Get(uri string) (io.ReadCloser, error) {
req, err := http.NewRequest("GET", uri, nil) req, err := http.NewRequest(http.MethodGet, uri, nil)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to create a new HTTP request obj: %w", err) return nil, fmt.Errorf("failed to create a new HTTP request obj: %w", err)
} }
@@ -61,10 +61,12 @@ func (f *Fetcher) Get(uri string) (io.ReadCloser, error) {
if len(f.Cookies) > 0 { if len(f.Cookies) > 0 {
uriobj, _ := url.Parse(Baseuri) uriobj, _ := url.Parse(Baseuri)
slog.Debug("have cookies, sending them", slog.Debug("have cookies, sending them",
"sample-cookie-name", f.Cookies[0].Name, "sample-cookie-name", f.Cookies[0].Name,
"sample-cookie-expire", f.Cookies[0].Expires, "sample-cookie-expire", f.Cookies[0].Expires,
) )
f.Client.Jar.SetCookies(uriobj, f.Cookies) f.Client.Jar.SetCookies(uriobj, f.Cookies)
} }
@@ -73,7 +75,7 @@ func (f *Fetcher) Get(uri string) (io.ReadCloser, error) {
return nil, fmt.Errorf("failed to initiate HTTP request to %s: %w", uri, err) return nil, fmt.Errorf("failed to initiate HTTP request to %s: %w", uri, err)
} }
if res.StatusCode != 200 { if res.StatusCode != http.StatusOK {
return nil, errors.New("could not get page via HTTP") return nil, errors.New("could not get page via HTTP")
} }
@@ -86,12 +88,15 @@ func (f *Fetcher) Get(uri string) (io.ReadCloser, error) {
// fetch an image // fetch an image
func (f *Fetcher) Getimage(uri string) (io.ReadCloser, error) { func (f *Fetcher) Getimage(uri string) (io.ReadCloser, error) {
slog.Debug("fetching ad image", "uri", uri) slog.Debug("fetching ad image", "uri", uri)
body, err := f.Get(uri) body, err := f.Get(uri)
if err != nil { if err != nil {
if f.Config.IgnoreErrors { if f.Config.IgnoreErrors {
slog.Info("Failed to download image, error ignored", "error", err.Error()) slog.Info("Failed to download image, error ignored", "error", err.Error())
return nil, nil return nil, nil
} }
return nil, err return nil, err
} }

30
http.go
View File

@@ -33,17 +33,20 @@ import (
// easier associated in debug output // easier associated in debug output
var letters = []rune("ABCDEF0123456789") var letters = []rune("ABCDEF0123456789")
func getid() string { const IDLEN int = 8
b := make([]rune, 8)
for i := range b {
b[i] = letters[rand.Intn(len(letters))]
}
return string(b)
}
// retry after HTTP 50x errors or err!=nil // retry after HTTP 50x errors or err!=nil
const RetryCount = 3 const RetryCount = 3
func getid() string {
b := make([]rune, IDLEN)
for i := range b {
b[i] = letters[rand.Intn(len(letters))]
}
return string(b)
}
// used to inject debug log and implement retries // used to inject debug log and implement retries
type loggingTransport struct{} type loggingTransport struct{}
@@ -76,6 +79,7 @@ func drainBody(resp *http.Response) {
// unable to copy data? uff! // unable to copy data? uff!
panic(err) panic(err)
} }
resp.Body.Close() resp.Body.Close()
} }
} }
@@ -83,8 +87,8 @@ func drainBody(resp *http.Response) {
// the actual logging transport with retries // the actual logging transport with retries
func (t *loggingTransport) RoundTrip(req *http.Request) (*http.Response, error) { func (t *loggingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
// just requred for debugging // just required for debugging
id := getid() requestid := getid()
// clone the request body, put into request on retry // clone the request body, put into request on retry
var bodyBytes []byte var bodyBytes []byte
@@ -93,16 +97,16 @@ func (t *loggingTransport) RoundTrip(req *http.Request) (*http.Response, error)
req.Body = io.NopCloser(bytes.NewBuffer(bodyBytes)) req.Body = io.NopCloser(bytes.NewBuffer(bodyBytes))
} }
slog.Debug("REQUEST", "id", id, "uri", req.URL, "host", req.Host) slog.Debug("REQUEST", "id", requestid, "uri", req.URL, "host", req.Host)
// first try // first try
resp, err := http.DefaultTransport.RoundTrip(req) resp, err := http.DefaultTransport.RoundTrip(req)
if err == nil { if err == nil {
slog.Debug("RESPONSE", "id", id, "status", resp.StatusCode, slog.Debug("RESPONSE", "id", requestid, "status", resp.StatusCode,
"contentlength", resp.ContentLength) "contentlength", resp.ContentLength)
} }
// enter retry check and loop, if first req were successfull, leave loop immediately // enter retry check and loop, if first req were successful, leave loop immediately
retries := 0 retries := 0
for shouldRetry(err, resp) && retries < RetryCount { for shouldRetry(err, resp) && retries < RetryCount {
time.Sleep(backoff(retries)) time.Sleep(backoff(retries))
@@ -119,7 +123,7 @@ func (t *loggingTransport) RoundTrip(req *http.Request) (*http.Response, error)
resp, err = http.DefaultTransport.RoundTrip(req) resp, err = http.DefaultTransport.RoundTrip(req)
if err == nil { if err == nil {
slog.Debug("RESPONSE", "id", id, "status", resp.StatusCode, slog.Debug("RESPONSE", "id", requestid, "status", resp.StatusCode,
"contentlength", resp.ContentLength, "retry", retries) "contentlength", resp.ContentLength, "retry", retries)
} }

View File

@@ -133,7 +133,7 @@
.\" ======================================================================== .\" ========================================================================
.\" .\"
.IX Title "KLEINGEBAECK 1" .IX Title "KLEINGEBAECK 1"
.TH KLEINGEBAECK 1 "2024-01-24" "1" "User Commands" .TH KLEINGEBAECK 1 "2024-01-25" "1" "User Commands"
.\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" For nroff, turn off justification. Always turn off hyphenation; it makes
.\" way too many mistakes in technical documents. .\" way too many mistakes in technical documents.
.if n .ad l .if n .ad l
@@ -182,7 +182,7 @@ Format is pretty simple:
\& template = """ \& template = """
\& Title: {{.Title}} \& Title: {{.Title}}
\& Price: {{.Price}} \& Price: {{.Price}}
\& Id: {{.Id}} \& Id: {{.ID}}
\& Category: {{.Category}} \& Category: {{.Category}}
\& Condition: {{.Condition}} \& Condition: {{.Condition}}
\& Created: {{.Created}} \& Created: {{.Created}}
@@ -191,7 +191,7 @@ Format is pretty simple:
\& """ \& """
.Ve .Ve
.PP .PP
Be carefull if you want to change the template. The variable is a Be careful if you want to change the template. The variable is a
multiline string surrounded by three double quotes. You can left out multiline string surrounded by three double quotes. You can left out
certain fields and use any formatting you like. Refer to certain fields and use any formatting you like. Refer to
<https://pkg.go.dev/text/template> for details how to write a <https://pkg.go.dev/text/template> for details how to write a

View File

@@ -43,7 +43,7 @@ CONFIGURATION
template = """ template = """
Title: {{.Title}} Title: {{.Title}}
Price: {{.Price}} Price: {{.Price}}
Id: {{.Id}} Id: {{.ID}}
Category: {{.Category}} Category: {{.Category}}
Condition: {{.Condition}} Condition: {{.Condition}}
Created: {{.Created}} Created: {{.Created}}
@@ -51,7 +51,7 @@ CONFIGURATION
{{.Text}} {{.Text}}
""" """
Be carefull if you want to change the template. The variable is a Be careful if you want to change the template. The variable is a
multiline string surrounded by three double quotes. You can left out multiline string surrounded by three double quotes. You can left out
certain fields and use any formatting you like. Refer to certain fields and use any formatting you like. Refer to
<https://pkg.go.dev/text/template> for details how to write a template. <https://pkg.go.dev/text/template> for details how to write a template.

View File

@@ -43,7 +43,7 @@ Format is pretty simple:
template = """ template = """
Title: {{.Title}} Title: {{.Title}}
Price: {{.Price}} Price: {{.Price}}
Id: {{.Id}} Id: {{.ID}}
Category: {{.Category}} Category: {{.Category}}
Condition: {{.Condition}} Condition: {{.Condition}}
Created: {{.Created}} Created: {{.Created}}
@@ -51,7 +51,7 @@ Format is pretty simple:
{{.Text}} {{.Text}}
""" """
Be carefull if you want to change the template. The variable is a Be careful if you want to change the template. The variable is a
multiline string surrounded by three double quotes. You can left out multiline string surrounded by three double quotes. You can left out
certain fields and use any formatting you like. Refer to certain fields and use any formatting you like. Refer to
L<https://pkg.go.dev/text/template> for details how to write a L<https://pkg.go.dev/text/template> for details how to write a

39
main.go
View File

@@ -35,38 +35,43 @@ func main() {
os.Exit(Main(os.Stdout)) os.Exit(Main(os.Stdout))
} }
func Main(w io.Writer) int { func Main(output io.Writer) int {
logLevel := &slog.LevelVar{} logLevel := &slog.LevelVar{}
opts := &tint.Options{ opts := &tint.Options{
Level: logLevel, Level: logLevel,
AddSource: false, AddSource: false,
ReplaceAttr: func(groups []string, a slog.Attr) slog.Attr { ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
// Remove time from the output // Remove time from the output
if a.Key == slog.TimeKey { if attr.Key == slog.TimeKey {
return slog.Attr{} return slog.Attr{}
} }
return a
return attr
}, },
NoColor: IsNoTty(), NoColor: IsNoTty(),
} }
logLevel.Set(LevelNotice) logLevel.Set(LevelNotice)
handler := tint.NewHandler(w, opts)
handler := tint.NewHandler(output, opts)
logger := slog.New(handler) logger := slog.New(handler)
slog.SetDefault(logger) slog.SetDefault(logger)
conf, err := InitConfig(w) conf, err := InitConfig(output)
if err != nil { if err != nil {
return Die(err) return Die(err)
} }
if conf.Showversion { if conf.Showversion {
fmt.Fprintf(w, "This is kleingebaeck version %s\n", VERSION) fmt.Fprintf(output, "This is kleingebaeck version %s\n", VERSION)
return 0 return 0
} }
if conf.Showhelp { if conf.Showhelp {
fmt.Fprintln(w, Usage) fmt.Fprintln(output, Usage)
return 0 return 0
} }
@@ -75,6 +80,7 @@ func Main(w io.Writer) int {
if err != nil { if err != nil {
return Die(err) return Die(err)
} }
return 0 return 0
} }
@@ -92,7 +98,8 @@ func Main(w io.Writer) int {
} }
logLevel.Set(slog.LevelDebug) logLevel.Set(slog.LevelDebug)
handler := yadu.NewHandler(w, opts)
handler := yadu.NewHandler(output, opts)
debuglogger := slog.New(handler).With( debuglogger := slog.New(handler).With(
slog.Group("program_info", slog.Group("program_info",
slog.Int("pid", os.Getpid()), slog.Int("pid", os.Getpid()),
@@ -116,7 +123,8 @@ func Main(w io.Writer) int {
return Die(err) return Die(err)
} }
if len(conf.Adlinks) >= 1 { switch {
case len(conf.Adlinks) >= 1:
// directly backup ad listing[s] // directly backup ad listing[s]
for _, uri := range conf.Adlinks { for _, uri := range conf.Adlinks {
err := ScrapeAd(fetch, uri) err := ScrapeAd(fetch, uri)
@@ -124,25 +132,27 @@ func Main(w io.Writer) int {
return Die(err) return Die(err)
} }
} }
} else if conf.User > 0 { case conf.User > 0:
// backup all ads of the given user (via config or cmdline) // backup all ads of the given user (via config or cmdline)
err := ScrapeUser(fetch) err := ScrapeUser(fetch)
if err != nil { if err != nil {
return Die(err) return Die(err)
} }
} else { default:
return Die(errors.New("invalid or no user id or no ad link specified")) return Die(errors.New("invalid or no user id or no ad link specified"))
} }
if conf.StatsCountAds > 0 { if conf.StatsCountAds > 0 {
adstr := "ads" adstr := "ads"
if conf.StatsCountAds == 1 { if conf.StatsCountAds == 1 {
adstr = "ad" adstr = "ad"
} }
fmt.Fprintf(w, "Successfully downloaded %d %s with %d images to %s.\n",
fmt.Fprintf(output, "Successfully downloaded %d %s with %d images to %s.\n",
conf.StatsCountAds, adstr, conf.StatsCountImages, conf.Outdir) conf.StatsCountAds, adstr, conf.StatsCountImages, conf.Outdir)
} else { } else {
fmt.Fprintf(w, "No ads found.") fmt.Fprintf(output, "No ads found.")
} }
return 0 return 0
@@ -150,5 +160,6 @@ func Main(w io.Writer) int {
func Die(err error) int { func Die(err error) int {
slog.Error("Failure", "error", err.Error()) slog.Error("Failure", "error", err.Error())
return 1 return 1
} }

View File

@@ -43,7 +43,7 @@ const LISTTPL string = `<!DOCTYPE html>
{{ range . }} {{ range . }}
<h2 class="text-module-begin"> <h2 class="text-module-begin">
<a class="ellipsis" <a class="ellipsis"
href="/s-anzeige/{{ .Slug }}/{{ .Id }}">{{ .Title }}</a> href="/s-anzeige/{{ .Slug }}/{{ .ID }}">{{ .Title }}</a>
</h2> </h2>
{{ end }} {{ end }}
</body> </body>
@@ -247,7 +247,7 @@ var invalidtests = []Tests{
type AdConfig struct { type AdConfig struct {
Title string Title string
Slug string Slug string
Id string ID string
Price string Price string
Category string Category string
Condition string Condition string
@@ -259,7 +259,7 @@ type AdConfig struct {
var adsrc = []AdConfig{ var adsrc = []AdConfig{
{ {
Title: "First Ad", Title: "First Ad",
Id: "1", Price: "5€", ID: "1", Price: "5€",
Category: "Klimbim", Category: "Klimbim",
Text: "Thing to sale", Text: "Thing to sale",
Slug: "first-ad", Slug: "first-ad",
@@ -269,7 +269,7 @@ var adsrc = []AdConfig{
}, },
{ {
Title: "Secnd Ad", Title: "Secnd Ad",
Id: "2", Price: "5€", ID: "2", Price: "5€",
Category: "Kram", Category: "Kram",
Text: "Thing to sale", Text: "Thing to sale",
Slug: "second-ad", Slug: "second-ad",
@@ -279,7 +279,7 @@ var adsrc = []AdConfig{
}, },
{ {
Title: "Third Ad", Title: "Third Ad",
Id: "3", ID: "3",
Price: "5€", Price: "5€",
Category: "Kuddelmuddel", Category: "Kuddelmuddel",
Text: "Thing to sale", Text: "Thing to sale",
@@ -290,7 +290,7 @@ var adsrc = []AdConfig{
}, },
{ {
Title: "Forth Ad", Title: "Forth Ad",
Id: "4", ID: "4",
Price: "5€", Price: "5€",
Category: "Krempel", Category: "Krempel",
Text: "Thing to sale", Text: "Thing to sale",
@@ -301,7 +301,7 @@ var adsrc = []AdConfig{
}, },
{ {
Title: "Fifth Ad", Title: "Fifth Ad",
Id: "5", ID: "5",
Price: "5€", Price: "5€",
Category: "Kladderadatsch", Category: "Kladderadatsch",
Text: "Thing to sale", Text: "Thing to sale",
@@ -312,7 +312,7 @@ var adsrc = []AdConfig{
}, },
{ {
Title: "Sixth Ad", Title: "Sixth Ad",
Id: "6", ID: "6",
Price: "5€", Price: "5€",
Category: "Klunker", Category: "Klunker",
Text: "Thing to sale", Text: "Thing to sale",
@@ -334,17 +334,17 @@ type Adsource struct {
} }
// Render a HTML template for an adlisting or an ad // Render a HTML template for an adlisting or an ad
func GetTemplate(l []AdConfig, a AdConfig, htmltemplate string) string { func GetTemplate(adconfigs []AdConfig, adconfig AdConfig, htmltemplate string) string {
tmpl, err := tpl.New("template").Parse(htmltemplate) tmpl, err := tpl.New("template").Parse(htmltemplate)
if err != nil { if err != nil {
panic(err) panic(err)
} }
var out bytes.Buffer var out bytes.Buffer
if len(a.Id) == 0 { if len(adconfig.ID) == 0 {
err = tmpl.Execute(&out, l) err = tmpl.Execute(&out, adconfigs)
} else { } else {
err = tmpl.Execute(&out, a) err = tmpl.Execute(&out, adconfig)
} }
if err != nil { if err != nil {
@@ -391,10 +391,9 @@ func InitValidSources() []Adsource {
// prepare urls for the ads // prepare urls for the ads
for _, ad := range adsrc { for _, ad := range adsrc {
ads = append(ads, Adsource{ ads = append(ads, Adsource{
uri: fmt.Sprintf("%s/s-anzeige/%s/%s", Baseuri, ad.Slug, ad.Id), uri: fmt.Sprintf("%s/s-anzeige/%s/%s", Baseuri, ad.Slug, ad.ID),
content: GetTemplate(nil, ad, ADTPL), content: GetTemplate(nil, ad, ADTPL),
}) })
//panic(GetTemplate(nil, ad, ADTPL))
} }
return ads return ads
@@ -447,46 +446,48 @@ func GetImage(path string) []byte {
// setup httpmock // setup httpmock
func SetIntercept(ads []Adsource) { func SetIntercept(ads []Adsource) {
ch := http.Header{} headers := http.Header{}
ch.Add("Set-Cookie", "session=permanent") headers.Add("Set-Cookie", "session=permanent")
for _, ad := range ads { for _, advertisement := range ads {
if ad.status == 0 { if advertisement.status == 0 {
ad.status = 200 advertisement.status = 200
} }
httpmock.RegisterResponder("GET", ad.uri, httpmock.RegisterResponder("GET", advertisement.uri,
httpmock.NewStringResponder(ad.status, ad.content).HeaderAdd(ch)) httpmock.NewStringResponder(advertisement.status, advertisement.content).HeaderAdd(headers))
} }
// we just use 2 images, put this here // we just use 2 images, put this here
for _, image := range []string{"t/1.jpg", "t/2.jpg"} { for _, image := range []string{"t/1.jpg", "t/2.jpg"} {
httpmock.RegisterResponder("GET", image, httpmock.RegisterResponder("GET", image,
httpmock.NewBytesResponder(200, GetImage(image)).HeaderAdd(ch)) httpmock.NewBytesResponder(200, GetImage(image)).HeaderAdd(headers))
} }
} }
func VerifyAd(ad AdConfig) error { func VerifyAd(advertisement AdConfig) error {
body := ad.Title + ad.Price + ad.Id + "Kleinanzeigen => " + body := advertisement.Title + advertisement.Price + advertisement.ID + "Kleinanzeigen => " +
ad.Category + ad.Condition + ad.Created advertisement.Category + advertisement.Condition + advertisement.Created
// prepare ad dir name using DefaultAdNameTemplate // prepare ad dir name using DefaultAdNameTemplate
c := Config{Adnametemplate: "{{ .Slug }}"} c := Config{Adnametemplate: "{{ .Slug }}"}
adstruct := Ad{Slug: ad.Slug, Id: ad.Id} adstruct := Ad{Slug: advertisement.Slug, ID: advertisement.ID}
addir, err := AdDirName(&c, &adstruct) addir, err := AdDirName(&c, &adstruct)
if err != nil { if err != nil {
return err return err
} }
file := fmt.Sprintf("t/out/%s/Adlisting.txt", addir) file := fmt.Sprintf("t/out/%s/Adlisting.txt", addir)
content, err := os.ReadFile(file) content, err := os.ReadFile(file)
if err != nil { if err != nil {
return err return fmt.Errorf("unable to read adlisting file: %w", err)
} }
if body != strings.TrimSpace(string(content)) { if body != strings.TrimSpace(string(content)) {
msg := fmt.Sprintf("ad content doesn't match.\nExpect: %s\n Got: %s\n", body, content) msg := fmt.Sprintf("ad content doesn't match.\nExpect: %s\n Got: %s\n", body, content)
return errors.New(msg) return errors.New(msg)
} }
@@ -504,20 +505,21 @@ func TestMain(t *testing.T) {
SetIntercept(InitValidSources()) SetIntercept(InitValidSources())
// run commandline tests // run commandline tests
for _, tt := range tests { for _, test := range tests {
var buf bytes.Buffer var buf bytes.Buffer
os.Args = strings.Split(tt.args, " ")
os.Args = strings.Split(test.args, " ")
ret := Main(&buf) ret := Main(&buf)
if ret != tt.exitcode { if ret != test.exitcode {
t.Errorf("%s with cmd <%s> did not exit with %d but %d", t.Errorf("%s with cmd <%s> did not exit with %d but %d",
tt.name, tt.args, tt.exitcode, ret) test.name, test.args, test.exitcode, ret)
} }
if !strings.Contains(buf.String(), tt.expect) { if !strings.Contains(buf.String(), test.expect) {
t.Errorf("%s with cmd <%s> output did not match.\nExpect: %s\n Got: %s\n", t.Errorf("%s with cmd <%s> output did not match.\nExpect: %s\n Got: %s\n",
tt.name, tt.args, tt.expect, buf.String()) test.name, test.args, test.expect, buf.String())
} }
} }
@@ -540,20 +542,21 @@ func TestMainInvalids(t *testing.T) {
SetIntercept(InitInvalidSources()) SetIntercept(InitInvalidSources())
// run commandline tests // run commandline tests
for _, tt := range invalidtests { for _, test := range invalidtests {
var buf bytes.Buffer var buf bytes.Buffer
os.Args = strings.Split(tt.args, " ")
os.Args = strings.Split(test.args, " ")
ret := Main(&buf) ret := Main(&buf)
if ret != tt.exitcode { if ret != test.exitcode {
t.Errorf("%s with cmd <%s> did not exit with %d but %d", t.Errorf("%s with cmd <%s> did not exit with %d but %d",
tt.name, tt.args, tt.exitcode, ret) test.name, test.args, test.exitcode, ret)
} }
if !strings.Contains(buf.String(), tt.expect) { if !strings.Contains(buf.String(), test.expect) {
t.Errorf("%s with cmd <%s> output did not match.\nExpect: %s\n Got: %s\n", t.Errorf("%s with cmd <%s> output did not match.\nExpect: %s\n Got: %s\n",
tt.name, tt.args, tt.expect, buf.String()) test.name, test.args, test.expect, buf.String())
} }
} }
} }

View File

@@ -54,7 +54,7 @@ func ScrapeUser(fetch *Fetcher) error {
err = goq.NewDecoder(body).Decode(&index) err = goq.NewDecoder(body).Decode(&index)
if err != nil { if err != nil {
return err return fmt.Errorf("failed to goquery decode HTML index body: %w", err)
} }
if len(index.Links) == 0 { if len(index.Links) == 0 {
@@ -92,12 +92,12 @@ func ScrapeAd(fetch *Fetcher, uri string) error {
// extract slug and id from uri // extract slug and id from uri
uriparts := strings.Split(uri, "/") uriparts := strings.Split(uri, "/")
if len(uriparts) < SlugUriPartNum { if len(uriparts) < SlugURIPartNum {
return fmt.Errorf("invalid uri: %s", uri) return fmt.Errorf("invalid uri: %s", uri)
} }
advertisement.Slug = uriparts[4] advertisement.Slug = uriparts[4]
advertisement.Id = uriparts[5] advertisement.ID = uriparts[5]
// get the ad // get the ad
slog.Debug("fetching ad page", "uri", uri) slog.Debug("fetching ad page", "uri", uri)
@@ -111,7 +111,7 @@ func ScrapeAd(fetch *Fetcher, uri string) error {
// extract ad contents with goquery/goq // extract ad contents with goquery/goq
err = goq.NewDecoder(body).Decode(&advertisement) err = goq.NewDecoder(body).Decode(&advertisement)
if err != nil { if err != nil {
return fmt.Errorf("failed to goquery decode HTML body: %w", err) return fmt.Errorf("failed to goquery decode HTML ad body: %w", err)
} }
if len(advertisement.CategoryTree) > 0 { if len(advertisement.CategoryTree) > 0 {
@@ -120,6 +120,7 @@ func ScrapeAd(fetch *Fetcher, uri string) error {
if advertisement.Incomplete() { if advertisement.Incomplete() {
slog.Debug("got ad", "ad", advertisement) slog.Debug("got ad", "ad", advertisement)
return fmt.Errorf("could not extract ad data from page, got empty struct") return fmt.Errorf("could not extract ad data from page, got empty struct")
} }
@@ -183,6 +184,7 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error {
if !fetch.Config.ForceDownload { if !fetch.Config.ForceDownload {
if image.SimilarExists(cache) { if image.SimilarExists(cache) {
slog.Debug("similar image exists, not written", "uri", image.URI) slog.Debug("similar image exists, not written", "uri", image.URI)
return nil return nil
} }
} }
@@ -193,13 +195,14 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error {
} }
slog.Debug("wrote image", "image", image, "size", len(buf2), "throttle", throttle) slog.Debug("wrote image", "image", image, "size", len(buf2), "throttle", throttle)
return nil return nil
}) })
img++ img++
} }
if err := egroup.Wait(); err != nil { if err := egroup.Wait(); err != nil {
return err return fmt.Errorf("failed to finalize error waitgroup: %w", err)
} }
fetch.Config.IncrImgs(len(advertisement.Images)) fetch.Config.IncrImgs(len(advertisement.Images))

View File

@@ -28,14 +28,15 @@ import (
tpl "text/template" tpl "text/template"
) )
func AdDirName(c *Config, ad *Ad) (string, error) { func AdDirName(conf *Config, advertisement *Ad) (string, error) {
tmpl, err := tpl.New("adname").Parse(c.Adnametemplate) tmpl, err := tpl.New("adname").Parse(conf.Adnametemplate)
if err != nil { if err != nil {
return "", fmt.Errorf("failed to parse adname template: %w", err) return "", fmt.Errorf("failed to parse adname template: %w", err)
} }
buf := bytes.Buffer{} buf := bytes.Buffer{}
err = tmpl.Execute(&buf, ad)
err = tmpl.Execute(&buf, advertisement)
if err != nil { if err != nil {
return "", fmt.Errorf("failed to execute adname template: %w", err) return "", fmt.Errorf("failed to execute adname template: %w", err)
} }
@@ -43,15 +44,16 @@ func AdDirName(c *Config, ad *Ad) (string, error) {
return buf.String(), nil return buf.String(), nil
} }
func WriteAd(c *Config, ad *Ad) (string, error) { func WriteAd(conf *Config, advertisement *Ad) (string, error) {
// prepare ad dir name // prepare ad dir name
addir, err := AdDirName(c, ad) addir, err := AdDirName(conf, advertisement)
if err != nil { if err != nil {
return "", err return "", err
} }
// prepare output dir // prepare output dir
dir := filepath.Join(c.Outdir, addir) dir := filepath.Join(conf.Outdir, addir)
err = Mkdir(dir) err = Mkdir(dir)
if err != nil { if err != nil {
return "", err return "", err
@@ -59,24 +61,25 @@ func WriteAd(c *Config, ad *Ad) (string, error) {
// write ad file // write ad file
listingfile := filepath.Join(dir, "Adlisting.txt") listingfile := filepath.Join(dir, "Adlisting.txt")
listingfd, err := os.Create(listingfile) listingfd, err := os.Create(listingfile)
if err != nil { if err != nil {
return "", fmt.Errorf("failed to create Adlisting.txt: %w", err) return "", fmt.Errorf("failed to create Adlisting.txt: %w", err)
} }
defer listingfd.Close() defer listingfd.Close()
if runtime.GOOS == "windows" { if runtime.GOOS == WIN {
ad.Text = strings.ReplaceAll(ad.Text, "<br/>", "\r\n") advertisement.Text = strings.ReplaceAll(advertisement.Text, "<br/>", "\r\n")
} else { } else {
ad.Text = strings.ReplaceAll(ad.Text, "<br/>", "\n") advertisement.Text = strings.ReplaceAll(advertisement.Text, "<br/>", "\n")
} }
tmpl, err := tpl.New("adlisting").Parse(c.Template) tmpl, err := tpl.New("adlisting").Parse(conf.Template)
if err != nil { if err != nil {
return "", fmt.Errorf("failed to parse adlisting template: %w", err) return "", fmt.Errorf("failed to parse adlisting template: %w", err)
} }
err = tmpl.Execute(listingfd, ad) err = tmpl.Execute(listingfd, advertisement)
if err != nil { if err != nil {
return "", fmt.Errorf("failed to execute adlisting template: %w", err) return "", fmt.Errorf("failed to execute adlisting template: %w", err)
} }
@@ -127,5 +130,6 @@ func fileExists(filename string) bool {
if os.IsNotExist(err) { if os.IsNotExist(err) {
return false return false
} }
return !info.IsDir() return !info.IsDir()
} }

View File

@@ -26,6 +26,8 @@ import (
// doesn't show up in the coverage report for unknown reasons, so // doesn't show up in the coverage report for unknown reasons, so
// here's a single test for it // here's a single test for it
func TestWriteImage(t *testing.T) { func TestWriteImage(t *testing.T) {
t.Parallel()
buf := []byte{1, 2, 3, 4, 5, 6, 7, 8} buf := []byte{1, 2, 3, 4, 5, 6, 7, 8}
file := "t/out/t.jpg" file := "t/out/t.jpg"
@@ -33,5 +35,4 @@ func TestWriteImage(t *testing.T) {
if err != nil { if err != nil {
t.Errorf("Could not write mock image to %s: %s", file, err.Error()) t.Errorf("Could not write mock image to %s: %s", file, err.Error())
} }
} }

View File

@@ -1,6 +1,6 @@
# empty config for Main() unit tests to force unit tests NOT to use an # empty config for Main() unit tests to force unit tests NOT to use an
# eventually existing ~/.kleingebaeck! # eventually existing ~/.kleingebaeck!
template=""" template="""
{{.Title}}{{.Price}}{{.Id}}{{.Category}}{{.Condition}}{{.Created}} {{.Title}}{{.Price}}{{.ID}}{{.Category}}{{.Condition}}{{.Created}}
""" """

View File

@@ -2,5 +2,5 @@ user = 1
loglevel = "verbose" loglevel = "verbose"
outdir = "t/out" outdir = "t/out"
template=""" template="""
{{.Title}}{{.Price}}{{.Id}}{{.Category}}{{.Condition}}{{.Created}} {{.Title}}{{.Price}}{{.ID}}{{.Category}}{{.Condition}}{{.Created}}
""" """

View File

@@ -45,7 +45,8 @@ func man() error {
man := exec.Command("less", "-") man := exec.Command("less", "-")
var b bytes.Buffer var b bytes.Buffer
b.Write([]byte(manpage))
b.WriteString(manpage)
man.Stdout = os.Stdout man.Stdout = os.Stdout
man.Stdin = &b man.Stdin = &b
@@ -62,7 +63,7 @@ func man() error {
// returns TRUE if stdout is NOT a tty or windows // returns TRUE if stdout is NOT a tty or windows
func IsNoTty() bool { func IsNoTty() bool {
if runtime.GOOS == "windows" || !isatty.IsTerminal(os.Stdout.Fd()) { if runtime.GOOS == WIN || !isatty.IsTerminal(os.Stdout.Fd()) {
return true return true
} }