mirror of
https://codeberg.org/scip/kleingebaeck.git
synced 2025-12-16 12:01:00 +01:00
fix linter errors, enhance error handling, rename Id to ID in tpls
This commit is contained in:
2
Makefile
2
Makefile
@@ -62,7 +62,7 @@ lint:
|
||||
golangci-lint run
|
||||
|
||||
lint-full:
|
||||
golangci-lint run --enable-all --exclude-use-default --disable exhaustivestruct,exhaustruct,depguard,interfacer,deadcode,golint,structcheck,scopelint,varcheck,ifshort,maligned,nosnakecase,godot,funlen,gofumpt,cyclop
|
||||
golangci-lint run --enable-all --exclude-use-default --disable exhaustivestruct,exhaustruct,depguard,interfacer,deadcode,golint,structcheck,scopelint,varcheck,ifshort,maligned,nosnakecase,godot,funlen,gofumpt,cyclop,noctx,gochecknoglobals,paralleltest
|
||||
|
||||
testfuzzy: clean
|
||||
go test -fuzz ./... $(ARGS)
|
||||
|
||||
6
ad.go
6
ad.go
@@ -30,7 +30,7 @@ type Index struct {
|
||||
type Ad struct {
|
||||
Title string `goquery:"h1"`
|
||||
Slug string
|
||||
Id string
|
||||
ID string
|
||||
Condition string `goquery:".addetailslist--detail--value,text"`
|
||||
Category string
|
||||
CategoryTree []string `goquery:".breadcrump-link,text"`
|
||||
@@ -46,7 +46,7 @@ func (ad *Ad) LogValue() slog.Value {
|
||||
return slog.GroupValue(
|
||||
slog.String("title", ad.Title),
|
||||
slog.String("price", ad.Price),
|
||||
slog.String("id", ad.Id),
|
||||
slog.String("id", ad.ID),
|
||||
slog.Int("imagecount", len(ad.Images)),
|
||||
slog.Int("bodysize", len(ad.Text)),
|
||||
slog.String("categorytree", strings.Join(ad.CategoryTree, "+")),
|
||||
@@ -76,7 +76,7 @@ func (ad *Ad) CalculateExpire() {
|
||||
if len(ad.Created) > 0 {
|
||||
ts, err := time.Parse("02.01.2006", ad.Created)
|
||||
if err == nil {
|
||||
ad.Expire = ts.AddDate(0, 2, 1).Format("02.01.2006")
|
||||
ad.Expire = ts.AddDate(0, ExpireMonths, ExpireDays).Format("02.01.2006")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
15
config.go
15
config.go
@@ -34,16 +34,16 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
VERSION string = "0.3.1"
|
||||
VERSION string = "0.3.2"
|
||||
Baseuri string = "https://www.kleinanzeigen.de"
|
||||
Listuri string = "/s-bestandsliste.html"
|
||||
Defaultdir string = "."
|
||||
|
||||
DefaultTemplate string = "Title: {{.Title}}\nPrice: {{.Price}}\nId: {{.Id}}\n" +
|
||||
DefaultTemplate string = "Title: {{.Title}}\nPrice: {{.Price}}\nId: {{.ID}}\n" +
|
||||
"Category: {{.Category}}\nCondition: {{.Condition}}\n" +
|
||||
"Created: {{.Created}}\nExpire: {{.Expire}}\n\n{{.Text}}\n"
|
||||
|
||||
DefaultTemplateWin string = "Title: {{.Title}}\r\nPrice: {{.Price}}\r\nId: {{.Id}}\r\n" +
|
||||
DefaultTemplateWin string = "Title: {{.Title}}\r\nPrice: {{.Price}}\r\nId: {{.ID}}\r\n" +
|
||||
"Category: {{.Category}}\r\nCondition: {{.Condition}}\r\n" +
|
||||
"Created: {{.Created}}\r\nExpires: {{.Expire}}\r\n\r\n{{.Text}}\r\n"
|
||||
|
||||
@@ -57,7 +57,12 @@ const (
|
||||
MaxThrottle int = 20
|
||||
|
||||
// we extract the slug from the uri
|
||||
SlugUriPartNum int = 6
|
||||
SlugURIPartNum int = 6
|
||||
|
||||
ExpireMonths int = 2
|
||||
ExpireDays int = 1
|
||||
|
||||
WIN string = "windows"
|
||||
)
|
||||
|
||||
const Usage string = `This is kleingebaeck, the kleinanzeigen.de backup tool.
|
||||
@@ -114,7 +119,7 @@ func InitConfig(output io.Writer) (*Config, error) {
|
||||
|
||||
// determine template based on os
|
||||
template := DefaultTemplate
|
||||
if runtime.GOOS == "windows" {
|
||||
if runtime.GOOS == WIN {
|
||||
template = DefaultTemplateWin
|
||||
}
|
||||
|
||||
|
||||
9
fetch.go
9
fetch.go
@@ -52,7 +52,7 @@ func NewFetcher(conf *Config) (*Fetcher, error) {
|
||||
}
|
||||
|
||||
func (f *Fetcher) Get(uri string) (io.ReadCloser, error) {
|
||||
req, err := http.NewRequest("GET", uri, nil)
|
||||
req, err := http.NewRequest(http.MethodGet, uri, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create a new HTTP request obj: %w", err)
|
||||
}
|
||||
@@ -61,10 +61,12 @@ func (f *Fetcher) Get(uri string) (io.ReadCloser, error) {
|
||||
|
||||
if len(f.Cookies) > 0 {
|
||||
uriobj, _ := url.Parse(Baseuri)
|
||||
|
||||
slog.Debug("have cookies, sending them",
|
||||
"sample-cookie-name", f.Cookies[0].Name,
|
||||
"sample-cookie-expire", f.Cookies[0].Expires,
|
||||
)
|
||||
|
||||
f.Client.Jar.SetCookies(uriobj, f.Cookies)
|
||||
}
|
||||
|
||||
@@ -73,7 +75,7 @@ func (f *Fetcher) Get(uri string) (io.ReadCloser, error) {
|
||||
return nil, fmt.Errorf("failed to initiate HTTP request to %s: %w", uri, err)
|
||||
}
|
||||
|
||||
if res.StatusCode != 200 {
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return nil, errors.New("could not get page via HTTP")
|
||||
}
|
||||
|
||||
@@ -86,12 +88,15 @@ func (f *Fetcher) Get(uri string) (io.ReadCloser, error) {
|
||||
// fetch an image
|
||||
func (f *Fetcher) Getimage(uri string) (io.ReadCloser, error) {
|
||||
slog.Debug("fetching ad image", "uri", uri)
|
||||
|
||||
body, err := f.Get(uri)
|
||||
if err != nil {
|
||||
if f.Config.IgnoreErrors {
|
||||
slog.Info("Failed to download image, error ignored", "error", err.Error())
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return nil, err
|
||||
}
|
||||
|
||||
|
||||
30
http.go
30
http.go
@@ -33,17 +33,20 @@ import (
|
||||
// easier associated in debug output
|
||||
var letters = []rune("ABCDEF0123456789")
|
||||
|
||||
func getid() string {
|
||||
b := make([]rune, 8)
|
||||
for i := range b {
|
||||
b[i] = letters[rand.Intn(len(letters))]
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
const IDLEN int = 8
|
||||
|
||||
// retry after HTTP 50x errors or err!=nil
|
||||
const RetryCount = 3
|
||||
|
||||
func getid() string {
|
||||
b := make([]rune, IDLEN)
|
||||
for i := range b {
|
||||
b[i] = letters[rand.Intn(len(letters))]
|
||||
}
|
||||
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// used to inject debug log and implement retries
|
||||
type loggingTransport struct{}
|
||||
|
||||
@@ -76,6 +79,7 @@ func drainBody(resp *http.Response) {
|
||||
// unable to copy data? uff!
|
||||
panic(err)
|
||||
}
|
||||
|
||||
resp.Body.Close()
|
||||
}
|
||||
}
|
||||
@@ -83,8 +87,8 @@ func drainBody(resp *http.Response) {
|
||||
|
||||
// the actual logging transport with retries
|
||||
func (t *loggingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
// just requred for debugging
|
||||
id := getid()
|
||||
// just required for debugging
|
||||
requestid := getid()
|
||||
|
||||
// clone the request body, put into request on retry
|
||||
var bodyBytes []byte
|
||||
@@ -93,16 +97,16 @@ func (t *loggingTransport) RoundTrip(req *http.Request) (*http.Response, error)
|
||||
req.Body = io.NopCloser(bytes.NewBuffer(bodyBytes))
|
||||
}
|
||||
|
||||
slog.Debug("REQUEST", "id", id, "uri", req.URL, "host", req.Host)
|
||||
slog.Debug("REQUEST", "id", requestid, "uri", req.URL, "host", req.Host)
|
||||
|
||||
// first try
|
||||
resp, err := http.DefaultTransport.RoundTrip(req)
|
||||
if err == nil {
|
||||
slog.Debug("RESPONSE", "id", id, "status", resp.StatusCode,
|
||||
slog.Debug("RESPONSE", "id", requestid, "status", resp.StatusCode,
|
||||
"contentlength", resp.ContentLength)
|
||||
}
|
||||
|
||||
// enter retry check and loop, if first req were successfull, leave loop immediately
|
||||
// enter retry check and loop, if first req were successful, leave loop immediately
|
||||
retries := 0
|
||||
for shouldRetry(err, resp) && retries < RetryCount {
|
||||
time.Sleep(backoff(retries))
|
||||
@@ -119,7 +123,7 @@ func (t *loggingTransport) RoundTrip(req *http.Request) (*http.Response, error)
|
||||
resp, err = http.DefaultTransport.RoundTrip(req)
|
||||
|
||||
if err == nil {
|
||||
slog.Debug("RESPONSE", "id", id, "status", resp.StatusCode,
|
||||
slog.Debug("RESPONSE", "id", requestid, "status", resp.StatusCode,
|
||||
"contentlength", resp.ContentLength, "retry", retries)
|
||||
}
|
||||
|
||||
|
||||
@@ -133,7 +133,7 @@
|
||||
.\" ========================================================================
|
||||
.\"
|
||||
.IX Title "KLEINGEBAECK 1"
|
||||
.TH KLEINGEBAECK 1 "2024-01-24" "1" "User Commands"
|
||||
.TH KLEINGEBAECK 1 "2024-01-25" "1" "User Commands"
|
||||
.\" For nroff, turn off justification. Always turn off hyphenation; it makes
|
||||
.\" way too many mistakes in technical documents.
|
||||
.if n .ad l
|
||||
@@ -182,7 +182,7 @@ Format is pretty simple:
|
||||
\& template = """
|
||||
\& Title: {{.Title}}
|
||||
\& Price: {{.Price}}
|
||||
\& Id: {{.Id}}
|
||||
\& Id: {{.ID}}
|
||||
\& Category: {{.Category}}
|
||||
\& Condition: {{.Condition}}
|
||||
\& Created: {{.Created}}
|
||||
@@ -191,7 +191,7 @@ Format is pretty simple:
|
||||
\& """
|
||||
.Ve
|
||||
.PP
|
||||
Be carefull if you want to change the template. The variable is a
|
||||
Be careful if you want to change the template. The variable is a
|
||||
multiline string surrounded by three double quotes. You can left out
|
||||
certain fields and use any formatting you like. Refer to
|
||||
<https://pkg.go.dev/text/template> for details how to write a
|
||||
|
||||
@@ -43,7 +43,7 @@ CONFIGURATION
|
||||
template = """
|
||||
Title: {{.Title}}
|
||||
Price: {{.Price}}
|
||||
Id: {{.Id}}
|
||||
Id: {{.ID}}
|
||||
Category: {{.Category}}
|
||||
Condition: {{.Condition}}
|
||||
Created: {{.Created}}
|
||||
@@ -51,7 +51,7 @@ CONFIGURATION
|
||||
{{.Text}}
|
||||
"""
|
||||
|
||||
Be carefull if you want to change the template. The variable is a
|
||||
Be careful if you want to change the template. The variable is a
|
||||
multiline string surrounded by three double quotes. You can left out
|
||||
certain fields and use any formatting you like. Refer to
|
||||
<https://pkg.go.dev/text/template> for details how to write a template.
|
||||
|
||||
@@ -43,7 +43,7 @@ Format is pretty simple:
|
||||
template = """
|
||||
Title: {{.Title}}
|
||||
Price: {{.Price}}
|
||||
Id: {{.Id}}
|
||||
Id: {{.ID}}
|
||||
Category: {{.Category}}
|
||||
Condition: {{.Condition}}
|
||||
Created: {{.Created}}
|
||||
@@ -51,7 +51,7 @@ Format is pretty simple:
|
||||
{{.Text}}
|
||||
"""
|
||||
|
||||
Be carefull if you want to change the template. The variable is a
|
||||
Be careful if you want to change the template. The variable is a
|
||||
multiline string surrounded by three double quotes. You can left out
|
||||
certain fields and use any formatting you like. Refer to
|
||||
L<https://pkg.go.dev/text/template> for details how to write a
|
||||
|
||||
39
main.go
39
main.go
@@ -35,38 +35,43 @@ func main() {
|
||||
os.Exit(Main(os.Stdout))
|
||||
}
|
||||
|
||||
func Main(w io.Writer) int {
|
||||
func Main(output io.Writer) int {
|
||||
logLevel := &slog.LevelVar{}
|
||||
opts := &tint.Options{
|
||||
Level: logLevel,
|
||||
AddSource: false,
|
||||
ReplaceAttr: func(groups []string, a slog.Attr) slog.Attr {
|
||||
ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
|
||||
// Remove time from the output
|
||||
if a.Key == slog.TimeKey {
|
||||
if attr.Key == slog.TimeKey {
|
||||
return slog.Attr{}
|
||||
}
|
||||
return a
|
||||
|
||||
return attr
|
||||
},
|
||||
NoColor: IsNoTty(),
|
||||
}
|
||||
|
||||
logLevel.Set(LevelNotice)
|
||||
handler := tint.NewHandler(w, opts)
|
||||
|
||||
handler := tint.NewHandler(output, opts)
|
||||
logger := slog.New(handler)
|
||||
|
||||
slog.SetDefault(logger)
|
||||
|
||||
conf, err := InitConfig(w)
|
||||
conf, err := InitConfig(output)
|
||||
if err != nil {
|
||||
return Die(err)
|
||||
}
|
||||
|
||||
if conf.Showversion {
|
||||
fmt.Fprintf(w, "This is kleingebaeck version %s\n", VERSION)
|
||||
fmt.Fprintf(output, "This is kleingebaeck version %s\n", VERSION)
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
if conf.Showhelp {
|
||||
fmt.Fprintln(w, Usage)
|
||||
fmt.Fprintln(output, Usage)
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
@@ -75,6 +80,7 @@ func Main(w io.Writer) int {
|
||||
if err != nil {
|
||||
return Die(err)
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
@@ -92,7 +98,8 @@ func Main(w io.Writer) int {
|
||||
}
|
||||
|
||||
logLevel.Set(slog.LevelDebug)
|
||||
handler := yadu.NewHandler(w, opts)
|
||||
|
||||
handler := yadu.NewHandler(output, opts)
|
||||
debuglogger := slog.New(handler).With(
|
||||
slog.Group("program_info",
|
||||
slog.Int("pid", os.Getpid()),
|
||||
@@ -116,7 +123,8 @@ func Main(w io.Writer) int {
|
||||
return Die(err)
|
||||
}
|
||||
|
||||
if len(conf.Adlinks) >= 1 {
|
||||
switch {
|
||||
case len(conf.Adlinks) >= 1:
|
||||
// directly backup ad listing[s]
|
||||
for _, uri := range conf.Adlinks {
|
||||
err := ScrapeAd(fetch, uri)
|
||||
@@ -124,25 +132,27 @@ func Main(w io.Writer) int {
|
||||
return Die(err)
|
||||
}
|
||||
}
|
||||
} else if conf.User > 0 {
|
||||
case conf.User > 0:
|
||||
// backup all ads of the given user (via config or cmdline)
|
||||
err := ScrapeUser(fetch)
|
||||
if err != nil {
|
||||
return Die(err)
|
||||
}
|
||||
} else {
|
||||
default:
|
||||
return Die(errors.New("invalid or no user id or no ad link specified"))
|
||||
}
|
||||
|
||||
if conf.StatsCountAds > 0 {
|
||||
adstr := "ads"
|
||||
|
||||
if conf.StatsCountAds == 1 {
|
||||
adstr = "ad"
|
||||
}
|
||||
fmt.Fprintf(w, "Successfully downloaded %d %s with %d images to %s.\n",
|
||||
|
||||
fmt.Fprintf(output, "Successfully downloaded %d %s with %d images to %s.\n",
|
||||
conf.StatsCountAds, adstr, conf.StatsCountImages, conf.Outdir)
|
||||
} else {
|
||||
fmt.Fprintf(w, "No ads found.")
|
||||
fmt.Fprintf(output, "No ads found.")
|
||||
}
|
||||
|
||||
return 0
|
||||
@@ -150,5 +160,6 @@ func Main(w io.Writer) int {
|
||||
|
||||
func Die(err error) int {
|
||||
slog.Error("Failure", "error", err.Error())
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
83
main_test.go
83
main_test.go
@@ -43,7 +43,7 @@ const LISTTPL string = `<!DOCTYPE html>
|
||||
{{ range . }}
|
||||
<h2 class="text-module-begin">
|
||||
<a class="ellipsis"
|
||||
href="/s-anzeige/{{ .Slug }}/{{ .Id }}">{{ .Title }}</a>
|
||||
href="/s-anzeige/{{ .Slug }}/{{ .ID }}">{{ .Title }}</a>
|
||||
</h2>
|
||||
{{ end }}
|
||||
</body>
|
||||
@@ -247,7 +247,7 @@ var invalidtests = []Tests{
|
||||
type AdConfig struct {
|
||||
Title string
|
||||
Slug string
|
||||
Id string
|
||||
ID string
|
||||
Price string
|
||||
Category string
|
||||
Condition string
|
||||
@@ -259,7 +259,7 @@ type AdConfig struct {
|
||||
var adsrc = []AdConfig{
|
||||
{
|
||||
Title: "First Ad",
|
||||
Id: "1", Price: "5€",
|
||||
ID: "1", Price: "5€",
|
||||
Category: "Klimbim",
|
||||
Text: "Thing to sale",
|
||||
Slug: "first-ad",
|
||||
@@ -269,7 +269,7 @@ var adsrc = []AdConfig{
|
||||
},
|
||||
{
|
||||
Title: "Secnd Ad",
|
||||
Id: "2", Price: "5€",
|
||||
ID: "2", Price: "5€",
|
||||
Category: "Kram",
|
||||
Text: "Thing to sale",
|
||||
Slug: "second-ad",
|
||||
@@ -279,7 +279,7 @@ var adsrc = []AdConfig{
|
||||
},
|
||||
{
|
||||
Title: "Third Ad",
|
||||
Id: "3",
|
||||
ID: "3",
|
||||
Price: "5€",
|
||||
Category: "Kuddelmuddel",
|
||||
Text: "Thing to sale",
|
||||
@@ -290,7 +290,7 @@ var adsrc = []AdConfig{
|
||||
},
|
||||
{
|
||||
Title: "Forth Ad",
|
||||
Id: "4",
|
||||
ID: "4",
|
||||
Price: "5€",
|
||||
Category: "Krempel",
|
||||
Text: "Thing to sale",
|
||||
@@ -301,7 +301,7 @@ var adsrc = []AdConfig{
|
||||
},
|
||||
{
|
||||
Title: "Fifth Ad",
|
||||
Id: "5",
|
||||
ID: "5",
|
||||
Price: "5€",
|
||||
Category: "Kladderadatsch",
|
||||
Text: "Thing to sale",
|
||||
@@ -312,7 +312,7 @@ var adsrc = []AdConfig{
|
||||
},
|
||||
{
|
||||
Title: "Sixth Ad",
|
||||
Id: "6",
|
||||
ID: "6",
|
||||
Price: "5€",
|
||||
Category: "Klunker",
|
||||
Text: "Thing to sale",
|
||||
@@ -334,17 +334,17 @@ type Adsource struct {
|
||||
}
|
||||
|
||||
// Render a HTML template for an adlisting or an ad
|
||||
func GetTemplate(l []AdConfig, a AdConfig, htmltemplate string) string {
|
||||
func GetTemplate(adconfigs []AdConfig, adconfig AdConfig, htmltemplate string) string {
|
||||
tmpl, err := tpl.New("template").Parse(htmltemplate)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
if len(a.Id) == 0 {
|
||||
err = tmpl.Execute(&out, l)
|
||||
if len(adconfig.ID) == 0 {
|
||||
err = tmpl.Execute(&out, adconfigs)
|
||||
} else {
|
||||
err = tmpl.Execute(&out, a)
|
||||
err = tmpl.Execute(&out, adconfig)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
@@ -391,10 +391,9 @@ func InitValidSources() []Adsource {
|
||||
// prepare urls for the ads
|
||||
for _, ad := range adsrc {
|
||||
ads = append(ads, Adsource{
|
||||
uri: fmt.Sprintf("%s/s-anzeige/%s/%s", Baseuri, ad.Slug, ad.Id),
|
||||
uri: fmt.Sprintf("%s/s-anzeige/%s/%s", Baseuri, ad.Slug, ad.ID),
|
||||
content: GetTemplate(nil, ad, ADTPL),
|
||||
})
|
||||
//panic(GetTemplate(nil, ad, ADTPL))
|
||||
}
|
||||
|
||||
return ads
|
||||
@@ -447,46 +446,48 @@ func GetImage(path string) []byte {
|
||||
|
||||
// setup httpmock
|
||||
func SetIntercept(ads []Adsource) {
|
||||
ch := http.Header{}
|
||||
ch.Add("Set-Cookie", "session=permanent")
|
||||
headers := http.Header{}
|
||||
headers.Add("Set-Cookie", "session=permanent")
|
||||
|
||||
for _, ad := range ads {
|
||||
if ad.status == 0 {
|
||||
ad.status = 200
|
||||
for _, advertisement := range ads {
|
||||
if advertisement.status == 0 {
|
||||
advertisement.status = 200
|
||||
}
|
||||
|
||||
httpmock.RegisterResponder("GET", ad.uri,
|
||||
httpmock.NewStringResponder(ad.status, ad.content).HeaderAdd(ch))
|
||||
httpmock.RegisterResponder("GET", advertisement.uri,
|
||||
httpmock.NewStringResponder(advertisement.status, advertisement.content).HeaderAdd(headers))
|
||||
}
|
||||
|
||||
// we just use 2 images, put this here
|
||||
for _, image := range []string{"t/1.jpg", "t/2.jpg"} {
|
||||
httpmock.RegisterResponder("GET", image,
|
||||
httpmock.NewBytesResponder(200, GetImage(image)).HeaderAdd(ch))
|
||||
httpmock.NewBytesResponder(200, GetImage(image)).HeaderAdd(headers))
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func VerifyAd(ad AdConfig) error {
|
||||
body := ad.Title + ad.Price + ad.Id + "Kleinanzeigen => " +
|
||||
ad.Category + ad.Condition + ad.Created
|
||||
func VerifyAd(advertisement AdConfig) error {
|
||||
body := advertisement.Title + advertisement.Price + advertisement.ID + "Kleinanzeigen => " +
|
||||
advertisement.Category + advertisement.Condition + advertisement.Created
|
||||
|
||||
// prepare ad dir name using DefaultAdNameTemplate
|
||||
c := Config{Adnametemplate: "{{ .Slug }}"}
|
||||
adstruct := Ad{Slug: ad.Slug, Id: ad.Id}
|
||||
adstruct := Ad{Slug: advertisement.Slug, ID: advertisement.ID}
|
||||
|
||||
addir, err := AdDirName(&c, &adstruct)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
file := fmt.Sprintf("t/out/%s/Adlisting.txt", addir)
|
||||
|
||||
content, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("unable to read adlisting file: %w", err)
|
||||
}
|
||||
|
||||
if body != strings.TrimSpace(string(content)) {
|
||||
msg := fmt.Sprintf("ad content doesn't match.\nExpect: %s\n Got: %s\n", body, content)
|
||||
|
||||
return errors.New(msg)
|
||||
}
|
||||
|
||||
@@ -504,20 +505,21 @@ func TestMain(t *testing.T) {
|
||||
SetIntercept(InitValidSources())
|
||||
|
||||
// run commandline tests
|
||||
for _, tt := range tests {
|
||||
for _, test := range tests {
|
||||
var buf bytes.Buffer
|
||||
os.Args = strings.Split(tt.args, " ")
|
||||
|
||||
os.Args = strings.Split(test.args, " ")
|
||||
|
||||
ret := Main(&buf)
|
||||
|
||||
if ret != tt.exitcode {
|
||||
if ret != test.exitcode {
|
||||
t.Errorf("%s with cmd <%s> did not exit with %d but %d",
|
||||
tt.name, tt.args, tt.exitcode, ret)
|
||||
test.name, test.args, test.exitcode, ret)
|
||||
}
|
||||
|
||||
if !strings.Contains(buf.String(), tt.expect) {
|
||||
if !strings.Contains(buf.String(), test.expect) {
|
||||
t.Errorf("%s with cmd <%s> output did not match.\nExpect: %s\n Got: %s\n",
|
||||
tt.name, tt.args, tt.expect, buf.String())
|
||||
test.name, test.args, test.expect, buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -540,20 +542,21 @@ func TestMainInvalids(t *testing.T) {
|
||||
SetIntercept(InitInvalidSources())
|
||||
|
||||
// run commandline tests
|
||||
for _, tt := range invalidtests {
|
||||
for _, test := range invalidtests {
|
||||
var buf bytes.Buffer
|
||||
os.Args = strings.Split(tt.args, " ")
|
||||
|
||||
os.Args = strings.Split(test.args, " ")
|
||||
|
||||
ret := Main(&buf)
|
||||
|
||||
if ret != tt.exitcode {
|
||||
if ret != test.exitcode {
|
||||
t.Errorf("%s with cmd <%s> did not exit with %d but %d",
|
||||
tt.name, tt.args, tt.exitcode, ret)
|
||||
test.name, test.args, test.exitcode, ret)
|
||||
}
|
||||
|
||||
if !strings.Contains(buf.String(), tt.expect) {
|
||||
if !strings.Contains(buf.String(), test.expect) {
|
||||
t.Errorf("%s with cmd <%s> output did not match.\nExpect: %s\n Got: %s\n",
|
||||
tt.name, tt.args, tt.expect, buf.String())
|
||||
test.name, test.args, test.expect, buf.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
13
scrape.go
13
scrape.go
@@ -54,7 +54,7 @@ func ScrapeUser(fetch *Fetcher) error {
|
||||
|
||||
err = goq.NewDecoder(body).Decode(&index)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("failed to goquery decode HTML index body: %w", err)
|
||||
}
|
||||
|
||||
if len(index.Links) == 0 {
|
||||
@@ -92,12 +92,12 @@ func ScrapeAd(fetch *Fetcher, uri string) error {
|
||||
|
||||
// extract slug and id from uri
|
||||
uriparts := strings.Split(uri, "/")
|
||||
if len(uriparts) < SlugUriPartNum {
|
||||
if len(uriparts) < SlugURIPartNum {
|
||||
return fmt.Errorf("invalid uri: %s", uri)
|
||||
}
|
||||
|
||||
advertisement.Slug = uriparts[4]
|
||||
advertisement.Id = uriparts[5]
|
||||
advertisement.ID = uriparts[5]
|
||||
|
||||
// get the ad
|
||||
slog.Debug("fetching ad page", "uri", uri)
|
||||
@@ -111,7 +111,7 @@ func ScrapeAd(fetch *Fetcher, uri string) error {
|
||||
// extract ad contents with goquery/goq
|
||||
err = goq.NewDecoder(body).Decode(&advertisement)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to goquery decode HTML body: %w", err)
|
||||
return fmt.Errorf("failed to goquery decode HTML ad body: %w", err)
|
||||
}
|
||||
|
||||
if len(advertisement.CategoryTree) > 0 {
|
||||
@@ -120,6 +120,7 @@ func ScrapeAd(fetch *Fetcher, uri string) error {
|
||||
|
||||
if advertisement.Incomplete() {
|
||||
slog.Debug("got ad", "ad", advertisement)
|
||||
|
||||
return fmt.Errorf("could not extract ad data from page, got empty struct")
|
||||
}
|
||||
|
||||
@@ -183,6 +184,7 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error {
|
||||
if !fetch.Config.ForceDownload {
|
||||
if image.SimilarExists(cache) {
|
||||
slog.Debug("similar image exists, not written", "uri", image.URI)
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -193,13 +195,14 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error {
|
||||
}
|
||||
|
||||
slog.Debug("wrote image", "image", image, "size", len(buf2), "throttle", throttle)
|
||||
|
||||
return nil
|
||||
})
|
||||
img++
|
||||
}
|
||||
|
||||
if err := egroup.Wait(); err != nil {
|
||||
return err
|
||||
return fmt.Errorf("failed to finalize error waitgroup: %w", err)
|
||||
}
|
||||
|
||||
fetch.Config.IncrImgs(len(advertisement.Images))
|
||||
|
||||
26
store.go
26
store.go
@@ -28,14 +28,15 @@ import (
|
||||
tpl "text/template"
|
||||
)
|
||||
|
||||
func AdDirName(c *Config, ad *Ad) (string, error) {
|
||||
tmpl, err := tpl.New("adname").Parse(c.Adnametemplate)
|
||||
func AdDirName(conf *Config, advertisement *Ad) (string, error) {
|
||||
tmpl, err := tpl.New("adname").Parse(conf.Adnametemplate)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to parse adname template: %w", err)
|
||||
}
|
||||
|
||||
buf := bytes.Buffer{}
|
||||
err = tmpl.Execute(&buf, ad)
|
||||
|
||||
err = tmpl.Execute(&buf, advertisement)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to execute adname template: %w", err)
|
||||
}
|
||||
@@ -43,15 +44,16 @@ func AdDirName(c *Config, ad *Ad) (string, error) {
|
||||
return buf.String(), nil
|
||||
}
|
||||
|
||||
func WriteAd(c *Config, ad *Ad) (string, error) {
|
||||
func WriteAd(conf *Config, advertisement *Ad) (string, error) {
|
||||
// prepare ad dir name
|
||||
addir, err := AdDirName(c, ad)
|
||||
addir, err := AdDirName(conf, advertisement)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// prepare output dir
|
||||
dir := filepath.Join(c.Outdir, addir)
|
||||
dir := filepath.Join(conf.Outdir, addir)
|
||||
|
||||
err = Mkdir(dir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -59,24 +61,25 @@ func WriteAd(c *Config, ad *Ad) (string, error) {
|
||||
|
||||
// write ad file
|
||||
listingfile := filepath.Join(dir, "Adlisting.txt")
|
||||
|
||||
listingfd, err := os.Create(listingfile)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create Adlisting.txt: %w", err)
|
||||
}
|
||||
defer listingfd.Close()
|
||||
|
||||
if runtime.GOOS == "windows" {
|
||||
ad.Text = strings.ReplaceAll(ad.Text, "<br/>", "\r\n")
|
||||
if runtime.GOOS == WIN {
|
||||
advertisement.Text = strings.ReplaceAll(advertisement.Text, "<br/>", "\r\n")
|
||||
} else {
|
||||
ad.Text = strings.ReplaceAll(ad.Text, "<br/>", "\n")
|
||||
advertisement.Text = strings.ReplaceAll(advertisement.Text, "<br/>", "\n")
|
||||
}
|
||||
|
||||
tmpl, err := tpl.New("adlisting").Parse(c.Template)
|
||||
tmpl, err := tpl.New("adlisting").Parse(conf.Template)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to parse adlisting template: %w", err)
|
||||
}
|
||||
|
||||
err = tmpl.Execute(listingfd, ad)
|
||||
err = tmpl.Execute(listingfd, advertisement)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to execute adlisting template: %w", err)
|
||||
}
|
||||
@@ -127,5 +130,6 @@ func fileExists(filename string) bool {
|
||||
if os.IsNotExist(err) {
|
||||
return false
|
||||
}
|
||||
|
||||
return !info.IsDir()
|
||||
}
|
||||
|
||||
@@ -26,6 +26,8 @@ import (
|
||||
// doesn't show up in the coverage report for unknown reasons, so
|
||||
// here's a single test for it
|
||||
func TestWriteImage(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
buf := []byte{1, 2, 3, 4, 5, 6, 7, 8}
|
||||
file := "t/out/t.jpg"
|
||||
|
||||
@@ -33,5 +35,4 @@ func TestWriteImage(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Errorf("Could not write mock image to %s: %s", file, err.Error())
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# empty config for Main() unit tests to force unit tests NOT to use an
|
||||
# eventually existing ~/.kleingebaeck!
|
||||
template="""
|
||||
{{.Title}}{{.Price}}{{.Id}}{{.Category}}{{.Condition}}{{.Created}}
|
||||
{{.Title}}{{.Price}}{{.ID}}{{.Category}}{{.Condition}}{{.Created}}
|
||||
"""
|
||||
|
||||
|
||||
@@ -2,5 +2,5 @@ user = 1
|
||||
loglevel = "verbose"
|
||||
outdir = "t/out"
|
||||
template="""
|
||||
{{.Title}}{{.Price}}{{.Id}}{{.Category}}{{.Condition}}{{.Created}}
|
||||
{{.Title}}{{.Price}}{{.ID}}{{.Category}}{{.Condition}}{{.Created}}
|
||||
"""
|
||||
|
||||
5
util.go
5
util.go
@@ -45,7 +45,8 @@ func man() error {
|
||||
man := exec.Command("less", "-")
|
||||
|
||||
var b bytes.Buffer
|
||||
b.Write([]byte(manpage))
|
||||
|
||||
b.WriteString(manpage)
|
||||
|
||||
man.Stdout = os.Stdout
|
||||
man.Stdin = &b
|
||||
@@ -62,7 +63,7 @@ func man() error {
|
||||
|
||||
// returns TRUE if stdout is NOT a tty or windows
|
||||
func IsNoTty() bool {
|
||||
if runtime.GOOS == "windows" || !isatty.IsTerminal(os.Stdout.Fd()) {
|
||||
if runtime.GOOS == WIN || !isatty.IsTerminal(os.Stdout.Fd()) {
|
||||
return true
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user