diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4066875..9172556 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -24,6 +24,17 @@ jobs: - name: test run: make test + - name: Update coverage report + uses: ncruces/go-coverage-report@main + with: + report: true + chart: true + amend: true + if: | + matrix.os == 'ubuntu-latest' && + github.event_name == 'push' + continue-on-error: true + golangci: name: lint runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index d1b1031..b992fd1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ test kleingebaeck releases +t/out diff --git a/README.md b/README.md index 5fbd83d..7146627 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ [![License](https://img.shields.io/badge/license-GPL-blue.svg)](https://github.com/tlinden/kleingebaeck/blob/master/LICENSE) [![Go Report Card](https://goreportcard.com/badge/github.com/tlinden/kleingebaeck)](https://goreportcard.com/report/github.com/tlinden/kleingebaeck) [![Actions](https://github.com/tlinden/kleingebaeck/actions/workflows/ci.yaml/badge.svg)](https://github.com/tlinden/kleingebaeck/actions) +[![Go Coverage](https://github.com/tlinden/kleingebaeck/wiki/coverage.svg)](https://raw.githack.com/wiki/tlinden/kleingebaeck/coverage.html) ![GitHub License](https://img.shields.io/github/license/tlinden/kleingebaeck) [![GitHub release](https://img.shields.io/github/v/release/tlinden/kleingebaeck?color=%2300a719)](https://github.com/TLINDEN/kleingebaeck/releases/latest) diff --git a/config.go b/config.go index 862a8e1..be53e30 100644 --- a/config.go +++ b/config.go @@ -19,6 +19,7 @@ package main import ( "errors" "fmt" + "io" "os" "path/filepath" "runtime" @@ -87,7 +88,7 @@ func (c *Config) IncrImgs(num int) { } // load commandline flags and config file -func InitConfig() (*Config, error) { +func InitConfig(w io.Writer) (*Config, error) { var k = koanf.New(".") // determine template based on os @@ -103,13 +104,13 @@ func InitConfig() (*Config, error) { "loglevel": "notice", "userid": 0, }, "."), nil); err != nil { - return nil, err - } + return nil, err + } // setup custom usage f := flag.NewFlagSet("config", flag.ContinueOnError) f.Usage = func() { - fmt.Println(Usage) + fmt.Fprintln(w, Usage) os.Exit(0) } @@ -125,8 +126,8 @@ func InitConfig() (*Config, error) { f.BoolP("manual", "m", false, "show manual") if err := f.Parse(os.Args[1:]); err != nil { - return nil, err - } + return nil, err + } // generate a list of config files to try to load, including the // one provided via -c, if any diff --git a/main.go b/main.go index fdefc9a..5f334c3 100644 --- a/main.go +++ b/main.go @@ -20,6 +20,7 @@ package main import ( "errors" "fmt" + "io" "log/slog" "os" "runtime/debug" @@ -30,10 +31,10 @@ import ( const LevelNotice = slog.Level(2) func main() { - os.Exit(Main()) + os.Exit(Main(os.Stdout)) } -func Main() int { +func Main(w io.Writer) int { logLevel := &slog.LevelVar{} opts := &tint.Options{ Level: logLevel, @@ -49,22 +50,22 @@ func Main() int { } logLevel.Set(LevelNotice) - var handler slog.Handler = tint.NewHandler(os.Stdout, opts) + handler := tint.NewHandler(w, opts) logger := slog.New(handler) slog.SetDefault(logger) - conf, err := InitConfig() + conf, err := InitConfig(w) if err != nil { return Die(err) } if conf.Showversion { - fmt.Printf("This is kleingebaeck version %s\n", VERSION) + fmt.Fprintf(w, "This is kleingebaeck version %s\n", VERSION) return 0 } if conf.Showhelp { - fmt.Println(Usage) + fmt.Fprintln(w, Usage) return 0 } @@ -90,7 +91,7 @@ func Main() int { } logLevel.Set(slog.LevelDebug) - var handler slog.Handler = tint.NewHandler(os.Stdout, opts) + handler := tint.NewHandler(w, opts) debuglogger := slog.New(handler).With( slog.Group("program_info", slog.Int("pid", os.Getpid()), @@ -100,6 +101,8 @@ func Main() int { slog.SetDefault(debuglogger) } + // defaultlogger := log.Default() + // defaultlogger.SetOutput(w) slog.Debug("config", "conf", conf) // prepare output dir @@ -131,10 +134,10 @@ func Main() int { if conf.StatsCountAds == 1 { adstr = "ad" } - fmt.Printf("Successfully downloaded %d %s with %d images to %s.\n", + fmt.Fprintf(w, "Successfully downloaded %d %s with %d images to %s.\n", conf.StatsCountAds, adstr, conf.StatsCountImages, conf.Outdir) } else { - fmt.Printf("No ads found.") + fmt.Fprintf(w, "No ads found.") } return 0 diff --git a/main_test.go b/main_test.go new file mode 100644 index 0000000..bceb8af --- /dev/null +++ b/main_test.go @@ -0,0 +1,520 @@ +/* +Copyright © 2023 Thomas von Dein + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package main + +import ( + "bytes" + "errors" + "fmt" + "os" + "strings" + "testing" + tpl "text/template" + + "github.com/jarcoal/httpmock" +) + +// the ad list, aka: +// https://www.kleinanzeigen.de/s-bestandsliste.html?userId=XXXXXX +// Note, that this HTML code is reduced to the max, so that it only +// contains the stuff required to satisfy goquery +const LISTTPL string = ` + + + Ads + + +{{ range . }} +

+ {{ .Title }} +

+{{ end }} + + +` + +// an actual ad listing, aka: +// https://www.kleinanzeigen.de/s-anzeige/ad-text-slug/1010101010 +// Note, that this HTML code is reduced to the max, so that it only +// contains the stuff required to satisfy goquery +const ADTPL string = `DOCTYPE html> + + + Ad Listing + + + + {{ range $image := .Images }} +
+ +
+ {{ end }} + +

+ {{ .Title }}

+
+

+ {{ .Price }}

+
+ +
+
{{ .Created }}
+
+ +
+ +
+ +
+

+ {{ .Text }} +

+
+ + +` + +const EMPTYPAGE string = `DOCTYPE html> + + + + +` + +const ( + EMPTYURI string = `https://www.kleinanzeigen.de/s-anzeige/empty/1` + INVALIDPATHURI string = `https://www.kleinanzeigen.de/anzeige/name/1` + INVALID404URI string = `https://www.kleinanzeigen.de/anzeige/name/1/foo/bar` + INVALIDURI string = `https://foo.bar/weird/things` +) + +var base = "kleingebaeck -c t/config-empty.conf" + +type Tests struct { + name string + args string + expect string + exitcode int +} + +var tests = []Tests{ + { + name: "version", + args: base + " -V", + expect: "This is", + exitcode: 0, + }, + { + name: "help", + args: base + " -h", + expect: "Usage:", + exitcode: 0, + }, + { + name: "debug", + args: base + " -d", + expect: "program_info", + exitcode: 1, + }, + { + name: "no-args-no-user", + args: base, + expect: "invalid or no user id", + exitcode: 1, + }, + { + name: "download-single-ad", + args: base + " -o t/out https://www.kleinanzeigen.de/s-anzeige/first-ad/1", + expect: "Successfully downloaded 1 ad with 2 images to t/out", + exitcode: 0, + }, + { + name: "download-single-ad-verbose", + args: base + " -o t/out https://www.kleinanzeigen.de/s-anzeige/first-ad/1 -v", + expect: "wrote ad listing", + exitcode: 0, + }, + { + name: "download-single-ad-debug", + args: base + " -o t/out https://www.kleinanzeigen.de/s-anzeige/first-ad/1 -d", + expect: "extracted ad listing program_info.pid=", + exitcode: 0, + }, + { + name: "download-all-ads", + args: base + " -o t/out -u 1", + expect: "Successfully downloaded 6 ads with 12 images to t/out", + exitcode: 0, + }, + { + name: "download-all-ads-using-config", + args: "kleingebaeck -c t/fullconfig.conf", + expect: "Successfully downloaded 6 ads with 12 images to t/out", + exitcode: 0, + }, +} + +var invalidtests = []Tests{ + { + name: "empty-ad", + args: base + " " + EMPTYURI, + expect: "could not extract ad data from page, got empty struct", + exitcode: 1, + }, + { + name: "invalid-ad", + args: base + " " + INVALIDURI, + expect: "invalid uri", + exitcode: 1, + }, + { + name: "invalid-path", + args: base + " " + INVALIDPATHURI, + expect: "could not extract ad data from page, got empty struct", + exitcode: 1, + }, + { + name: "404", + args: base + " " + INVALID404URI, + expect: "could not get page via HTTP", + exitcode: 1, + }, + { + name: "outdir-no-exists", + args: base + " -o t/foo/bar/out https://www.kleinanzeigen.de/s-anzeige/first-ad/1 -v", + expect: "Failure", + exitcode: 1, + }, + { + name: "wrong-flag", + args: base + " -X", + expect: "unknown shorthand flag: 'X' in -X", + exitcode: 1, + }, + { + name: "no-config", + args: "kleingebaeck -c t/invalid.conf", + expect: "error loading config file", + exitcode: 1, + }, +} + +type AdConfig struct { + Title string + Slug string + Id string + Price string + Category string + Condition string + Created string + Text string + Images []string // files in ./t/ +} + +var adsrc = []AdConfig{ + { + Title: "First Ad", + Id: "1", Price: "5€", + Category: "Klimbim", + Text: "Thing to sale", + Slug: "first-ad", + Condition: "works", + Created: "Yesterday", + Images: []string{"t/1.jpg", "t/2.jpg"}, + }, + { + Title: "Secnd Ad", + Id: "2", Price: "5€", + Category: "Kram", + Text: "Thing to sale", + Slug: "second-ad", + Condition: "works", + Created: "Yesterday", + Images: []string{"t/1.jpg", "t/2.jpg"}, + }, + { + Title: "Third Ad", + Id: "3", + Price: "5€", + Category: "Kuddelmuddel", + Text: "Thing to sale", + Slug: "third-ad", + Condition: "works", + Created: "Yesterday", + Images: []string{"t/1.jpg", "t/2.jpg"}, + }, + { + Title: "Forth Ad", + Id: "4", + Price: "5€", + Category: "Krempel", + Text: "Thing to sale", + Slug: "fourth-ad", + Condition: "works", + Created: "Yesterday", + Images: []string{"t/1.jpg", "t/2.jpg"}, + }, + { + Title: "Fifth Ad", + Id: "5", + Price: "5€", + Category: "Kladderadatsch", + Text: "Thing to sale", + Slug: "fifth-ad", + Condition: "works", + Created: "Yesterday", + Images: []string{"t/1.jpg", "t/2.jpg"}, + }, + { + Title: "Sixth Ad", + Id: "6", + Price: "5€", + Category: "Klunker", + Text: "Thing to sale", + Slug: "sixth-ad", + Condition: "works", + Created: "Yesterday", + Images: []string{"t/1.jpg", "t/2.jpg"}, + }, +} + +// An Adsource is used to construct a httpmock responder for a +// particular url. So, the code (scrape.go) scrapes +// https://kleinanzeigen.de, but in reality httpmock captures the +// request and responds with our mock data +type Adsource struct { + uri string + content string + status int +} + +// Render a HTML template for an adlisting or an ad +func GetTemplate(l []AdConfig, a AdConfig, htmltemplate string) string { + tmpl, err := tpl.New("template").Parse(htmltemplate) + if err != nil { + panic(err) + } + + var out bytes.Buffer + if len(a.Id) == 0 { + err = tmpl.Execute(&out, l) + } else { + err = tmpl.Execute(&out, a) + } + + if err != nil { + panic(err) + } + + return out.String() +} + +// Initialize the valid sources for the httpmock responder +func InitValidSources() []Adsource { + // valid ad listing page 1 + list1 := []AdConfig{ + adsrc[0], adsrc[1], adsrc[2], + } + + // valid ad listing page 2 + list2 := []AdConfig{ + adsrc[3], adsrc[4], adsrc[5], + } + + // valid ad listing page 3, which is empty + list3 := []AdConfig{} + + // used to signal GetTemplate() to render a listing + empty := AdConfig{} + + // prepare urls for the listing pages + ads := []Adsource{ + { + uri: fmt.Sprintf("%s%s?userId=1", Baseuri, Listuri), + content: GetTemplate(list1, empty, LISTTPL), + }, + { + uri: fmt.Sprintf("%s%s?userId=1&pageNum=2", Baseuri, Listuri), + content: GetTemplate(list2, empty, LISTTPL), + }, + { + uri: fmt.Sprintf("%s%s?userId=1&pageNum=3", Baseuri, Listuri), + content: GetTemplate(list3, empty, LISTTPL), + }, + } + + // prepare urls for the ads + for _, ad := range adsrc { + ads = append(ads, Adsource{ + uri: fmt.Sprintf("%s/s-anzeige/%s/%s", Baseuri, ad.Slug, ad.Id), + content: GetTemplate(nil, ad, ADTPL), + }) + //panic(GetTemplate(nil, ad, ADTPL)) + } + + return ads +} + +func InitInvalidSources() []Adsource { + empty := AdConfig{} + ads := []Adsource{ + { + // valid ad page but without content + uri: fmt.Sprintf("%s/s-anzeige/empty/1", Baseuri), + content: GetTemplate(nil, empty, EMPTYPAGE), + }, + { + // some random foreign webpage + uri: INVALIDURI, + content: GetTemplate(nil, empty, "foo"), + }, + { + // some invalid page path + uri: fmt.Sprintf("%s/anzeige/name/1", Baseuri), + content: GetTemplate(nil, empty, ""), + }, + { + // some none-ad page + uri: fmt.Sprintf("%s/anzeige/name/1/foo/bar", Baseuri), + content: GetTemplate(nil, empty, "HTTP 404: /eine-anzeige/ does not exist!"), + status: 404, + }, + } + + return ads +} + +// load a test image from disk +func GetImage(path string) []byte { + dat, err := os.ReadFile(path) + if err != nil { + panic(err) + } + + return dat +} + +// setup httpmock +func SetIntercept(ads []Adsource) { + for _, ad := range ads { + if ad.status == 0 { + ad.status = 200 + } + + httpmock.RegisterResponder("GET", ad.uri, + httpmock.NewStringResponder(ad.status, ad.content)) + } + + // we just use 2 images, put this here + for _, image := range []string{"t/1.jpg", "t/2.jpg"} { + httpmock.RegisterResponder("GET", image, httpmock.NewBytesResponder(200, GetImage(image))) + } + +} + +func VerifyAd(ad AdConfig) error { + body := ad.Title + ad.Price + ad.Id + ad.Category + ad.Condition + ad.Created + + file := fmt.Sprintf("t/out/%s/Adlisting.txt", ad.Slug) + content, err := os.ReadFile(file) + if err != nil { + return err + } + + if body != strings.TrimSpace(string(content)) { + msg := fmt.Sprintf("ad content doesn't match.\nExpect: %s\n Got: %s\n", body, content) + return errors.New(msg) + } + + return nil +} + +func TestMain(t *testing.T) { + oldargs := os.Args + defer func() { os.Args = oldargs }() + + httpmock.Activate() + defer httpmock.DeactivateAndReset() + + // prepare httpmock responders + SetIntercept(InitValidSources()) + + // run commandline tests + for _, tt := range tests { + var buf bytes.Buffer + os.Args = strings.Split(tt.args, " ") + + ret := Main(&buf) + + if ret != tt.exitcode { + t.Errorf("%s with cmd <%s> did not exit with %d but %d", + tt.name, tt.args, tt.exitcode, ret) + } + + if !strings.Contains(buf.String(), tt.expect) { + t.Errorf("%s with cmd <%s> output did not match.\nExpect: %s\n Got: %s\n", + tt.name, tt.args, tt.expect, buf.String()) + } + } + + // verify if downloaded ads match + for _, ad := range adsrc { + if err := VerifyAd(ad); err != nil { + t.Errorf(err.Error()) + } + } +} + +func TestMainInvalids(t *testing.T) { + oldargs := os.Args + defer func() { os.Args = oldargs }() + + httpmock.Activate() + defer httpmock.DeactivateAndReset() + + // prepare httpmock responders + SetIntercept(InitInvalidSources()) + + // run commandline tests + for _, tt := range invalidtests { + var buf bytes.Buffer + os.Args = strings.Split(tt.args, " ") + + ret := Main(&buf) + + if ret != tt.exitcode { + t.Errorf("%s with cmd <%s> did not exit with %d but %d", + tt.name, tt.args, tt.exitcode, ret) + } + + if !strings.Contains(buf.String(), tt.expect) { + t.Errorf("%s with cmd <%s> output did not match.\nExpect: %s\n Got: %s\n", + tt.name, tt.args, tt.expect, buf.String()) + } + } +} diff --git a/scrape.go b/scrape.go index 7ab0bfb..25e7d73 100644 --- a/scrape.go +++ b/scrape.go @@ -196,7 +196,7 @@ func Getimage(uri, fileName string) error { defer response.Body.Close() if response.StatusCode != 200 { - return errors.New("received non 200 response code") + return errors.New("could not get image via HTTP") } err = WriteImage(fileName, response.Body) diff --git a/scrape_test.go b/scrape_test.go deleted file mode 100644 index fd3c20f..0000000 --- a/scrape_test.go +++ /dev/null @@ -1,345 +0,0 @@ -/* -Copyright © 2023 Thomas von Dein - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ - -package main - -import ( - "bytes" - "fmt" - "os" - "strings" - "testing" - tpl "text/template" - - "github.com/jarcoal/httpmock" -) - -// used to fill an ad template and the ad listing page template -type AdConfig struct { - Title string - Slug string - Id string - Price string - Category string - Condition string - Created string - Text string - Images []string // files in ./t/ -} - -// the ad list, aka: -// https://www.kleinanzeigen.de/s-bestandsliste.html?userId=XXXXXX -// Note, that this HTML code is reduced to the max, so that it only -// contains the stuff required to satisfy goquery -const LISTTPL string = ` - - - Ads - - -{{ range . }} -

- {{ .Title }} -

-{{ end }} - - -` - -// an actual ad listing, aka: -// https://www.kleinanzeigen.de/s-anzeige/ad-text-slug/1010101010 -// Note, that this HTML code is reduced to the max, so that it only -// contains the stuff required to satisfy goquery -const ADTPL string = `DOCTYPE html> - - - Ad Listing - - - - {{ range $image := .Images }} -
- -
- {{ end }} - -

- {{ .Title }}

-
-

- {{ .Price }}

-
- -
-
{{ .Created }}
-
- -
-
    -
  • - Art - {{ .Category }} -
  • -
  • - Zustand - {{ .Condition }} -
  • -
-
- -
-

- {{ .Text }} -

-
- - -` - -const EMPTYPAGE string = `DOCTYPE html> - - - - -` - -const EMPTYURI string = `https://www.kleinanzeigen.de/s-anzeige/empty/1` -const INVALIDURI string = `https://foo.bar/weird/things` - -// An Adsource is used to construct a httpmock responder for a -// particular url. So, the code (scrape.go) scrapes -// https://kleinanzeigen.de, but in reality httpmock captures the -// request and responds with our mock data -type Adsource struct { - uri string - content string - status int -} - -// Render a HTML template for an adlisting or an ad -func GetTemplate(l []AdConfig, a AdConfig, htmltemplate string) string { - tmpl, err := tpl.New("template").Parse(htmltemplate) - if err != nil { - panic(err) - } - - var out bytes.Buffer - if len(a.Id) == 0 { - err = tmpl.Execute(&out, l) - } else { - err = tmpl.Execute(&out, a) - } - - if err != nil { - panic(err) - } - - return out.String() -} - -func InitAds() []AdConfig { - return []AdConfig{ - {Title: "First Ad", Id: "1", Price: "5€", Category: "Klimbim", Text: "Thing to sale", Slug: "first-ad", - Condition: "works", Created: "Yesterday", Images: []string{"t/1.jpg", "t/2.jpg"}}, - {Title: "Secnd Ad", Id: "2", Price: "5€", Category: "Kram", Text: "Thing to sale", Slug: "second-ad", - Condition: "works", Created: "Yesterday", Images: []string{"t/1.jpg", "t/2.jpg"}}, - {Title: "Third Ad", Id: "3", Price: "5€", Category: "Kuddelmuddel", Text: "Thing to sale", Slug: "third-ad", - Condition: "works", Created: "Yesterday", Images: []string{"t/1.jpg", "t/2.jpg"}}, - {Title: "Forth Ad", Id: "4", Price: "5€", Category: "Krempel", Text: "Thing to sale", Slug: "fourth-ad", - Condition: "works", Created: "Yesterday", Images: []string{"t/1.jpg", "t/2.jpg"}}, - {Title: "Fifth Ad", Id: "5", Price: "5€", Category: "Kladderadatsch", Text: "Thing to sale", Slug: "fifth-ad", - Condition: "works", Created: "Yesterday", Images: []string{"t/1.jpg", "t/2.jpg"}}, - {Title: "Sixth Ad", Id: "6", Price: "5€", Category: "Klunker", Text: "Thing to sale", Slug: "sixth-ad", - Condition: "works", Created: "Yesterday", Images: []string{"t/1.jpg", "t/2.jpg"}}, - } -} - -// Initialize the valid sources for the httpmock responder -func InitValidSources(conf *Config) []Adsource { - // all our valid ads - adsrc := InitAds() - - // valid ad listing page 1 - list1 := []AdConfig{ - adsrc[0], adsrc[1], adsrc[2], - } - - // valid ad listing page 2 - list2 := []AdConfig{ - adsrc[3], adsrc[4], adsrc[5], - } - - // valid ad listing page 3, which is empty - list3 := []AdConfig{} - - // used to signal GetTemplate() to render a listing - empty := AdConfig{} - - // prepare urls for the listing pages - ads := []Adsource{ - { - uri: fmt.Sprintf("%s%s?userId=%d", Baseuri, Listuri, conf.User), - content: GetTemplate(list1, empty, LISTTPL), - }, - { - uri: fmt.Sprintf("%s%s?userId=%d&pageNum=2", Baseuri, Listuri, conf.User), - content: GetTemplate(list2, empty, LISTTPL), - }, - { - uri: fmt.Sprintf("%s%s?userId=%d&pageNum=3", Baseuri, Listuri, conf.User), - content: GetTemplate(list3, empty, LISTTPL), - }, - } - - // prepare urls for the ads - for _, ad := range adsrc { - ads = append(ads, Adsource{ - uri: fmt.Sprintf("%s/s-anzeige/%s/%s", Baseuri, ad.Slug, ad.Id), - content: GetTemplate(nil, ad, ADTPL), - }) - //panic(GetTemplate(nil, ad, ADTPL)) - } - - return ads -} - -func InitInvalidSources(conf *Config) []Adsource { - empty := AdConfig{} - ads := []Adsource{ - { - // valid ad page but without content - uri: fmt.Sprintf("%s/s-anzeige/empty/1", Baseuri), - content: GetTemplate(nil, empty, EMPTYPAGE), - }, - { - // some random foreign webpage - uri: INVALIDURI, - content: GetTemplate(nil, empty, "foo"), - }, - { - // some invalid page path - uri: fmt.Sprintf("%s/anzeige/name/1", Baseuri), - content: GetTemplate(nil, empty, ""), - }, - { - // some none-ad page - uri: fmt.Sprintf("%s/anzeige/name/1/foo/bar", Baseuri), - content: GetTemplate(nil, empty, "HTTP 404: /eine-anzeige/ does not exist!"), - status: 404, - }, - } - - return ads -} - -// load a test image from disk -func GetImage(path string) []byte { - dat, err := os.ReadFile(path) - if err != nil { - panic(err) - } - - return dat -} - -// setup httpmock -func SetIntercept(conf *Config) { - ads := InitValidSources(conf) - eads := InitInvalidSources(conf) - - ads = append(ads, eads...) - - for _, ad := range ads { - if ad.status == 0 { - ad.status = 200 - } - - httpmock.RegisterResponder("GET", ad.uri, - httpmock.NewStringResponder(ad.status, ad.content)) - } - - // we just use 2 images, put this here - for _, image := range []string{"t/1.jpg", "t/2.jpg"} { - httpmock.RegisterResponder("GET", image, httpmock.NewBytesResponder(200, GetImage(image))) - } - -} - -// the actual test, calls Start() from scrape, which recursively -// scrapes ads from a user -func TestStart(t *testing.T) { - httpmock.Activate() - defer httpmock.DeactivateAndReset() - - // fake config - conf := &Config{User: 1, Outdir: "t/out", Template: DefaultTemplate} - - // prepare httpmock responders - SetIntercept(conf) - - // run - if err := Start(conf); err != nil { - t.Errorf("failed to scrape: %s", err.Error()) - } - - // verify - for _, ad := range InitAds() { - file := fmt.Sprintf("t/out/%s/Adlisting.txt", ad.Slug) - content, err := os.ReadFile(file) - if err != nil { - t.Errorf("failed to read adlisting: %s", err.Error()) - } - - if !strings.Contains(string(content), ad.Category) && !strings.Contains(string(content), ad.Title) { - t.Errorf("failed to verify: %s content doesn't contain expected data", file) - } - } - - // uncomment to see slogs - //t.Errorf("debug") -} - -func TestSingleFail(t *testing.T) { - httpmock.Activate() - defer httpmock.DeactivateAndReset() - - // fake config - conf := &Config{Outdir: "t/out", Template: DefaultTemplate, Adlinks: []string{EMPTYURI}} - - SetIntercept(conf) - - // check empty ad - if err := Scrape(conf, EMPTYURI); err == nil { - t.Errorf("scrape returned empty ad") - } - - // wrong uri - if err := Scrape(conf, INVALIDURI); err == nil { - t.Errorf("scrape returned ad from invalid web site") - } - - // wrong path - if err := Scrape(conf, fmt.Sprintf("%s/anzeige/name/1", Baseuri)); err == nil { - t.Errorf("scrape returned ad from invalid page") - } - - // wrong path - if err := Scrape(conf, fmt.Sprintf("%s/anzeige/name/1/foo/bar", Baseuri)); err == nil { - t.Errorf("scrape returned ad from 404 page") - } -} diff --git a/t/config-empty.conf b/t/config-empty.conf new file mode 100644 index 0000000..107a152 --- /dev/null +++ b/t/config-empty.conf @@ -0,0 +1,6 @@ +# empty config for Main() unit tests to force unit tests NOT to use an +# eventually existing ~/.kleingebaeck! +template=""" +{{.Title}}{{.Price}}{{.Id}}{{.Category}}{{.Condition}}{{.Created}} +""" + diff --git a/t/fullconfig.conf b/t/fullconfig.conf new file mode 100644 index 0000000..9265883 --- /dev/null +++ b/t/fullconfig.conf @@ -0,0 +1,6 @@ +user = 1 +loglevel = "verbose" +outdir = "t/out" +template=""" +{{.Title}}{{.Price}}{{.Id}}{{.Category}}{{.Condition}}{{.Created}} +""" diff --git a/t/invalid.conf b/t/invalid.conf new file mode 100644 index 0000000..f7b6536 --- /dev/null +++ b/t/invalid.conf @@ -0,0 +1 @@ +user = "