attempt to fix #117. code works, tests are still failing

This commit is contained in:
2025-02-10 16:13:21 +01:00
parent 09948a6b39
commit 602085d15e
5 changed files with 38 additions and 29 deletions

14
ad.go
View File

@@ -31,10 +31,8 @@ type Ad struct {
Title string `goquery:"h1"`
Slug string
ID string
Details []string `goquery:".addetailslist--detail--value,text"`
Condition string // post processed from details
Type string // post processed from details
Color string // post processed from details
Details string `goquery:".addetailslist--detail,text"`
Attributes map[string]string // processed afterwards
Category string
CategoryTree []string `goquery:".breadcrump-link,text"`
Price string `goquery:"h2#viewad-price"`
@@ -53,19 +51,11 @@ func (ad *Ad) LogValue() slog.Value {
slog.Int("imagecount", len(ad.Images)),
slog.Int("bodysize", len(ad.Text)),
slog.String("categorytree", strings.Join(ad.CategoryTree, "+")),
slog.String("condition", ad.Condition),
slog.String("created", ad.Created),
slog.String("expire", ad.Expire),
)
}
// static set of conditions available, used for post processing details
var CONDITIONS = []string{"Neu", "Gut", "Sehr Gut", "In Ordnung"}
var COLORS = []string{"Beige", "Blau", "Braun", "Bunt", "Burgunderrot",
"Creme", "Gelb", "Gold", "Grau", "Grün", "Holz", "Khaki", "Lavelndel",
"Lila", "Orange", "Pink", "Print", "Rot", "Schwarz", "Silber",
"Transparent", "Türkis", "Weiß", "Sonstige"}
// check for completeness. I erected these fields to be mandatory
// (though I really don't know if they really are). I consider images
// and meta optional. So, if either of the checked fields here is

View File

@@ -40,11 +40,11 @@ const (
Defaultdir string = "."
DefaultTemplate string = "Title: {{.Title}}\nPrice: {{.Price}}\nId: {{.ID}}\n" +
"Category: {{.Category}}\nCondition: {{.Condition}}\nType: {{.Type}}\nColor: {{.Color}}\n" +
"Category: {{.Category}}\n{{ range $key,$val := .Attributes }}{{ $key }}: {{ $val }}\n{{ end }}" +
"Created: {{.Created}}\nExpire: {{.Expire}}\n\n{{.Text}}\n"
DefaultTemplateWin string = "Title: {{.Title}}\r\nPrice: {{.Price}}\r\nId: {{.ID}}\r\n" +
"Category: {{.Category}}\r\nCondition: {{.Condition}}\r\nType: {{.Type}}\r\nColor: {{.Color}}\r\n" +
"Category: {{.Category}}\r\n{{ range $key,$val := .Attributes }}{{ $key }}: {{ $val }}\r\n{{ end }}\r\n" +
"Created: {{.Created}}\r\nExpires: {{.Expire}}\r\n\r\n{{.Text}}\r\n"
DefaultUserAgent string = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " +

View File

@@ -22,7 +22,7 @@ outdir = "test"
#Price: {{.Price}}
#Id: {{.Id}}
#Category: {{.Category}}
#Condition: {{.Condition}}
#Condition: {{.Attributes.Condition}}
#Type: {{.Type}}
#Created: {{.Created}}

View File

@@ -45,9 +45,8 @@ Format is pretty simple:
Price: {{.Price}}
Id: {{.ID}}
Category: {{.Category}}
Condition: {{.Condition}}
Type: {{.Type}}
Color: {{.Color}}
{{ range $key,$val := .Attributes }}{{ $key }}: {{ $val }}
{{ end }}
Created: {{.Created}}
{{.Text}}

View File

@@ -18,11 +18,11 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"bufio"
"bytes"
"fmt"
"log/slog"
"path/filepath"
"slices"
"strconv"
"strings"
"time"
@@ -125,16 +125,7 @@ func ScrapeAd(fetch *Fetcher, uri string) error {
return fmt.Errorf("could not extract ad data from page, got empty struct")
}
for _, detail := range advertisement.Details {
switch {
case slices.Contains(CONDITIONS, detail):
advertisement.Condition = detail
case slices.Contains(COLORS, detail):
advertisement.Color = detail
default:
advertisement.Type = detail
}
}
advertisement.Attributes = DecodeAttributes(advertisement.Details)
advertisement.CalculateExpire()
@@ -167,6 +158,35 @@ func ScrapeAd(fetch *Fetcher, uri string) error {
return ScrapeImages(fetch, advertisement, addir)
}
func DecodeAttributes(attributes string) map[string]string {
rd := strings.NewReader(attributes)
scanner := bufio.NewScanner(rd)
isattr := true
attr := ""
attrmap := map[string]string{}
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" {
continue
}
if isattr {
attr = line
} else {
attrmap[attr] = line
}
isattr = !isattr
}
fmt.Println(attributes)
return attrmap
}
func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error {
// fetch images
img := 1