Compare commits

..

32 Commits

Author SHA1 Message Date
T.v.Dein
4030d04b06 Add year,month,day support to Adnametemplate (#123)
* add year, month and day to adnametemplate as well
2025-02-27 17:58:05 +01:00
eff0af0b34 pie only on linux 2025-02-19 18:08:15 +01:00
34b1ad9d1e remove symbols and crap from released binaries 2025-02-19 18:01:05 +01:00
T.v.Dein
6675c4d232 Fix/timeformat (#122)
* Fix #121: confused day with month thanks to time.Format
* Add outdir template variable example
2025-02-10 22:20:25 +01:00
T.v.Dein
46be48af38 Generic attributes (#120)
* fix #117: use a generic attribute parser, still support fixed attrs
2025-02-10 18:20:54 +01:00
T.v.Dein
09948a6b39 add color detail as well (#119)
Co-authored-by: Thomas von Dein <tom@vondein.org>
2025-02-06 20:13:08 +01:00
T.v.Dein
bc01391872 Fix ad condition parsing (#118)
* fix #117: use details slice and pre-set to properly extract condition
* also added the type part of the detail content (original de: "Art")

---------

Co-authored-by: Thomas von Dein <tom@vondein.org>
2025-02-06 13:48:20 +01:00
cd3d00adbe add changelog builder, update release builder 2025-02-05 17:54:47 +01:00
528ecdd43d updated deps 2025-02-01 18:04:25 +01:00
dependabot[bot]
5cb928518d Bump docker/build-push-action from 6.10.0 to 6.13.0
Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 6.10.0 to 6.13.0.
- [Release notes](https://github.com/docker/build-push-action/releases)
- [Commits](48aba3b46d...ca877d9245)

---
updated-dependencies:
- dependency-name: docker/build-push-action
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-02-01 18:03:43 +01:00
dependabot[bot]
d8c7409c7a Bump github.com/lmittmann/tint from 1.0.6 to 1.0.7
Bumps [github.com/lmittmann/tint](https://github.com/lmittmann/tint) from 1.0.6 to 1.0.7.
- [Release notes](https://github.com/lmittmann/tint/releases)
- [Commits](https://github.com/lmittmann/tint/compare/v1.0.6...v1.0.7)

---
updated-dependencies:
- dependency-name: github.com/lmittmann/tint
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-02-01 18:03:33 +01:00
dependabot[bot]
1cd6eb5134 Bump github.com/spf13/pflag from 1.0.5 to 1.0.6
Bumps [github.com/spf13/pflag](https://github.com/spf13/pflag) from 1.0.5 to 1.0.6.
- [Release notes](https://github.com/spf13/pflag/releases)
- [Commits](https://github.com/spf13/pflag/compare/v1.0.5...v1.0.6)

---
updated-dependencies:
- dependency-name: github.com/spf13/pflag
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-02-01 18:03:11 +01:00
9e2983a85c build release binaries using ci workflow 2025-01-18 10:53:41 +01:00
6eddd08e4a added warning about security update 2025-01-17 12:22:20 +01:00
34dfc25e87 bump version 2025-01-17 12:00:21 +01:00
dependabot[bot]
0bc6a0ae59 Bump github.com/lmittmann/tint from 1.0.5 to 1.0.6
Bumps [github.com/lmittmann/tint](https://github.com/lmittmann/tint) from 1.0.5 to 1.0.6.
- [Release notes](https://github.com/lmittmann/tint/releases)
- [Commits](https://github.com/lmittmann/tint/compare/v1.0.5...v1.0.6)

---
updated-dependencies:
- dependency-name: github.com/lmittmann/tint
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-01-06 18:44:09 +01:00
dependabot[bot]
14c554563a Bump golang.org/x/sync from 0.9.0 to 0.10.0
Bumps [golang.org/x/sync](https://github.com/golang/sync) from 0.9.0 to 0.10.0.
- [Commits](https://github.com/golang/sync/compare/v0.9.0...v0.10.0)

---
updated-dependencies:
- dependency-name: golang.org/x/sync
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-01-06 18:43:57 +01:00
475a9a2fd3 add test files in other formats 2025-01-06 18:42:19 +01:00
fbd9a5a621 make clear that this is not the complete filename 2025-01-06 18:42:19 +01:00
7014c97dee added image formats test 2025-01-06 18:42:19 +01:00
91edfeb19a support hashing with all formats 2025-01-06 18:42:19 +01:00
3b3435515c check seek err 2025-01-06 18:42:19 +01:00
2239a83f76 properly check image format for storing and distance hashing 2025-01-06 18:42:19 +01:00
31b27beee5 fix error handling when trying to open files 2024-12-13 11:23:24 +01:00
dependabot[bot]
a4be51f498 Bump docker/build-push-action from 6.9.0 to 6.10.0
Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 6.9.0 to 6.10.0.
- [Release notes](https://github.com/docker/build-push-action/releases)
- [Commits](4f58ea7922...48aba3b46d)

---
updated-dependencies:
- dependency-name: docker/build-push-action
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-12-13 11:11:31 +01:00
dependabot[bot]
6b5af984cc Bump golang.org/x/sync from 0.8.0 to 0.9.0
Bumps [golang.org/x/sync](https://github.com/golang/sync) from 0.8.0 to 0.9.0.
- [Commits](https://github.com/golang/sync/compare/v0.8.0...v0.9.0)

---
updated-dependencies:
- dependency-name: golang.org/x/sync
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-12-13 11:09:28 +01:00
dependabot[bot]
f5d3853388 Bump github.com/knadh/koanf/v2 from 2.1.1 to 2.1.2
Bumps [github.com/knadh/koanf/v2](https://github.com/knadh/koanf) from 2.1.1 to 2.1.2.
- [Release notes](https://github.com/knadh/koanf/releases)
- [Commits](https://github.com/knadh/koanf/compare/v2.1.1...v2.1.2)

---
updated-dependencies:
- dependency-name: github.com/knadh/koanf/v2
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-12-13 11:08:36 +01:00
07ebd7afad bump version 2024-11-01 12:21:04 +01:00
dependabot[bot]
5e82881b69 Bump github.com/knadh/koanf/providers/file from 1.1.0 to 1.1.2
Bumps [github.com/knadh/koanf/providers/file](https://github.com/knadh/koanf) from 1.1.0 to 1.1.2.
- [Release notes](https://github.com/knadh/koanf/releases)
- [Commits](https://github.com/knadh/koanf/compare/v1.1.0...providers/file/v1.1.2)

---
updated-dependencies:
- dependency-name: github.com/knadh/koanf/providers/file
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-11-01 12:20:45 +01:00
c9a75e3f91 bump version 2024-10-02 10:52:24 +02:00
dependabot[bot]
2fd2028cbe Bump github.com/knadh/koanf/providers/env from 0.1.0 to 1.0.0
Bumps [github.com/knadh/koanf/providers/env](https://github.com/knadh/koanf) from 0.1.0 to 1.0.0.
- [Release notes](https://github.com/knadh/koanf/releases)
- [Commits](https://github.com/knadh/koanf/compare/v0.1.0...v1.0.0)

---
updated-dependencies:
- dependency-name: github.com/knadh/koanf/providers/env
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-10-02 10:49:44 +02:00
dependabot[bot]
766f35d1d5 Bump docker/build-push-action from 6.7.0 to 6.9.0
Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 6.7.0 to 6.9.0.
- [Release notes](https://github.com/docker/build-push-action/releases)
- [Commits](5cd11c3a4c...4f58ea7922)

---
updated-dependencies:
- dependency-name: docker/build-push-action
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-10-02 10:49:17 +02:00
21 changed files with 466 additions and 88 deletions

View File

@@ -22,13 +22,13 @@ jobs:
password: ${{ secrets.GITHUB_TOKEN }} password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push Docker image - name: Build and push Docker image
uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991
with: with:
push: true push: true
tags: ghcr.io/tlinden/kleingebaeck:${{ github.ref_name}} tags: ghcr.io/tlinden/kleingebaeck:${{ github.ref_name}}
- name: Build and push latest Docker image - name: Build and push latest Docker image
uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991
with: with:
push: true push: true
tags: ghcr.io/tlinden/kleingebaeck:latest tags: ghcr.io/tlinden/kleingebaeck:latest

87
.github/workflows/release.yaml vendored Normal file
View File

@@ -0,0 +1,87 @@
name: build-release
on:
push:
tags:
- "v*.*.*"
jobs:
release:
name: Build Release Assets
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: 1.22.11
- name: Build the executables
run: ./mkrel.sh kleingebaeck ${{ github.ref_name}}
- name: List the executables
run: ls -l ./releases
- name: Upload the binaries
uses: svenstaro/upload-release-action@v2
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
tag: ${{ github.ref_name }}
file: ./releases/*
file_glob: true
- name: Build Changelog
id: github_release
uses: mikepenz/release-changelog-builder-action@v5
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
mode: "PR"
configurationJson: |
{
"template": "#{{CHANGELOG}}\n\n**Full Changelog**: #{{RELEASE_DIFF}}",
"pr_template": "- #{{TITLE}} (##{{NUMBER}}) by #{{AUTHOR}}\n#{{BODY}}",
"empty_template": "- no changes",
"categories": [
{
"title": "## New Features",
"labels": ["add", "feature"]
},
{
"title": "## Bug Fixes",
"labels": ["fix", "bug", "revert"]
},
{
"title": "## Documentation Enhancements",
"labels": ["doc"]
},
{
"title": "## Refactoring Efforts",
"labels": ["refactor"]
},
{
"title": "## Miscellaneus Changes",
"labels": []
}
],
"ignore_labels": [
"duplicate", "good first issue", "help wanted", "invalid", "question", "wontfix"
],
"label_extractor": [
{
"pattern": "(.) (.+)",
"target": "$1"
},
{
"pattern": "(.) (.+)",
"target": "$1",
"on_property": "title"
}
]
}
- name: Create Release
uses: softprops/action-gh-release@v2
with:
body: ${{steps.github_release.outputs.changelog}}

View File

@@ -82,8 +82,8 @@ goupdate:
buildall: buildall:
./mkrel.sh $(tool) $(VERSION) ./mkrel.sh $(tool) $(VERSION)
release: buildall release:
gh release create v$(VERSION) --generate-notes releases/* gh release create v$(VERSION) --generate-notes
show-versions: buildlocal show-versions: buildlocal
@echo "### kleingebaeck version:" @echo "### kleingebaeck version:"

View File

@@ -17,6 +17,15 @@ Anzeige gespeichert werden. In dem Verzeichnis wird eine Datei
`Adlisting.txt` erstellt, in der sich die Inhalte der Anzeige wie `Adlisting.txt` erstellt, in der sich die Inhalte der Anzeige wie
Titel, Preis, Text etc befinden. Bilder werden natürlich auch heruntergeladen. Titel, Preis, Text etc befinden. Bilder werden natürlich auch heruntergeladen.
## ACHTUNG - SICHERHEITS-UPDATE
Fertige vorcompilierte Programme älter als Version `v0.3.12` sind von
Schwachstellen in der Behandlung von HTTP und Zertifikaten
betroffen. Falls Du eine ältere Kleingebäck-Version im Einsatz hast,
bitte update auf Version `v0.3.12` oder höher. Bitte lies auch die [Release Notes für
v0.3.12](https://github.com/TLINDEN/kleingebaeck/releases/tag/v0.3.12)
für mehr Details.
## Screenshots ## Screenshots
Das ist die Hauptseite meines kleinanzeigen.de Accounts: Das ist die Hauptseite meines kleinanzeigen.de Accounts:

View File

@@ -18,6 +18,15 @@ directory, each ad into its own subdirectory. The backup will contain
a textfile `Adlisting.txt` which contains the ad contents as the a textfile `Adlisting.txt` which contains the ad contents as the
title, body, price etc. All images will be downloaded as well. title, body, price etc. All images will be downloaded as well.
## CAUTION - SECURITY UPDATE
Binary releases prior to version `v0.3.11` are affected by
vulnerabilities in HTTP and certificate handling. If you are using
such a binary, please update to `v0.3.12` or higher. Please also refer
to the [Release Notes of
v0.3.12](https://github.com/TLINDEN/kleingebaeck/releases/tag/v0.3.12)
for more details.
## Screenshots ## Screenshots
This is the index of my kleinanzeigen.de Account: This is the index of my kleinanzeigen.de Account:
@@ -195,6 +204,7 @@ Price: 99 € VB
Id: 1919191919 Id: 1919191919
Category: Sachbücher Category: Sachbücher
Condition: Sehr Gut Condition: Sehr Gut
Type: Buch
Created: 10.12.2023 Created: 10.12.2023
This is the description text. This is the description text.

75
ad.go
View File

@@ -1,5 +1,5 @@
/* /*
Copyright © 2023-2024 Thomas von Dein Copyright © 2023-2025 Thomas von Dein
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
@@ -18,6 +18,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package main package main
import ( import (
"bufio"
"log/slog" "log/slog"
"strings" "strings"
"time" "time"
@@ -31,7 +32,12 @@ type Ad struct {
Title string `goquery:"h1"` Title string `goquery:"h1"`
Slug string Slug string
ID string ID string
Condition string `goquery:".addetailslist--detail--value,text"` Details string `goquery:".addetailslist--detail,text"`
Attributes map[string]string // processed afterwards
Condition string // post processed from details for backward compatibility
Type string // post processed from details for backward compatibility
Color string // post processed from details for backward compatibility
Material string // post processed from details for backward compatibility
Category string Category string
CategoryTree []string `goquery:".breadcrump-link,text"` CategoryTree []string `goquery:".breadcrump-link,text"`
Price string `goquery:"h2#viewad-price"` Price string `goquery:"h2#viewad-price"`
@@ -39,6 +45,9 @@ type Ad struct {
Text string `goquery:"p#viewad-description-text,html"` Text string `goquery:"p#viewad-description-text,html"`
Images []string `goquery:".galleryimage-element img,[src]"` Images []string `goquery:".galleryimage-element img,[src]"`
Expire string Expire string
// runtime computed
Year, Day, Month string
} }
// Used by slog to pretty print an ad // Used by slog to pretty print an ad
@@ -50,7 +59,6 @@ func (ad *Ad) LogValue() slog.Value {
slog.Int("imagecount", len(ad.Images)), slog.Int("imagecount", len(ad.Images)),
slog.Int("bodysize", len(ad.Text)), slog.Int("bodysize", len(ad.Text)),
slog.String("categorytree", strings.Join(ad.CategoryTree, "+")), slog.String("categorytree", strings.Join(ad.CategoryTree, "+")),
slog.String("condition", ad.Condition),
slog.String("created", ad.Created), slog.String("created", ad.Created),
slog.String("expire", ad.Expire), slog.String("expire", ad.Expire),
) )
@@ -80,3 +88,64 @@ func (ad *Ad) CalculateExpire() {
} }
} }
} }
/*
Decode attributes like color or condition. See
https://github.com/TLINDEN/kleingebaeck/issues/117
for more details. In short: the HTML delivered by
kleinanzeigen.de has no css attribute for the keys
so we cannot extract key=>value mappings of the
ad details but have to parse them manually.
The ad.Details member contains this after goq run:
Art
Weitere Kinderzimmermöbel
Farbe
Holz
Zustand
In Ordnung
We parse this into ad.Attributes and fill in some
static members for backward compatibility reasons.
*/
func (ad *Ad) DecodeAttributes() {
rd := strings.NewReader(ad.Details)
scanner := bufio.NewScanner(rd)
isattr := true
attr := ""
attrmap := map[string]string{}
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" {
continue
}
if isattr {
attr = line
} else {
attrmap[attr] = line
}
isattr = !isattr
}
ad.Attributes = attrmap
switch {
case Exists(ad.Attributes, "Zustand"):
ad.Condition = ad.Attributes["Zustand"]
case Exists(ad.Attributes, "Farbe"):
ad.Color = ad.Attributes["Farbe"]
case Exists(ad.Attributes, "Art"):
ad.Type = ad.Attributes["Type"]
case Exists(ad.Attributes, "Material"):
ad.Material = ad.Attributes["Material"]
}
}

View File

@@ -1,5 +1,5 @@
/* /*
Copyright © 2023-2024 Thomas von Dein Copyright © 2023-2025 Thomas von Dein
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
@@ -34,17 +34,25 @@ import (
) )
const ( const (
VERSION string = "0.3.7" VERSION string = "0.3.18"
Baseuri string = "https://www.kleinanzeigen.de" Baseuri string = "https://www.kleinanzeigen.de"
Listuri string = "/s-bestandsliste.html" Listuri string = "/s-bestandsliste.html"
Defaultdir string = "." Defaultdir string = "."
/*
Also possible: loop through .Attributes:
DefaultTemplate string = "Title: {{.Title}}\nPrice: {{.Price}}\nId: {{.ID}}\n" +
"Category: {{.Category}}\n{{ range $key,$val := .Attributes }}{{ $key }}: {{ $val }}\n{{ end }}" +
"Created: {{.Created}}\nExpire: {{.Expire}}\n\n{{.Text}}\n"
*/
DefaultTemplate string = "Title: {{.Title}}\nPrice: {{.Price}}\nId: {{.ID}}\n" + DefaultTemplate string = "Title: {{.Title}}\nPrice: {{.Price}}\nId: {{.ID}}\n" +
"Category: {{.Category}}\nCondition: {{.Condition}}\n" + "Category: {{.Category}}\nCondition: {{.Condition}}\nType: {{.Type}}\nColor: {{.Color}}\n" +
"Created: {{.Created}}\nExpire: {{.Expire}}\n\n{{.Text}}\n" "Created: {{.Created}}\nExpire: {{.Expire}}\n\n{{.Text}}\n"
DefaultTemplateWin string = "Title: {{.Title}}\r\nPrice: {{.Price}}\r\nId: {{.ID}}\r\n" + DefaultTemplateWin string = "Title: {{.Title}}\r\nPrice: {{.Price}}\r\nId: {{.ID}}\r\n" +
"Category: {{.Category}}\r\nCondition: {{.Condition}}\r\n" + "Category: {{.Category}}\r\nCondition: {{.Condition}}\r\nType: {{.Type}}\r\nColor: {{.Color}}\r\n" +
"Created: {{.Created}}\r\nExpires: {{.Expire}}\r\n\r\n{{.Text}}\r\n" "Created: {{.Created}}\r\nExpires: {{.Expire}}\r\n\r\n{{.Text}}\r\n"
DefaultUserAgent string = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + DefaultUserAgent string = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " +
@@ -183,13 +191,22 @@ func InitConfig(output io.Writer) (*Config, error) {
// Load the config file[s] // Load the config file[s]
for _, cfgfile := range configfiles { for _, cfgfile := range configfiles {
if path, err := os.Stat(cfgfile); !os.IsNotExist(err) { path, err := os.Stat(cfgfile)
if !path.IsDir() {
if err := kloader.Load(file.Provider(cfgfile), toml.Parser()); err != nil { if err != nil {
return nil, fmt.Errorf("error loading config file: %w", err) // ignore non-existent files, but bail out on any other errors
} if !os.IsNotExist(err) {
return nil, fmt.Errorf("failed to stat config file: %w", err)
} }
} // else: we ignore the file if it doesn't exists
continue
}
if !path.IsDir() {
if err := kloader.Load(file.Provider(cfgfile), toml.Parser()); err != nil {
return nil, fmt.Errorf("error loading config file: %w", err)
}
}
} }
// env overrides config file // env overrides config file

View File

@@ -12,19 +12,36 @@ user = 00000000
loglevel = "verbose" loglevel = "verbose"
# directory where to store downloaded ads. kleingebaeck will try to # directory where to store downloaded ads. kleingebaeck will try to
# create it. must be a quoted string. # create it. must be a quoted string. You can also include a couple of
outdir = "test" # template variables, e.g:
# outdir = "test-{{.Year}}-{{.Month}}-{{.Day}}"
outdir = "test"
# template for stored adlistings. To enable it, remove the comment # template for stored adlistings.
# chars up until the last #""" template="""
#template=""" Title: {{.Title}}
#Title: {{.Title}} Price: {{.Price}}
#Price: {{.Price}} Id: {{.Id}}
#Id: {{.Id}} Category: {{.Category}}
#Category: {{.Category}} Condition: {{.Condition}}
#Condition: {{.Condition}} Type: {{.Type}}
#Created: {{.Created}} Created: {{.Created}}
#{{.Text}} {{.Text}}
# """ """
# Ads may contain more attributes than just the Condition. To print
# all attributes, loop over all of them:
template="""
Title: {{.Title}}
Price: {{.Price}}
Id: {{.Id}}
Category: {{.Category}}
{{ range $key,$val := .Attributes }}{{ $key }}: {{ $val }}
{{ end }}
Type: {{.Type}}
Created: {{.Created}}
{{.Text}}
"""

20
go.mod
View File

@@ -9,31 +9,33 @@ require (
github.com/jarcoal/httpmock v1.3.1 github.com/jarcoal/httpmock v1.3.1
github.com/knadh/koanf/parsers/toml v0.1.0 github.com/knadh/koanf/parsers/toml v0.1.0
github.com/knadh/koanf/providers/confmap v0.1.0 github.com/knadh/koanf/providers/confmap v0.1.0
github.com/knadh/koanf/providers/env v0.1.0 github.com/knadh/koanf/providers/env v1.0.0
github.com/knadh/koanf/providers/file v1.1.0 github.com/knadh/koanf/providers/file v1.1.2
github.com/knadh/koanf/providers/posflag v0.1.0 github.com/knadh/koanf/providers/posflag v0.1.0
github.com/knadh/koanf/v2 v2.1.1 github.com/knadh/koanf/v2 v2.1.2
github.com/lmittmann/tint v1.0.5 github.com/lmittmann/tint v1.0.7
github.com/mattn/go-isatty v0.0.20 github.com/mattn/go-isatty v0.0.20
github.com/spf13/pflag v1.0.5 github.com/spf13/pflag v1.0.6
github.com/tlinden/yadu v0.1.3 github.com/tlinden/yadu v0.1.3
golang.org/x/sync v0.8.0 golang.org/x/image v0.23.0
golang.org/x/sync v0.10.0
) )
require ( require (
github.com/PuerkitoBio/goquery v1.5.1 // indirect github.com/PuerkitoBio/goquery v1.5.1 // indirect
github.com/alecthomas/repr v0.4.0 // indirect
github.com/andybalholm/cascadia v1.1.0 // indirect github.com/andybalholm/cascadia v1.1.0 // indirect
github.com/fatih/color v1.16.0 // indirect github.com/fatih/color v1.16.0 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 // indirect github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
github.com/knadh/koanf/maps v0.1.1 // indirect github.com/knadh/koanf/maps v0.1.1 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-colorable v0.1.14 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml v1.9.5 // indirect
golang.org/x/net v0.23.0 // indirect golang.org/x/net v0.23.0 // indirect
golang.org/x/sys v0.21.0 // indirect golang.org/x/sys v0.29.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect
) )

47
go.sum
View File

@@ -3,6 +3,8 @@ astuart.co/goq v1.0.0/go.mod h1:+fokcnFrO8Pw2fj8drdStJvzoMFebJH69rw8IC21rno=
github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE= github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
@@ -15,8 +17,8 @@ github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 h1:TQcrn6Wq+sKGkpyPvppOz99zsMBaUOKXq6HSv655U1c= github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss=
github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww= github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww=
@@ -27,18 +29,24 @@ github.com/knadh/koanf/parsers/toml v0.1.0 h1:S2hLqS4TgWZYj4/7mI5m1CQQcWurxUz6OD
github.com/knadh/koanf/parsers/toml v0.1.0/go.mod h1:yUprhq6eo3GbyVXFFMdbfZSo928ksS+uo0FFqNMnO18= github.com/knadh/koanf/parsers/toml v0.1.0/go.mod h1:yUprhq6eo3GbyVXFFMdbfZSo928ksS+uo0FFqNMnO18=
github.com/knadh/koanf/providers/confmap v0.1.0 h1:gOkxhHkemwG4LezxxN8DMOFopOPghxRVp7JbIvdvqzU= github.com/knadh/koanf/providers/confmap v0.1.0 h1:gOkxhHkemwG4LezxxN8DMOFopOPghxRVp7JbIvdvqzU=
github.com/knadh/koanf/providers/confmap v0.1.0/go.mod h1:2uLhxQzJnyHKfxG927awZC7+fyHFdQkd697K4MdLnIU= github.com/knadh/koanf/providers/confmap v0.1.0/go.mod h1:2uLhxQzJnyHKfxG927awZC7+fyHFdQkd697K4MdLnIU=
github.com/knadh/koanf/providers/env v0.1.0 h1:LqKteXqfOWyx5Ab9VfGHmjY9BvRXi+clwyZozgVRiKg= github.com/knadh/koanf/providers/env v1.0.0 h1:ufePaI9BnWH+ajuxGGiJ8pdTG0uLEUWC7/HDDPGLah0=
github.com/knadh/koanf/providers/env v0.1.0/go.mod h1:RE8K9GbACJkeEnkl8L/Qcj8p4ZyPXZIQ191HJi44ZaQ= github.com/knadh/koanf/providers/env v1.0.0/go.mod h1:mzFyRZueYhb37oPmC1HAv/oGEEuyvJDA98r3XAa8Gak=
github.com/knadh/koanf/providers/file v1.1.0 h1:MTjA+gRrVl1zqgetEAIaXHqYje0XSosxSiMD4/7kz0o= github.com/knadh/koanf/providers/file v1.1.2 h1:aCC36YGOgV5lTtAFz2qkgtWdeQsgfxUkxDOe+2nQY3w=
github.com/knadh/koanf/providers/file v1.1.0/go.mod h1:/faSBcv2mxPVjFrXck95qeoyoZ5myJ6uxN8OOVNJJCI= github.com/knadh/koanf/providers/file v1.1.2/go.mod h1:/faSBcv2mxPVjFrXck95qeoyoZ5myJ6uxN8OOVNJJCI=
github.com/knadh/koanf/providers/posflag v0.1.0 h1:mKJlLrKPcAP7Ootf4pBZWJ6J+4wHYujwipe7Ie3qW6U= github.com/knadh/koanf/providers/posflag v0.1.0 h1:mKJlLrKPcAP7Ootf4pBZWJ6J+4wHYujwipe7Ie3qW6U=
github.com/knadh/koanf/providers/posflag v0.1.0/go.mod h1:SYg03v/t8ISBNrMBRMlojH8OsKowbkXV7giIbBVgbz0= github.com/knadh/koanf/providers/posflag v0.1.0/go.mod h1:SYg03v/t8ISBNrMBRMlojH8OsKowbkXV7giIbBVgbz0=
github.com/knadh/koanf/v2 v2.1.1 h1:/R8eXqasSTsmDCsAyYj+81Wteg8AqrV9CP6gvsTsOmM= github.com/knadh/koanf/v2 v2.1.2 h1:I2rtLRqXRy1p01m/utEtpZSSA6dcJbgGVuE27kW2PzQ=
github.com/knadh/koanf/v2 v2.1.1/go.mod h1:4mnTRbZCK+ALuBXHZMjDfG9y714L7TykVnZkXbMU3Es= github.com/knadh/koanf/v2 v2.1.2/go.mod h1:Gphfaen0q1Fc1HTgJgSTC4oRX9R2R5ErYMZJy8fLJBo=
github.com/lmittmann/tint v1.0.5 h1:NQclAutOfYsqs2F1Lenue6OoWCajs5wJcP3DfWVpePw= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
github.com/lmittmann/tint v1.0.5/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/lmittmann/tint v1.0.7 h1:D/0OqWZ0YOGZ6AyC+5Y2kD8PBEzBk6rFHVSfOqCkF9Y=
github.com/lmittmann/tint v1.0.7/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
@@ -54,30 +62,35 @@ github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3v
github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/tlinden/yadu v0.1.3 h1:5cRCUmj+l5yvlM2irtpFBIJwVV2DPEgYSaWvF19FtcY= github.com/tlinden/yadu v0.1.3 h1:5cRCUmj+l5yvlM2irtpFBIJwVV2DPEgYSaWvF19FtcY=
github.com/tlinden/yadu v0.1.3/go.mod h1:l3bRmHKL9zGAR6pnBHY2HRPxBecf7L74BoBgOOpTcUA= github.com/tlinden/yadu v0.1.3/go.mod h1:l3bRmHKL9zGAR6pnBHY2HRPxBecf7L74BoBgOOpTcUA=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/image v0.23.0 h1:HseQ7c2OpPKTPVzNjG5fwJsOTCiiwS4QdsYi5XU6H68=
golang.org/x/image v0.23.0/go.mod h1:wJJBTdLfCCf3tiHa1fNxpZmUI4mmoZvwMCPP0ddoNKY=
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190606173856-1492cefac77f/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190606173856-1492cefac77f/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -20,11 +20,16 @@ package main
import ( import (
"bytes" "bytes"
"fmt" "fmt"
"image/jpeg" "image"
_ "image/gif"
_ "image/jpeg"
_ "image/png"
"log/slog" "log/slog"
"os" "os"
"path/filepath" "path/filepath"
_ "golang.org/x/image/webp"
"github.com/corona10/goimagehash" "github.com/corona10/goimagehash"
) )
@@ -35,6 +40,7 @@ type Image struct {
Hash *goimagehash.ImageHash Hash *goimagehash.ImageHash
Data *bytes.Reader Data *bytes.Reader
URI string URI string
Mime string
} }
// used for logging to avoid printing Data // used for logging to avoid printing Data
@@ -49,21 +55,52 @@ func (img *Image) LogValue() slog.Value {
// holds all images of an ad // holds all images of an ad
type Cache []*goimagehash.ImageHash type Cache []*goimagehash.ImageHash
func NewImage(buf *bytes.Reader, filename, uri string) *Image { // filename comes from the scraper, it contains directory/base w/o suffix
func NewImage(buf *bytes.Reader, filename, uri string) (*Image, error) {
_, imgconfig, err := image.DecodeConfig(buf)
if err != nil {
return nil, fmt.Errorf("failed to decode image: %w", err)
}
_, err = buf.Seek(0, 0)
if err != nil {
return nil, fmt.Errorf("failed to seek(0) on image buffer: %w", err)
}
if imgconfig == "jpeg" {
// we're using the format as file extension, but have used
// "jpg" in the past, so to be backwards compatible, stay with
// it.
imgconfig = "jpg"
}
if imgconfig == "" {
return nil, fmt.Errorf("failed to process image: unknown or unsupported image format (supported: jpg,png,gif,webp)")
}
filename += "." + imgconfig
img := &Image{ img := &Image{
Filename: filename, Filename: filename,
URI: uri, URI: uri,
Data: buf, Data: buf,
Mime: imgconfig,
} }
return img slog.Debug("image MIME", "mime", img.Mime)
return img, nil
} }
// Calculate diff hash of the image // Calculate diff hash of the image
func (img *Image) CalcHash() error { func (img *Image) CalcHash() error {
jpgdata, err := jpeg.Decode(img.Data) jpgdata, format, err := image.Decode(img.Data)
if err != nil { if err != nil {
return fmt.Errorf("failed to decode JPEG image: %w", err) return fmt.Errorf("failed to decode image: %w", err)
}
if format == "" {
return fmt.Errorf("failed to decode image: unknown or unsupported image format (supported: jpg,png,gif,webp)")
} }
hash1, err := goimagehash.DifferenceHash(jpgdata) hash1, err := goimagehash.DifferenceHash(jpgdata)
@@ -133,12 +170,19 @@ func ReadImages(addir string, dont bool) (Cache, error) {
reader := bytes.NewReader(data.Bytes()) reader := bytes.NewReader(data.Bytes())
img := NewImage(reader, filename, "") img, err := NewImage(reader, filename, "")
if err != nil {
return nil, err
}
if err := img.CalcHash(); err != nil { if err := img.CalcHash(); err != nil {
return nil, err return nil, err
} }
slog.Debug("Caching image from file system", "image", img, "hash", img.Hash.ToString()) if img.Hash != nil {
slog.Debug("Caching image from file system", "image", img, "hash", img.Hash.ToString())
}
cache = append(cache, img.Hash) cache = append(cache, img.Hash)
} }
} }

View File

@@ -133,7 +133,7 @@
.\" ======================================================================== .\" ========================================================================
.\" .\"
.IX Title "KLEINGEBAECK 1" .IX Title "KLEINGEBAECK 1"
.TH KLEINGEBAECK 1 "2024-02-10" "1" "User Commands" .TH KLEINGEBAECK 1 "2025-02-27" "1" "User Commands"
.\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" For nroff, turn off justification. Always turn off hyphenation; it makes
.\" way too many mistakes in technical documents. .\" way too many mistakes in technical documents.
.if n .ad l .if n .ad l
@@ -260,6 +260,29 @@ The ad directory name can be modified using the following ad values:
.PP .PP
It can only be configured in the config file. By default only It can only be configured in the config file. By default only
\&\f(CW\*(C`{{.Slug}}\*(C'\fR is being used, this is the title of the ad in url format. \&\f(CW\*(C`{{.Slug}}\*(C'\fR is being used, this is the title of the ad in url format.
.SS "\s-1AD NAME TEMPLATE\s0"
.IX Subsection "AD NAME TEMPLATE"
The name of the directory per ad can be tuned as well:
.ie n .IP """{{.Year}}""" 4
.el .IP "\f(CW{{.Year}}\fR" 4
.IX Item "{{.Year}}"
.PD 0
.ie n .IP """{{.Month}}""" 4
.el .IP "\f(CW{{.Month}}\fR" 4
.IX Item "{{.Month}}"
.ie n .IP """{{.Day}}""" 4
.el .IP "\f(CW{{.Day}}\fR" 4
.IX Item "{{.Day}}"
.ie n .IP """{{.Slug}}""" 4
.el .IP "\f(CW{{.Slug}}\fR" 4
.IX Item "{{.Slug}}"
.ie n .IP """{{.Category}}""" 4
.el .IP "\f(CW{{.Category}}\fR" 4
.IX Item "{{.Category}}"
.ie n .IP """{{.ID}}""" 4
.el .IP "\f(CW{{.ID}}\fR" 4
.IX Item "{{.ID}}"
.PD
.SS "\s-1AD TEMPLATE\s0" .SS "\s-1AD TEMPLATE\s0"
.IX Subsection "AD TEMPLATE" .IX Subsection "AD TEMPLATE"
The ad listing itself can be modified as well, using the same The ad listing itself can be modified as well, using the same
@@ -267,12 +290,13 @@ variables as the ad name template above.
.PP .PP
This is the default template: This is the default template:
.PP .PP
.Vb 7 .Vb 8
\& Title: {{.Title}} \& Title: {{.Title}}
\& Price: {{.Price}} \& Price: {{.Price}}
\& Id: {{.ID}} \& Id: {{.ID}}
\& Category: {{.Category}} \& Category: {{.Category}}
\& Condition: {{.Condition}} \& Condition: {{.Condition}}
\& Type: {{.Type}}
\& Created: {{.Created}} \& Created: {{.Created}}
\& Expire: {{.Expire}} \& Expire: {{.Expire}}
\& \&
@@ -343,7 +367,7 @@ Also there's currently no parallelization implemented. This will
change in the future. change in the future.
.SH "LICENSE" .SH "LICENSE"
.IX Header "LICENSE" .IX Header "LICENSE"
Copyright 2023\-2024 Thomas von Dein Copyright 2023\-2025 Thomas von Dein
.PP .PP
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the \s-1GNU\s0 General Public License as published by it under the terms of the \s-1GNU\s0 General Public License as published by

View File

@@ -100,6 +100,16 @@ TEMPLATES
It can only be configured in the config file. By default only It can only be configured in the config file. By default only
"{{.Slug}}" is being used, this is the title of the ad in url format. "{{.Slug}}" is being used, this is the title of the ad in url format.
AD NAME TEMPLATE
The name of the directory per ad can be tuned as well:
"{{.Year}}"
"{{.Month}}"
"{{.Day}}"
"{{.Slug}}"
"{{.Category}}"
"{{.ID}}"
AD TEMPLATE AD TEMPLATE
The ad listing itself can be modified as well, using the same variables The ad listing itself can be modified as well, using the same variables
as the ad name template above. as the ad name template above.
@@ -111,6 +121,7 @@ TEMPLATES
Id: {{.ID}} Id: {{.ID}}
Category: {{.Category}} Category: {{.Category}}
Condition: {{.Condition}} Condition: {{.Condition}}
Type: {{.Type}}
Created: {{.Created}} Created: {{.Created}}
Expire: {{.Expire}} Expire: {{.Expire}}
@@ -173,7 +184,7 @@ LIMITATIONS
in the future. in the future.
LICENSE LICENSE
Copyright 2023-2024 Thomas von Dein Copyright 2023-2025 Thomas von Dein
This program is free software: you can redistribute it and/or modify it This program is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by the under the terms of the GNU General Public License as published by the

View File

@@ -119,6 +119,27 @@ The ad directory name can be modified using the following ad values:
It can only be configured in the config file. By default only It can only be configured in the config file. By default only
C<{{.Slug}}> is being used, this is the title of the ad in url format. C<{{.Slug}}> is being used, this is the title of the ad in url format.
=head2 AD NAME TEMPLATE
The name of the directory per ad can be tuned as well:
=over
=item C<{{.Year}}>
=item C<{{.Month}}>
=item C<{{.Day}}>
=item C<{{.Slug}}>
=item C<{{.Category}}>
=item C<{{.ID}}>
=back
=head2 AD TEMPLATE =head2 AD TEMPLATE
The ad listing itself can be modified as well, using the same The ad listing itself can be modified as well, using the same
@@ -131,6 +152,7 @@ This is the default template:
Id: {{.ID}} Id: {{.ID}}
Category: {{.Category}} Category: {{.Category}}
Condition: {{.Condition}} Condition: {{.Condition}}
Type: {{.Type}}
Created: {{.Created}} Created: {{.Created}}
Expire: {{.Expire}} Expire: {{.Expire}}
@@ -201,7 +223,7 @@ change in the future.
=head1 LICENSE =head1 LICENSE
Copyright 2023-2024 Thomas von Dein Copyright 2023-2025 Thomas von Dein
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by

View File

@@ -93,6 +93,10 @@ const ADTPL string = `DOCTYPE html>
<li class="addetailslist--detail"> <li class="addetailslist--detail">
Zustand<span class="addetailslist--detail--value" > Zustand<span class="addetailslist--detail--value" >
{{ .Condition }}</span> {{ .Condition }}</span>
Farbe<span class="addetailslist--detail--value" >
{{ .Color }}</span>
Art<span class="addetailslist--detail--value" >
{{ .Type }}</span>
</li> </li>
</ul> </ul>
</div> </div>
@@ -182,13 +186,13 @@ var tests = []Tests{
{ {
name: "download-all-ads", name: "download-all-ads",
args: base + " -o t/out -u 1", args: base + " -o t/out -u 1",
expect: "Successfully downloaded 6 ads with 12 images to t/out", expect: "Successfully downloaded 7 ads with 16 images to t/out",
exitcode: 0, exitcode: 0,
}, },
{ {
name: "download-all-ads-using-config", name: "download-all-ads-using-config",
args: "kleingebaeck -c t/fullconfig.conf", args: "kleingebaeck -c t/fullconfig.conf",
expect: "Successfully downloaded 6 ads with 12 images to t/out", expect: "Successfully downloaded 7 ads with 16 images to t/out",
exitcode: 0, exitcode: 0,
}, },
} }
@@ -251,11 +255,14 @@ type AdConfig struct {
Price string Price string
Category string Category string
Condition string Condition string
Type string
Color string
Created string Created string
Text string Text string
Images []string // files in ./t/ Images []string // files in ./t/
} }
// used to generate ad listings returned by httpmock using templates
var adsrc = []AdConfig{ var adsrc = []AdConfig{
{ {
Title: "First Ad", Title: "First Ad",
@@ -263,7 +270,9 @@ var adsrc = []AdConfig{
Category: "Klimbim", Category: "Klimbim",
Text: "Thing to sale", Text: "Thing to sale",
Slug: "first-ad", Slug: "first-ad",
Condition: "works", Condition: "Sehr Gut",
Color: "Grün",
Type: "Ball",
Created: "Yesterday", Created: "Yesterday",
Images: []string{"t/1.jpg", "t/2.jpg"}, Images: []string{"t/1.jpg", "t/2.jpg"},
}, },
@@ -273,7 +282,9 @@ var adsrc = []AdConfig{
Category: "Kram", Category: "Kram",
Text: "Thing to sale", Text: "Thing to sale",
Slug: "second-ad", Slug: "second-ad",
Condition: "works", Condition: "Gut",
Color: "Lila",
Type: "Schoki",
Created: "Yesterday", Created: "Yesterday",
Images: []string{"t/1.jpg", "t/2.jpg"}, Images: []string{"t/1.jpg", "t/2.jpg"},
}, },
@@ -284,7 +295,9 @@ var adsrc = []AdConfig{
Category: "Kuddelmuddel", Category: "Kuddelmuddel",
Text: "Thing to sale", Text: "Thing to sale",
Slug: "third-ad", Slug: "third-ad",
Condition: "works", Condition: "In Ordnung",
Color: "Blau",
Type: "Auto",
Created: "Yesterday", Created: "Yesterday",
Images: []string{"t/1.jpg", "t/2.jpg"}, Images: []string{"t/1.jpg", "t/2.jpg"},
}, },
@@ -295,7 +308,9 @@ var adsrc = []AdConfig{
Category: "Krempel", Category: "Krempel",
Text: "Thing to sale", Text: "Thing to sale",
Slug: "fourth-ad", Slug: "fourth-ad",
Condition: "works", Condition: "Neu",
Color: "Rot",
Type: "Spielzeut",
Created: "Yesterday", Created: "Yesterday",
Images: []string{"t/1.jpg", "t/2.jpg"}, Images: []string{"t/1.jpg", "t/2.jpg"},
}, },
@@ -306,7 +321,9 @@ var adsrc = []AdConfig{
Category: "Kladderadatsch", Category: "Kladderadatsch",
Text: "Thing to sale", Text: "Thing to sale",
Slug: "fifth-ad", Slug: "fifth-ad",
Condition: "works", Condition: "Sehr Gut",
Color: "Braun",
Type: "Parteibuch",
Created: "Yesterday", Created: "Yesterday",
Images: []string{"t/1.jpg", "t/2.jpg"}, Images: []string{"t/1.jpg", "t/2.jpg"},
}, },
@@ -317,10 +334,25 @@ var adsrc = []AdConfig{
Category: "Klunker", Category: "Klunker",
Text: "Thing to sale", Text: "Thing to sale",
Slug: "sixth-ad", Slug: "sixth-ad",
Condition: "works", Condition: "Sehr Gut",
Color: "Silber",
Type: "Ring",
Created: "Yesterday", Created: "Yesterday",
Images: []string{"t/1.jpg", "t/2.jpg"}, Images: []string{"t/1.jpg", "t/2.jpg"},
}, },
{
Title: "Ad with multiple img formats",
ID: "7",
Price: "5€",
Category: "Klunker",
Text: "Thing to sale",
Slug: "seventh-ad",
Condition: "Sehr Gut",
Color: "Gelpb",
Type: "Schmuck",
Created: "Yesterday",
Images: []string{"t/1.png", "t/1.gif", "t/1.webp", "t/1.jpg"},
},
} }
// An Adsource is used to construct a httpmock responder for a // An Adsource is used to construct a httpmock responder for a
@@ -363,7 +395,7 @@ func InitValidSources() []Adsource {
// valid ad listing page 2 // valid ad listing page 2
list2 := []AdConfig{ list2 := []AdConfig{
adsrc[3], adsrc[4], adsrc[5], adsrc[3], adsrc[4], adsrc[5], adsrc[6],
} }
// valid ad listing page 3, which is empty // valid ad listing page 3, which is empty
@@ -459,7 +491,7 @@ func SetIntercept(ads []Adsource) {
} }
// we just use 2 images, put this here // we just use 2 images, put this here
for _, image := range []string{"t/1.jpg", "t/2.jpg"} { for _, image := range []string{"t/1.jpg", "t/2.jpg", "t/1.png", "t/1.gif", "t/1.webp"} {
httpmock.RegisterResponder("GET", image, httpmock.RegisterResponder("GET", image,
httpmock.NewBytesResponder(200, GetImage(image)).HeaderAdd(headers)) httpmock.NewBytesResponder(200, GetImage(image)).HeaderAdd(headers))
} }

View File

@@ -45,15 +45,21 @@ for D in $DIST; do
os=${D/\/*/} os=${D/\/*/}
arch=${D/*\//} arch=${D/*\//}
binfile="releases/${tool}-${os}-${arch}-${version}" binfile="releases/${tool}-${os}-${arch}-${version}"
pie=""
if test "$os" = "windows"; then if test "$os" = "windows"; then
binfile="${binfile}.exe" binfile="${binfile}.exe"
fi fi
if test "$D" = "linux/amd64"; then
pie="-buildmode=pie"
fi
tardir="${tool}-${os}-${arch}-${version}" tardir="${tool}-${os}-${arch}-${version}"
tarfile="releases/${tool}-${os}-${arch}-${version}.tar.gz" tarfile="releases/${tool}-${os}-${arch}-${version}.tar.gz"
set -x set -x
GOOS=${os} GOARCH=${arch} go build -tags osusergo,netgo -ldflags "-extldflags=-static" -o ${binfile} GOOS=${os} GOARCH=${arch} go build -tags osusergo,netgo -ldflags "-extldflags=-static -w" --trimpath $pie -o ${binfile}
strip --strip-all ${binfile}
mkdir -p ${tardir} mkdir -p ${tardir}
cp ${binfile} README.md LICENSE ${tardir}/ cp ${binfile} README.md LICENSE ${tardir}/
echo 'tool = kleingebaeck echo 'tool = kleingebaeck

View File

@@ -1,5 +1,5 @@
/* /*
Copyright © 2023-2024 Thomas von Dein Copyright © 2023-2025 Thomas von Dein
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
@@ -88,7 +88,13 @@ func ScrapeUser(fetch *Fetcher) error {
// scrape an ad. uri is the full uri of the ad, dir is the basedir // scrape an ad. uri is the full uri of the ad, dir is the basedir
func ScrapeAd(fetch *Fetcher, uri string) error { func ScrapeAd(fetch *Fetcher, uri string) error {
advertisement := &Ad{} now := time.Now()
advertisement := &Ad{
Year: now.Format("2006"),
Month: now.Format("01"),
Day: now.Format("02"),
}
// extract slug and id from uri // extract slug and id from uri
uriparts := strings.Split(uri, "/") uriparts := strings.Split(uri, "/")
@@ -124,6 +130,7 @@ func ScrapeAd(fetch *Fetcher, uri string) error {
return fmt.Errorf("could not extract ad data from page, got empty struct") return fmt.Errorf("could not extract ad data from page, got empty struct")
} }
advertisement.DecodeAttributes()
advertisement.CalculateExpire() advertisement.CalculateExpire()
// prepare ad dir name // prepare ad dir name
@@ -170,7 +177,9 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error {
for _, imguri := range advertisement.Images { for _, imguri := range advertisement.Images {
imguri := imguri imguri := imguri
file := filepath.Join(adpath, fmt.Sprintf("%d.jpg", img))
// we append the suffix later in NewImage() based on image format
basefilename := filepath.Join(adpath, fmt.Sprintf("%d", img))
egroup.Go(func() error { egroup.Go(func() error {
// wait a little // wait a little
@@ -192,7 +201,11 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error {
reader := bytes.NewReader(buf.Bytes()) reader := bytes.NewReader(buf.Bytes())
image := NewImage(reader, file, imguri) image, err := NewImage(reader, basefilename, imguri)
if err != nil {
return err
}
err = image.CalcHash() err = image.CalcHash()
if err != nil { if err != nil {
return err return err
@@ -211,7 +224,7 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error {
return fmt.Errorf("failed to seek(0) on image reader: %w", err) return fmt.Errorf("failed to seek(0) on image reader: %w", err)
} }
err = WriteImage(file, reader) err = WriteImage(image.Filename, reader)
if err != nil { if err != nil {
return err return err
} }

View File

@@ -1,5 +1,5 @@
/* /*
Copyright © 2023-2024 Thomas von Dein Copyright © 2023-2025 Thomas von Dein
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
@@ -44,8 +44,8 @@ func OutDirName(conf *Config) (string, error) {
now := time.Now() now := time.Now()
data := OutdirData{ data := OutdirData{
Year: now.Format("2006"), Year: now.Format("2006"),
Month: now.Format("02"), Month: now.Format("01"),
Day: now.Format("01"), Day: now.Format("02"),
} }
err = tmpl.Execute(&buf, data) err = tmpl.Execute(&buf, data)
@@ -149,7 +149,9 @@ func ReadImage(filename string) (*bytes.Buffer, error) {
func fileExists(filename string) bool { func fileExists(filename string) bool {
info, err := os.Stat(filename) info, err := os.Stat(filename)
if os.IsNotExist(err) {
if err != nil {
// return false on any error
return false return false
} }

BIN
t/1.gif Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 62 B

BIN
t/1.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.2 KiB

BIN
t/1.webp Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.3 KiB