mirror of
https://codeberg.org/scip/kleingebaeck.git
synced 2025-12-16 20:11:01 +01:00
Compare commits
90 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b44c1ba1dc | ||
| b6350344be | |||
| e5ff2bbc5c | |||
|
|
230bbf3d53 | ||
|
|
b3b1aac63e | ||
|
|
ef68a8f36b | ||
|
|
734d94291b | ||
|
|
f90158c1e1 | ||
|
|
09184861e3 | ||
|
|
2a520c648d | ||
| 287b9995fe | |||
|
|
0b53e95e5e | ||
|
|
e95bdadb12 | ||
|
|
9959ad2468 | ||
|
|
8da83085c8 | ||
|
|
fdbec2f393 | ||
|
|
25bf602bba | ||
|
|
5bfabe8fd8 | ||
|
|
063aa8ba86 | ||
|
|
0654150d83 | ||
|
|
519d6f350a | ||
|
|
3c071d0447 | ||
|
|
cd9d79eda7 | ||
|
|
89b7f551d8 | ||
|
|
85bfb0be00 | ||
|
|
212c338ec9 | ||
| dd45dc996b | |||
| 1d5aa50423 | |||
|
|
15b2e7f2a7 | ||
|
|
4030d04b06 | ||
| eff0af0b34 | |||
| 34b1ad9d1e | |||
|
|
6675c4d232 | ||
|
|
46be48af38 | ||
|
|
09948a6b39 | ||
|
|
bc01391872 | ||
| cd3d00adbe | |||
| 528ecdd43d | |||
|
|
5cb928518d | ||
|
|
d8c7409c7a | ||
|
|
1cd6eb5134 | ||
| 9e2983a85c | |||
| 6eddd08e4a | |||
| 34dfc25e87 | |||
|
|
0bc6a0ae59 | ||
|
|
14c554563a | ||
| 475a9a2fd3 | |||
| fbd9a5a621 | |||
| 7014c97dee | |||
| 91edfeb19a | |||
| 3b3435515c | |||
| 2239a83f76 | |||
| 31b27beee5 | |||
|
|
a4be51f498 | ||
|
|
6b5af984cc | ||
|
|
f5d3853388 | ||
| 07ebd7afad | |||
|
|
5e82881b69 | ||
| c9a75e3f91 | |||
|
|
2fd2028cbe | ||
|
|
766f35d1d5 | ||
| e4edde082b | |||
|
|
6b95a01591 | ||
|
|
2f66758c9f | ||
|
|
f21b47e14f | ||
|
|
14f1c89f05 | ||
|
|
b7530843c5 | ||
|
|
44c5e40466 | ||
|
|
7b4dd50ebf | ||
|
|
5a6fe0f2fe | ||
| d7f5fbbe58 | |||
| ee550b1ed7 | |||
| e35815f27f | |||
| 9138770d6e | |||
| 834fbcd9c5 | |||
| 4c10ae89f8 | |||
| df6fc47ca3 | |||
| 5956a68e72 | |||
| eee0167574 | |||
| a3b2748479 | |||
| 8cc5a9e3ed | |||
| d2bcd7b505 | |||
| c59c2e2931 | |||
| 2288806105 | |||
| 5a2c277f0e | |||
| 612ed2aa79 | |||
| ed78731b3c | |||
| a84f0e1436 | |||
| d8d5be5c7d | |||
| bcf920c91e |
96
.gh-dash.yml
Normal file
96
.gh-dash.yml
Normal file
@@ -0,0 +1,96 @@
|
||||
prSections:
|
||||
- title: Responsible PRs
|
||||
filters: repo:tlinden/kleingebaeck is:open NOT dependabot
|
||||
layout:
|
||||
repoName:
|
||||
hidden: true
|
||||
|
||||
- title: Responsible Dependabot PRs
|
||||
filters: repo:tlinden/kleingebaeck is:open dependabot
|
||||
layout:
|
||||
repoName:
|
||||
hidden: true
|
||||
|
||||
issuesSections:
|
||||
- title: Responsible Issues
|
||||
filters: is:open repo:tlinden/kleingebaeck -author:@me
|
||||
layout:
|
||||
repoName:
|
||||
hidden: true
|
||||
|
||||
- title: Note-to-Self Issues
|
||||
filters: is:open repo:tlinden/kleingebaeck author:@me
|
||||
layout:
|
||||
creator:
|
||||
hidden: true
|
||||
repoName:
|
||||
hidden: true
|
||||
|
||||
defaults:
|
||||
preview:
|
||||
open: false
|
||||
width: 100
|
||||
|
||||
keybindings:
|
||||
universal:
|
||||
- key: "shift+down"
|
||||
builtin: pageDown
|
||||
- key: "shift+up"
|
||||
builtin: pageUp
|
||||
prs:
|
||||
- key: g
|
||||
name: gitu
|
||||
command: >
|
||||
cd {{.RepoPath}} && /home/scip/bin/gitu
|
||||
- key: M
|
||||
name: squash-merge
|
||||
command: gh pr merge --rebase --squash --admin --repo {{.RepoName}} {{.PrNumber}}
|
||||
- key: i
|
||||
name: show ci checks
|
||||
command: gh pr checks --repo {{.RepoName}} {{.PrNumber}} | glow -p
|
||||
- key: e
|
||||
name: edit pr
|
||||
command: ~/.config/gh-dash/edit-gh-pr {{.RepoName}} {{.PrNumber}}
|
||||
- key: E
|
||||
name: open repo in emacs
|
||||
command: emacsclient {{.RepoPath}} &
|
||||
issues:
|
||||
- key: v
|
||||
name: view
|
||||
command: gh issue view --repo {{.RepoName}} {{.IssueNumber}} | glow -p
|
||||
- key: l
|
||||
name: add label
|
||||
command: gh issue --repo {{.RepoName}} edit {{.IssueNumber}} --add-label $(gum choose bug enhancement question dependencies wontfix)
|
||||
- key: L
|
||||
name: remove label
|
||||
command: gh issue --repo {{.RepoName}} edit {{.IssueNumber}} --remove-label $(gum choose bug enhancement question dependencies wontfix)
|
||||
- key: E
|
||||
name: open repo in emacs
|
||||
command: emacsclient {{.RepoPath}} &
|
||||
|
||||
theme:
|
||||
ui:
|
||||
sectionsShowCount: true
|
||||
table:
|
||||
compact: false
|
||||
showSeparator: true
|
||||
colors:
|
||||
text:
|
||||
primary: "#E2E1ED"
|
||||
secondary: "#6770cb"
|
||||
inverted: "#242347"
|
||||
faint: "#b0793b"
|
||||
warning: "#E0AF68"
|
||||
success: "#3DF294"
|
||||
background:
|
||||
selected: "#1B1B33"
|
||||
border:
|
||||
primary: "#383B5B"
|
||||
secondary: "#39386B"
|
||||
faint: "#8d3e0b"
|
||||
|
||||
repoPaths:
|
||||
:owner/:repo: ~/dev/:repo
|
||||
|
||||
pager:
|
||||
diff: delta
|
||||
1
.github/ISSUE_TEMPLATE/note_to_self.md
vendored
1
.github/ISSUE_TEMPLATE/note_to_self.md
vendored
@@ -5,3 +5,4 @@ title: "[bug-report]"
|
||||
labels: bug
|
||||
assignees: TLINDEN
|
||||
|
||||
---
|
||||
|
||||
BIN
.github/assets/kleingebaecklogo.xcf
vendored
Normal file
BIN
.github/assets/kleingebaecklogo.xcf
vendored
Normal file
Binary file not shown.
10
.github/dependabot.yml
vendored
Normal file
10
.github/dependabot.yml
vendored
Normal file
@@ -0,0 +1,10 @@
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: "gomod"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
47
.github/workflows/ci.yaml
vendored
47
.github/workflows/ci.yaml
vendored
@@ -1,47 +0,0 @@
|
||||
name: build-and-test
|
||||
on: [push, pull_request]
|
||||
jobs:
|
||||
build:
|
||||
strategy:
|
||||
matrix:
|
||||
version: [1.21]
|
||||
os: [ubuntu-latest, windows-latest, macos-latest]
|
||||
name: Build
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: ${{ matrix.version }}
|
||||
id: go
|
||||
|
||||
- name: checkout
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: build
|
||||
run: go build
|
||||
|
||||
- name: test
|
||||
run: make test
|
||||
|
||||
- name: Update coverage report
|
||||
uses: ncruces/go-coverage-report@main
|
||||
with:
|
||||
report: true
|
||||
chart: true
|
||||
amend: true
|
||||
if: |
|
||||
matrix.os == 'ubuntu-latest' &&
|
||||
github.event_name == 'push'
|
||||
continue-on-error: true
|
||||
|
||||
golangci:
|
||||
name: lint
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: 1.21
|
||||
- uses: actions/checkout@v3
|
||||
- name: golangci-lint
|
||||
uses: golangci/golangci-lint-action@v3
|
||||
34
.github/workflows/pushimage.yaml
vendored
34
.github/workflows/pushimage.yaml
vendored
@@ -1,34 +0,0 @@
|
||||
name: build-push-image
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- 'v*'
|
||||
|
||||
jobs:
|
||||
build-and-push-image:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
packages: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Log in to the Container registry
|
||||
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
|
||||
with:
|
||||
registry: https://ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
|
||||
with:
|
||||
push: true
|
||||
tags: ghcr.io/tlinden/kleingebaeck:${{ github.ref_name}}
|
||||
|
||||
- name: Build and push latest Docker image
|
||||
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
|
||||
with:
|
||||
push: true
|
||||
tags: ghcr.io/tlinden/kleingebaeck:latest
|
||||
69
.goreleaser.yaml
Normal file
69
.goreleaser.yaml
Normal file
@@ -0,0 +1,69 @@
|
||||
# vim: set ts=2 sw=2 tw=0 fo=cnqoj
|
||||
|
||||
version: 2
|
||||
|
||||
before:
|
||||
hooks:
|
||||
- go mod tidy
|
||||
|
||||
gitea_urls:
|
||||
api: https://codeberg.org/api/v1
|
||||
download: https://codeberg.org
|
||||
|
||||
builds:
|
||||
- env:
|
||||
- CGO_ENABLED=0
|
||||
goos:
|
||||
- linux
|
||||
- windows
|
||||
- darwin
|
||||
- freebsd
|
||||
|
||||
archives:
|
||||
- formats: [tar.gz]
|
||||
# this name template makes the OS and Arch compatible with the results of `uname`.
|
||||
name_template: >-
|
||||
{{ .ProjectName }}_
|
||||
{{- title .Os }}_
|
||||
{{- if eq .Arch "amd64" }}x86_64
|
||||
{{- else if eq .Arch "386" }}i386
|
||||
{{- else }}{{ .Arch }}{{ end }}
|
||||
{{- if .Arm }}v{{ .Arm }}{{ end }}_{{ .Tag }}
|
||||
# use zip for windows archives
|
||||
format_overrides:
|
||||
- goos: windows
|
||||
formats: [zip]
|
||||
- goos: linux
|
||||
formats: [tar.gz,binary]
|
||||
files:
|
||||
- src: "*.md"
|
||||
strip_parent: true
|
||||
- src: "docs/*"
|
||||
strip_parent: true
|
||||
- src: Makefile.dist
|
||||
dst: Makefile
|
||||
wrap_in_directory: true
|
||||
|
||||
changelog:
|
||||
sort: asc
|
||||
filters:
|
||||
exclude:
|
||||
- "^docs:"
|
||||
- "^test:"
|
||||
groups:
|
||||
- title: Improved
|
||||
regexp: '^.*?(feat|add|new)(\([[:word:]]+\))??!?:.+$'
|
||||
order: 0
|
||||
- title: Fixed
|
||||
regexp: '^.*?(bug|fix)(\([[:word:]]+\))??!?:.+$'
|
||||
order: 1
|
||||
- title: Changed
|
||||
order: 999
|
||||
|
||||
release:
|
||||
header: "# Release Notes"
|
||||
footer: >-
|
||||
|
||||
---
|
||||
|
||||
Full Changelog: [{{ .PreviousTag }}...{{ .Tag }}](https://codeberg.org/scip/kleingebaeck/compare/{{ .PreviousTag }}...{{ .Tag }})
|
||||
36
.woodpecker/build.yaml
Normal file
36
.woodpecker/build.yaml
Normal file
@@ -0,0 +1,36 @@
|
||||
matrix:
|
||||
platform:
|
||||
- linux/amd64
|
||||
goversion:
|
||||
- 1.24
|
||||
|
||||
labels:
|
||||
platform: ${platform}
|
||||
|
||||
steps:
|
||||
build:
|
||||
when:
|
||||
event: [push]
|
||||
image: golang:${goversion}
|
||||
commands:
|
||||
- go get
|
||||
- go build
|
||||
|
||||
linter:
|
||||
when:
|
||||
event: [push]
|
||||
image: golang:${goversion}
|
||||
commands:
|
||||
- curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.5.0
|
||||
- golangci-lint --version
|
||||
- golangci-lint run ./...
|
||||
depends_on: [build]
|
||||
|
||||
test:
|
||||
when:
|
||||
event: [push]
|
||||
image: golang:${goversion}
|
||||
commands:
|
||||
- go get
|
||||
- go test -v -cover
|
||||
depends_on: [build,linter]
|
||||
32
.woodpecker/image.yaml
Normal file
32
.woodpecker/image.yaml
Normal file
@@ -0,0 +1,32 @@
|
||||
# https://woodpecker-ci.org/plugins/docker-buildx
|
||||
# enable Package unit and go to /scip/-/packages after building to link to proj
|
||||
|
||||
variables:
|
||||
- &repo codeberg.org/${CI_REPO_OWNER}/kleingebaeck
|
||||
|
||||
steps:
|
||||
dryrun:
|
||||
image: docker.io/woodpeckerci/plugin-docker-buildx:latest
|
||||
settings:
|
||||
dockerfile: Dockerfile
|
||||
platforms: linux/amd64
|
||||
dry_run: true
|
||||
repo: *repo
|
||||
tags: latest
|
||||
when:
|
||||
event: [pull_request]
|
||||
|
||||
publish:
|
||||
image: docker.io/woodpeckerci/plugin-docker-buildx:latest
|
||||
settings:
|
||||
dockerfile: Dockerfile
|
||||
platforms: linux/amd64
|
||||
repo: *repo
|
||||
registry: codeberg.org
|
||||
tags: latest,${CI_COMMIT_SHA:0:8},${CI_COMMIT_TAG}
|
||||
username: ${CI_REPO_OWNER}
|
||||
password:
|
||||
from_secret: REGISTRY_TOKEN
|
||||
when:
|
||||
event: [tag]
|
||||
branch: main
|
||||
15
.woodpecker/release.yaml
Normal file
15
.woodpecker/release.yaml
Normal file
@@ -0,0 +1,15 @@
|
||||
# build release
|
||||
|
||||
labels:
|
||||
platform: linux/amd64
|
||||
|
||||
steps:
|
||||
goreleaser:
|
||||
image: goreleaser/goreleaser
|
||||
when:
|
||||
event: [tag]
|
||||
environment:
|
||||
GITEA_TOKEN:
|
||||
from_secret: DEPLOY_TOKEN
|
||||
commands:
|
||||
- goreleaser release --clean --verbose
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.21-alpine as builder
|
||||
FROM golang:1.24-alpine as builder
|
||||
|
||||
RUN apk update
|
||||
RUN apk upgrade
|
||||
|
||||
5
Makefile
5
Makefile
@@ -63,6 +63,7 @@ lint:
|
||||
|
||||
lint-full:
|
||||
golangci-lint run --enable-all --exclude-use-default --disable exhaustivestruct,exhaustruct,depguard,interfacer,deadcode,golint,structcheck,scopelint,varcheck,ifshort,maligned,nosnakecase,godot,funlen,gofumpt,cyclop,noctx,gochecknoglobals,paralleltest
|
||||
gocritic check -enableAll *.go
|
||||
|
||||
testfuzzy: clean
|
||||
go test -fuzz ./... $(ARGS)
|
||||
@@ -81,8 +82,8 @@ goupdate:
|
||||
buildall:
|
||||
./mkrel.sh $(tool) $(VERSION)
|
||||
|
||||
release: buildall
|
||||
gh release create v$(VERSION) --generate-notes releases/*
|
||||
release:
|
||||
gh release create v$(VERSION) --generate-notes
|
||||
|
||||
show-versions: buildlocal
|
||||
@echo "### kleingebaeck version:"
|
||||
|
||||
20
Makefile.dist
Normal file
20
Makefile.dist
Normal file
@@ -0,0 +1,20 @@
|
||||
# -*-make-*-
|
||||
|
||||
.PHONY: install all
|
||||
|
||||
tool = rpn
|
||||
PREFIX = /usr/local
|
||||
UID = root
|
||||
GID = 0
|
||||
|
||||
all:
|
||||
@echo "Type 'sudo make install' to install the tool."
|
||||
@echo "To change prefix, type 'sudo make install PREFIX=/opt'"
|
||||
|
||||
install:
|
||||
install -d -o $(UID) -g $(GID) $(PREFIX)/bin
|
||||
install -d -o $(UID) -g $(GID) $(PREFIX)/man/man1
|
||||
install -d -o $(UID) -g $(GID) $(PREFIX)/share/doc
|
||||
install -o $(UID) -g $(GID) -m 555 $(tool) $(PREFIX)/sbin/
|
||||
install -o $(UID) -g $(GID) -m 444 $(tool).1 $(PREFIX)/man/man1/
|
||||
install -o $(UID) -g $(GID) -m 444 *.md $(PREFIX)/share/doc/
|
||||
87
README-de.md
87
README-de.md
@@ -1,13 +1,12 @@
|
||||
## Kleingebäck - kleinanzeigen.de Backup
|
||||
|
||||

|
||||

|
||||
|
||||
[](https://goreportcard.com/report/github.com/tlinden/kleingebaeck)
|
||||
[](https://github.com/tlinden/kleingebaeck/actions)
|
||||
[](https://raw.githack.com/wiki/tlinden/kleingebaeck/coverage.html)
|
||||
[](https://goreportcard.com/report/codeberg.org/scip/kleingebaeck)
|
||||
[](https://ci.codeberg.org/repos/15530)
|
||||

|
||||
[](https://github.com/TLINDEN/kleingebaeck/releases/latest)
|
||||
[](https://github.com/tlinden/kleingebaeck/blob/main/README.md)
|
||||
[](https://codeberg.org/scip/kleingebaeck/releases)
|
||||
[](https://codeberg.org/scip/kleingebaeck/raw/branch/main/README.md)
|
||||
|
||||
Mit diesem Tool kann man seine Anzeigen bei https://kleinanzeigen.de sichern.
|
||||
|
||||
@@ -17,37 +16,46 @@ Anzeige gespeichert werden. In dem Verzeichnis wird eine Datei
|
||||
`Adlisting.txt` erstellt, in der sich die Inhalte der Anzeige wie
|
||||
Titel, Preis, Text etc befinden. Bilder werden natürlich auch heruntergeladen.
|
||||
|
||||
## ACHTUNG - SICHERHEITS-UPDATE
|
||||
|
||||
Fertige vorcompilierte Programme älter als Version `v0.3.12` sind von
|
||||
Schwachstellen in der Behandlung von HTTP und Zertifikaten
|
||||
betroffen. Falls Du eine ältere Kleingebäck-Version im Einsatz hast,
|
||||
bitte update auf Version `v0.3.12` oder höher. Bitte lies auch die [Release Notes für
|
||||
v0.3.12](https://codeberg.org/scip/kleingebaeck/releases/tag/v0.3.12)
|
||||
für mehr Details.
|
||||
|
||||
## Screenshots
|
||||
|
||||
Das ist die Hauptseite meines kleinanzeigen.de Accounts:
|
||||
|
||||

|
||||

|
||||
|
||||
Sichern ich meine Anzeigen:
|
||||
|
||||

|
||||

|
||||
|
||||
Backupverzeichnis nach dem Download:
|
||||
|
||||

|
||||

|
||||
|
||||
Verzeichnis einer Anzeige:
|
||||
|
||||

|
||||

|
||||
|
||||
**Das gleiche unter Windows:**
|
||||
|
||||
Anzeigen Sichern:
|
||||
|
||||

|
||||

|
||||
|
||||
Backupverzeichnis nach dem Download
|
||||
|
||||

|
||||

|
||||
|
||||
Und eine Anzeige:
|
||||
|
||||

|
||||

|
||||
|
||||
## Installation
|
||||
|
||||
@@ -57,7 +65,7 @@ für seine Plattform herunter und kann direkt loslegen.
|
||||
|
||||
### Installation des vorcompilierten Programms
|
||||
|
||||
Auf der Seite [des letzten Releases](https://github.com/TLINDEN/kleingebaeck/releases/latest) findet man das Program für sein Betriebssystem und die Plattform (z.b. Windows + Intel)
|
||||
Auf der Seite [des letzten Releases](https://codeberg.org/scip/kleingebaeck/releases) findet man das Program für sein Betriebssystem und die Plattform (z.b. Windows + Intel)
|
||||
|
||||
Es gibt 2 Varianten:
|
||||
|
||||
@@ -96,7 +104,7 @@ hilfreich, aber nicht unbedingt erforderlich.
|
||||
Um das Programm zu compilieren, muss man folgende Schritte ausführen:
|
||||
|
||||
```shell
|
||||
git clone https://github.com/TLINDEN/kleingebaeck.git
|
||||
git clone https://codeberg.org/scip/kleingebaeck.git
|
||||
cd kleingebaeck
|
||||
go mod tidy
|
||||
make # (oder make)
|
||||
@@ -137,7 +145,7 @@ gemountet.
|
||||
Die Optionen `-u XXX -v` sind kleingebäck Optionen. Ersetze `XXX`
|
||||
durch Deine tatsächliche kleinanzeigen.de Userid.
|
||||
|
||||
Eine Liste verfügbarer Images findet man [hier](https://github.com/tlinden/kleingebaeck/pkgs/container/kleingebaeck/versions?filters%5Bversion_type%5D=tagged)
|
||||
Eine Liste verfügbarer Images findet man [hier](https://codeberg.org/scip/kleingebaeck/pkgs/container/kleingebaeck/versions?filters%5Bversion_type%5D=tagged)
|
||||
|
||||
## Kommandozeilen Optionen:
|
||||
|
||||
@@ -222,10 +230,53 @@ Sowie alle Bilder.
|
||||
Das Format kann man mit der Variable `template` in der Konfiguration
|
||||
ändern. Die `example.conf` enthält ein Beispiel für das Standard Template.
|
||||
|
||||
## Verhalten des Tools
|
||||
|
||||
Es gibt einige Dinge über das Verhalten von kleingebäck, über die Du
|
||||
Bescheid wissen solltest:
|
||||
|
||||
- alle HTML Seiten und Bilder werden immer heruntergeladen
|
||||
- es wird ein (konfigurierbarer) Useragent verwendet
|
||||
- HTTP Cookies werden beachtet
|
||||
- bei Fehlern wird dreimal mit unterschiedlichem Abstand erneut
|
||||
versucht
|
||||
- Bilder Downloads laufen parallelisiert mit leicht unterschiedlichen
|
||||
zeitlichen Abständen ab
|
||||
- Gleich aussehende Bilder werden nicht überschrieben
|
||||
|
||||
Der letzte Punkt muss genauer erläutert werden:
|
||||
|
||||
Wenn man bei Kleinanzeigen.de eine Anzeige einstellt und Bilder
|
||||
postet, werden diese dort in ihrer Grösse reduziert (durch Kompression
|
||||
und Verkleinerung der Bilder usw.). Diese reduzierten Bilder werden
|
||||
dann von kleingebäck heruntergeladen. Falls Du Deine original Bilder
|
||||
behalten hast, kannst Du diese danach in das Backupverzeichnis
|
||||
kopieren. Bei einem erneuten kleingebäck-Lauf werden diese Bilder dann
|
||||
nicht überschrieben.
|
||||
|
||||
Wir verwenden dafür einen Algorythmus namens [distance
|
||||
hashing](https://github.com/corona10/goimagehash). Dieser Algorithmus
|
||||
prüft die Ähnlichkeit von Bildern. Diese können in ihrer Auflösung,
|
||||
Kompression, Farbtiefe und vielem mehr manipuliert worden sein und
|
||||
trotzdem als das "gleiche Bild" erkannt werden (wohlgemerkt nicht "das
|
||||
selbe": die Dateien sind durchaus unterschiedlich!). Bis zu einer
|
||||
Distance von 5 überschreiben wir keine Bilder, weil wir dann davon
|
||||
ausgehen, dass das lokal Vorhandene das Original ist.
|
||||
|
||||
Bitte beachte aber, dass dies KEIN Cachingmechanismus ist: die Bilder
|
||||
werden trotzdem immer alle heruntergeladen. Das muss so sein, da wir
|
||||
uns nicht die Dateinamen anschauen können, da kleinanzeigen.de diese
|
||||
nämlich zu Zahlen umbenennt. Und die Dateinamen können sich auch
|
||||
ändern, wenn der User in der Anzeige die Bilder umarrangiert hat.
|
||||
|
||||
Du kannst dieses Verhalten mit der Option **--force** ausschalten. Du
|
||||
kannst ausserdem mit der Option **--ignoreerrors** auch alle Fehler
|
||||
ignorieren, die beim Bilderdownload auftreten könnten.
|
||||
|
||||
## Documentation
|
||||
|
||||
Die Dokumentation kann man
|
||||
[online](https://github.com/TLINDEN/kleingebaeck/blob/main/kleingebaeck.pod)
|
||||
[online](https://codeberg.org/scip/kleingebaeck/raw/branch/main/kleingebaeck.pod)
|
||||
oder lokal lesen mit: `kleingebaeck --manual`. Hat man das Tool mit
|
||||
dem Tarball installiert, funktioniert auch `man kleingebaeck`.
|
||||
|
||||
@@ -241,7 +292,7 @@ das doch der beste Weg, die Anfrage zu übersehen und zu vergessen.
|
||||
|
||||
Um einen Fehler, ein unerwartetes Verhalten, eine Feature Request oder
|
||||
einen Patch zu übermitteln, eröffne daher bitte einen Issue unter:
|
||||
https://github.com/TLINDEN/kleingebaeck/issues. Danke!
|
||||
https://codeberg.org/scip/kleingebaeck/issues. Danke!
|
||||
|
||||
Bitte gebe den fehlgeschlagenen Befehl an, rufe es auch mit Debugging
|
||||
`-d` auf.
|
||||
|
||||
86
README.md
86
README.md
@@ -1,13 +1,12 @@
|
||||
## Kleingebäck - kleinanzeigen.de Backup
|
||||
|
||||

|
||||

|
||||
|
||||
[](https://goreportcard.com/report/github.com/tlinden/kleingebaeck)
|
||||
[](https://github.com/tlinden/kleingebaeck/actions)
|
||||
[](https://raw.githack.com/wiki/tlinden/kleingebaeck/coverage.html)
|
||||
[](https://goreportcard.com/report/codeberg.org/scip/kleingebaeck)
|
||||
[](https://ci.codeberg.org/repos/15530)
|
||||

|
||||
[](https://github.com/TLINDEN/kleingebaeck/releases/latest)
|
||||
[](https://github.com/tlinden/kleingebaeck/blob/main/README-de.md)
|
||||
[](https://codeberg.org/scip/kleingebaeck/releases)
|
||||
[](https://codeberg.org/scip/kleingebaeck/raw/branch/main/README-de.md)
|
||||
|
||||
[Die deutsche Version des READMEs findet Ihr hier](README-de.md).
|
||||
|
||||
@@ -18,37 +17,46 @@ directory, each ad into its own subdirectory. The backup will contain
|
||||
a textfile `Adlisting.txt` which contains the ad contents as the
|
||||
title, body, price etc. All images will be downloaded as well.
|
||||
|
||||
## CAUTION - SECURITY UPDATE
|
||||
|
||||
Binary releases prior to version `v0.3.11` are affected by
|
||||
vulnerabilities in HTTP and certificate handling. If you are using
|
||||
such a binary, please update to `v0.3.12` or higher. Please also refer
|
||||
to the [Release Notes of
|
||||
v0.3.12](https://codeberg.org/scip/kleingebaeck/releases/tag/v0.3.12)
|
||||
for more details.
|
||||
|
||||
## Screenshots
|
||||
|
||||
This is the index of my kleinanzeigen.de Account:
|
||||
|
||||

|
||||

|
||||
|
||||
Here I download my ads on the commandline:
|
||||
|
||||

|
||||

|
||||
|
||||
And this is the backup directory after download:
|
||||
|
||||

|
||||

|
||||
|
||||
Here's a directory for one ad:
|
||||
|
||||

|
||||

|
||||
|
||||
**The same thing under windows:**
|
||||
|
||||
Downloading ads:
|
||||
|
||||

|
||||

|
||||
|
||||
Backup directory after download:
|
||||
|
||||

|
||||

|
||||
|
||||
And one ad listing directory:
|
||||
|
||||

|
||||

|
||||
|
||||
## Installation
|
||||
|
||||
@@ -59,7 +67,7 @@ releases page and you're good to go.
|
||||
### Installation using a pre-compiled binary
|
||||
|
||||
Go to the [latest release
|
||||
page](https://github.com/TLINDEN/kleingebaeck/releases/latest) and
|
||||
page](https://codeberg.org/scip/kleingebaeck/releases) and
|
||||
look for your OS and platform. There are two options to install the binary:
|
||||
|
||||
1. Directly download the binary for your platform,
|
||||
@@ -129,7 +137,7 @@ docker run. And the local directory `myads` will be mapped to
|
||||
The options `-u XXX -v` are kleingebaeck options, replace `XXX` with
|
||||
your actual kleinanzeigen.de user id.
|
||||
|
||||
A list of available images is [here](https://github.com/tlinden/kleingebaeck/pkgs/container/kleingebaeck/versions?filters%5Bversion_type%5D=tagged)
|
||||
A list of available images is [here](https://codeberg.org/scip/kleingebaeck/pkgs/container/kleingebaeck/versions?filters%5Bversion_type%5D=tagged)
|
||||
|
||||
## Commandline options:
|
||||
|
||||
@@ -192,9 +200,11 @@ somewhat like this:
|
||||
```default
|
||||
Title: A book I sell
|
||||
Price: 99 € VB
|
||||
Shipping: 6,90 €
|
||||
Id: 1919191919
|
||||
Category: Sachbücher
|
||||
Condition: Sehr Gut
|
||||
Type: Buch
|
||||
Created: 10.12.2023
|
||||
|
||||
This is the description text.
|
||||
@@ -207,9 +217,51 @@ variable. The supplied sample config contains the default template.
|
||||
|
||||
All images will be stored in the same directory.
|
||||
|
||||
## Tool Behavior
|
||||
|
||||
There are a bunch of things you might want to know about the behavior
|
||||
of the kleingebäck tool:
|
||||
|
||||
- all HTML pages and IMAGEs are always being downloaded
|
||||
- we use a (customizable) user agent
|
||||
- we respect HTTP cookies
|
||||
- in the case of an error, the tool does 3 retries, the time it waits
|
||||
between tries is longer for each retry
|
||||
- image download is parallized using small time differences to look
|
||||
more natural
|
||||
- same images are not being overwritten on subsequent download
|
||||
|
||||
|
||||
The latter needs to be elaborated a bit more:
|
||||
|
||||
If you publish an ad on kleinanzeigen.de and post images, those images
|
||||
will be reduced in size by the site (by compressing and down sizing
|
||||
them). This reduced images will be downloaded by kleingebäck. However,
|
||||
you may still own the original images and may want to put them into
|
||||
that backup directory so that you have all things for one ad together.
|
||||
|
||||
You can easily do that, because kleingebäck won't overwrite those
|
||||
original images. It uses something called a distance hash using
|
||||
[goimagehash](https://github.com/corona10/goimagehash). This
|
||||
algorithmus checks the similarity of images. If an image has been
|
||||
resized it is still very similar to the original one. We accept a
|
||||
maximum of a distance of 5, everything above leads to overwrite.
|
||||
|
||||
This works with resizes, cropped and otherwise manipulated images as
|
||||
long as the image still shows the original contents good enough.
|
||||
|
||||
Also note, that this is NOT a caching mechanism: the images will be
|
||||
downloaded anyway during each run. We also can't look at the file
|
||||
names because kleinanzeigen.de renames all images to numbers. And
|
||||
those might even change if the user re-arranges the images.
|
||||
|
||||
You can override this behavior using the **--force** option. Another
|
||||
option, **--ignoreerrors**, can be used to ignore all kinds of image
|
||||
errors.
|
||||
|
||||
## Documentation
|
||||
|
||||
You can read the documentation [online](https://github.com/TLINDEN/kleingebaeck/blob/main/kleingebaeck.pod) or locally once you have installed kleingebaeck with: `kleingebaeck --manual`.
|
||||
You can read the documentation [online](https://codeberg.org/scip/kleingebaeck/raw/branch/main/kleingebaeck.pod) or locally once you have installed kleingebaeck with: `kleingebaeck --manual`.
|
||||
|
||||
## Kleingebäck?
|
||||
|
||||
@@ -225,7 +277,7 @@ that's the best way for me to forget to do something.
|
||||
|
||||
In order to report a bug, unexpected behavior, feature requests or to
|
||||
submit a patch, please open an issue on github:
|
||||
https://github.com/TLINDEN/kleingebaeck/issues.
|
||||
https://codeberg.org/scip/kleingebaeck/issues.
|
||||
|
||||
Please repeat the failing command with debugging enabled `-d` and
|
||||
include the output in the issue.
|
||||
|
||||
17
SECURITY.md
Normal file
17
SECURITY.md
Normal file
@@ -0,0 +1,17 @@
|
||||
# Security Policy
|
||||
|
||||
## Supported Versions
|
||||
|
||||
Only the latest release is supported. If you find an issue (any
|
||||
issue!), please check with the latest release first.
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
I don't agree with the "responsible disclosure" process most projects
|
||||
(and companies) work these days.
|
||||
|
||||
So, if you find a vulnerability of any kind, please just open an
|
||||
[issue](https://codeberg.org/scip/kleingebaeck/issues). Please add
|
||||
all details required to reproduce the vulnerability. You won't be chased.
|
||||
|
||||
That's just all about it.
|
||||
91
ad.go
91
ad.go
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
Copyright © 2023 Thomas von Dein
|
||||
Copyright © 2023-2025 Thomas von Dein
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
@@ -18,6 +18,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -31,14 +32,23 @@ type Ad struct {
|
||||
Title string `goquery:"h1"`
|
||||
Slug string
|
||||
ID string
|
||||
Condition string `goquery:".addetailslist--detail--value,text"`
|
||||
Details string `goquery:".addetailslist--detail,text"`
|
||||
Attributes map[string]string // processed afterwards
|
||||
Condition string // post processed from details for backward compatibility
|
||||
Type string // post processed from details for backward compatibility
|
||||
Color string // post processed from details for backward compatibility
|
||||
Material string // post processed from details for backward compatibility
|
||||
Category string
|
||||
CategoryTree []string `goquery:".breadcrump-link,text"`
|
||||
Price string `goquery:"h2#viewad-price"`
|
||||
Created string `goquery:"#viewad-extra-info,text"`
|
||||
Text string `goquery:"p#viewad-description-text,html"`
|
||||
Images []string `goquery:".galleryimage-element img,[src]"`
|
||||
Shipping string `goquery:".boxedarticle--details--shipping,text"` // not always filled
|
||||
Expire string
|
||||
|
||||
// runtime computed
|
||||
Year, Day, Month string
|
||||
}
|
||||
|
||||
// Used by slog to pretty print an ad
|
||||
@@ -50,9 +60,10 @@ func (ad *Ad) LogValue() slog.Value {
|
||||
slog.Int("imagecount", len(ad.Images)),
|
||||
slog.Int("bodysize", len(ad.Text)),
|
||||
slog.String("categorytree", strings.Join(ad.CategoryTree, "+")),
|
||||
slog.String("condition", ad.Condition),
|
||||
slog.String("created", ad.Created),
|
||||
slog.String("expire", ad.Expire),
|
||||
slog.String("shipping", ad.Shipping),
|
||||
slog.String("details", ad.Details),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -73,10 +84,80 @@ func (ad *Ad) Incomplete() bool {
|
||||
}
|
||||
|
||||
func (ad *Ad) CalculateExpire() {
|
||||
if len(ad.Created) > 0 {
|
||||
if ad.Created != "" {
|
||||
ts, err := time.Parse("02.01.2006", ad.Created)
|
||||
if err == nil {
|
||||
ad.Expire = ts.AddDate(0, ExpireMonths, ExpireDays).Format("02.01.2006")
|
||||
ad.Expire = ts.AddDate(0, 0, ExpireDays).Format("02.01.2006")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
Decode attributes like color or condition. See
|
||||
https://codeberg.org/scip/kleingebaeck/issues/117
|
||||
for more details. In short: the HTML delivered by
|
||||
kleinanzeigen.de has no css attribute for the keys
|
||||
so we cannot extract key=>value mappings of the
|
||||
ad details but have to parse them manually.
|
||||
|
||||
The ad.Details member contains this after goq run:
|
||||
|
||||
Art
|
||||
|
||||
Weitere Kinderzimmermöbel
|
||||
|
||||
Farbe
|
||||
Holz
|
||||
|
||||
Zustand
|
||||
In Ordnung
|
||||
|
||||
We parse this into ad.Attributes and fill in some
|
||||
static members for backward compatibility reasons.
|
||||
*/
|
||||
func (ad *Ad) DecodeAttributes() {
|
||||
rd := strings.NewReader(ad.Details)
|
||||
scanner := bufio.NewScanner(rd)
|
||||
|
||||
isattr := true
|
||||
attr := ""
|
||||
attrmap := map[string]string{}
|
||||
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
if isattr {
|
||||
attr = line
|
||||
} else {
|
||||
attrmap[attr] = line
|
||||
}
|
||||
|
||||
isattr = !isattr
|
||||
}
|
||||
|
||||
ad.Attributes = attrmap
|
||||
|
||||
if Exists(ad.Attributes, "Zustand") {
|
||||
ad.Condition = ad.Attributes["Zustand"]
|
||||
}
|
||||
|
||||
if Exists(ad.Attributes, "Farbe") {
|
||||
ad.Color = ad.Attributes["Farbe"]
|
||||
}
|
||||
|
||||
if Exists(ad.Attributes, "Art") {
|
||||
ad.Type = ad.Attributes["Art"]
|
||||
}
|
||||
|
||||
if Exists(ad.Attributes, "Material") {
|
||||
ad.Material = ad.Attributes["Material"]
|
||||
}
|
||||
|
||||
slog.Debug("parsed attributes", "attributes", ad.Attributes)
|
||||
|
||||
ad.Shipping = strings.Replace(ad.Shipping, "+ Versand ab ", "", 1)
|
||||
}
|
||||
|
||||
66
config.go
66
config.go
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
Copyright © 2023-2024 Thomas von Dein
|
||||
Copyright © 2023-2025 Thomas von Dein
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
@@ -34,24 +34,34 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
VERSION string = "0.3.2"
|
||||
VERSION string = "0.3.24"
|
||||
Baseuri string = "https://www.kleinanzeigen.de"
|
||||
Listuri string = "/s-bestandsliste.html"
|
||||
Defaultdir string = "."
|
||||
|
||||
DefaultTemplate string = "Title: {{.Title}}\nPrice: {{.Price}}\nId: {{.ID}}\n" +
|
||||
"Category: {{.Category}}\nCondition: {{.Condition}}\n" +
|
||||
/*
|
||||
Also possible: loop through .Attributes:
|
||||
|
||||
DefaultTemplate string = "Title: {{.Title}}\nPrice: {{.Price}}\nId: {{.ID}}\n" +
|
||||
"Category: {{.Category}}\n{{ range $key,$val := .Attributes }}{{ $key }}: {{ $val }}\n{{ end }}" +
|
||||
"Created: {{.Created}}\nExpire: {{.Expire}}\n\n{{.Text}}\n"
|
||||
|
||||
*/
|
||||
DefaultTemplate string = "Title: {{.Title}}\nPrice: {{.Price}}\nShipping: {{.Shipping}}\nId: {{.ID}}\n" +
|
||||
"Category: {{.Category}}\nCondition: {{.Condition}}\nType: {{.Type}}\nColor: {{.Color}}\n" +
|
||||
"Created: {{.Created}}\nExpire: {{.Expire}}\n\n{{.Text}}\n"
|
||||
|
||||
DefaultTemplateWin string = "Title: {{.Title}}\r\nPrice: {{.Price}}\r\nId: {{.ID}}\r\n" +
|
||||
"Category: {{.Category}}\r\nCondition: {{.Condition}}\r\n" +
|
||||
DefaultTemplateWin string = "Title: {{.Title}}\r\nPrice: {{.Price}}\r\nShipping: {{.Shipping}}\r\nId: {{.ID}}\r\n" +
|
||||
"Category: {{.Category}}\r\nCondition: {{.Condition}}\r\nType: {{.Type}}\r\nColor: {{.Color}}\r\n" +
|
||||
"Created: {{.Created}}\r\nExpires: {{.Expire}}\r\n\r\n{{.Text}}\r\n"
|
||||
|
||||
DefaultUserAgent string = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " +
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36"
|
||||
|
||||
DefaultAdNameTemplate string = "{{.Slug}}"
|
||||
|
||||
DefaultOutdirTemplate string = "."
|
||||
|
||||
// for image download throttling
|
||||
MinThrottle int = 2
|
||||
MaxThrottle int = 20
|
||||
@@ -59,12 +69,18 @@ const (
|
||||
// we extract the slug from the uri
|
||||
SlugURIPartNum int = 6
|
||||
|
||||
ExpireMonths int = 2
|
||||
ExpireDays int = 1
|
||||
// We have to calculate the ad expiry because the real value can
|
||||
// only be seen by logged in users. The initial ad lifetime is 120
|
||||
// days. It can be extended by the user 8 days before expire by 60
|
||||
// days. But this is unknown to us, so we'll stick with our 120
|
||||
// days. They may be wrong for older ads. Don't rely on it!
|
||||
ExpireDays int = 120
|
||||
|
||||
WIN string = "windows"
|
||||
)
|
||||
|
||||
var DirsVisited map[string]int
|
||||
|
||||
const Usage string = `This is kleingebaeck, the kleinanzeigen.de backup tool.
|
||||
|
||||
Usage: kleingebaeck [-dvVhmoclu] [<ad-listing-url>,...]
|
||||
@@ -77,7 +93,7 @@ Options:
|
||||
-l --limit <num> Limit the ads to download to <num>, default: load all.
|
||||
-c --config <file> Use config file <file> (default: ~/.kleingebaeck).
|
||||
--ignoreerrors Ignore HTTP errors, may lead to incomplete ad backup.
|
||||
-f --force Download images even if they already exist.
|
||||
-f --force Overwrite images and ads even if the already exist.
|
||||
-m --manual Show manual.
|
||||
-h --help Show usage.
|
||||
-V --version Show program version.
|
||||
@@ -126,7 +142,7 @@ func InitConfig(output io.Writer) (*Config, error) {
|
||||
// Load default values using the confmap provider.
|
||||
if err := kloader.Load(confmap.Provider(map[string]interface{}{
|
||||
"template": template,
|
||||
"outdir": ".",
|
||||
"outdir": DefaultOutdirTemplate,
|
||||
"loglevel": "notice",
|
||||
"userid": 0,
|
||||
"adnametemplate": DefaultAdNameTemplate,
|
||||
@@ -138,7 +154,11 @@ func InitConfig(output io.Writer) (*Config, error) {
|
||||
// setup custom usage
|
||||
flagset := flag.NewFlagSet("config", flag.ContinueOnError)
|
||||
flagset.Usage = func() {
|
||||
fmt.Fprintln(output, Usage)
|
||||
_, err := fmt.Fprintln(output, Usage)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
@@ -153,6 +173,7 @@ func InitConfig(output io.Writer) (*Config, error) {
|
||||
flagset.BoolP("help", "h", false, "show usage")
|
||||
flagset.BoolP("manual", "m", false, "show manual")
|
||||
flagset.BoolP("force", "f", false, "force")
|
||||
flagset.BoolP("ignoreerrors", "", false, "ignore image download HTTP errors")
|
||||
|
||||
if err := flagset.Parse(os.Args[1:]); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse program arguments: %w", err)
|
||||
@@ -178,13 +199,22 @@ func InitConfig(output io.Writer) (*Config, error) {
|
||||
|
||||
// Load the config file[s]
|
||||
for _, cfgfile := range configfiles {
|
||||
if path, err := os.Stat(cfgfile); !os.IsNotExist(err) {
|
||||
if !path.IsDir() {
|
||||
if err := kloader.Load(file.Provider(cfgfile), toml.Parser()); err != nil {
|
||||
return nil, fmt.Errorf("error loading config file: %w", err)
|
||||
}
|
||||
path, err := os.Stat(cfgfile)
|
||||
|
||||
if err != nil {
|
||||
// ignore non-existent files, but bail out on any other errors
|
||||
if !os.IsNotExist(err) {
|
||||
return nil, fmt.Errorf("failed to stat config file: %w", err)
|
||||
}
|
||||
} // else: we ignore the file if it doesn't exists
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if !path.IsDir() {
|
||||
if err := kloader.Load(file.Provider(cfgfile), toml.Parser()); err != nil {
|
||||
return nil, fmt.Errorf("error loading config file: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// env overrides config file
|
||||
|
||||
44
example.conf
44
example.conf
@@ -12,19 +12,37 @@ user = 00000000
|
||||
loglevel = "verbose"
|
||||
|
||||
# directory where to store downloaded ads. kleingebaeck will try to
|
||||
# create it. must be a quoted string.
|
||||
outdir = "test"
|
||||
# create it. must be a quoted string. You can also include a couple of
|
||||
# template variables, e.g:
|
||||
# outdir = "test-{{.Year}}-{{.Month}}-{{.Day}}"
|
||||
outdir = "test"
|
||||
|
||||
# template for stored adlistings. To enable it, remove the comment
|
||||
# chars up until the last #"""
|
||||
#template="""
|
||||
#Title: {{.Title}}
|
||||
#Price: {{.Price}}
|
||||
#Id: {{.Id}}
|
||||
#Category: {{.Category}}
|
||||
#Condition: {{.Condition}}
|
||||
#Created: {{.Created}}
|
||||
# template for stored adlistings.
|
||||
template="""
|
||||
Title: {{.Title}}
|
||||
Price: {{.Price}}
|
||||
Shipping: {{.Shipping}}
|
||||
Id: {{.Id}}
|
||||
Category: {{.Category}}
|
||||
Condition: {{.Condition}}
|
||||
Type: {{.Type}}
|
||||
Created: {{.Created}}
|
||||
|
||||
#{{.Text}}
|
||||
# """
|
||||
{{.Text}}
|
||||
"""
|
||||
|
||||
# Ads may contain more attributes than just the Condition. To print
|
||||
# all attributes, loop over all of them:
|
||||
|
||||
template="""
|
||||
Title: {{.Title}}
|
||||
Price: {{.Price}}
|
||||
Id: {{.Id}}
|
||||
Category: {{.Category}}
|
||||
{{ range $key,$val := .Attributes }}{{ $key }}: {{ $val }}
|
||||
{{ end }}
|
||||
Type: {{.Type}}
|
||||
Created: {{.Created}}
|
||||
|
||||
{{.Text}}
|
||||
"""
|
||||
|
||||
2
fetch.go
2
fetch.go
@@ -52,7 +52,7 @@ func NewFetcher(conf *Config) (*Fetcher, error) {
|
||||
}
|
||||
|
||||
func (f *Fetcher) Get(uri string) (io.ReadCloser, error) {
|
||||
req, err := http.NewRequest(http.MethodGet, uri, nil)
|
||||
req, err := http.NewRequest(http.MethodGet, uri, http.NoBody)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create a new HTTP request obj: %w", err)
|
||||
}
|
||||
|
||||
42
go.mod
42
go.mod
@@ -1,39 +1,41 @@
|
||||
module kleingebaeck
|
||||
|
||||
go 1.21
|
||||
go 1.24.0
|
||||
|
||||
toolchain go1.24.5
|
||||
|
||||
require (
|
||||
astuart.co/goq v1.0.0
|
||||
github.com/jarcoal/httpmock v1.3.1
|
||||
github.com/corona10/goimagehash v1.1.0
|
||||
github.com/inconshreveable/mousetrap v1.1.0
|
||||
github.com/jarcoal/httpmock v1.4.1
|
||||
github.com/knadh/koanf/parsers/toml v0.1.0
|
||||
github.com/knadh/koanf/providers/confmap v0.1.0
|
||||
github.com/knadh/koanf/providers/env v0.1.0
|
||||
github.com/knadh/koanf/providers/file v0.1.0
|
||||
github.com/knadh/koanf/providers/posflag v0.1.0
|
||||
github.com/knadh/koanf/v2 v2.0.1
|
||||
github.com/lmittmann/tint v1.0.4
|
||||
github.com/knadh/koanf/providers/confmap v1.0.0
|
||||
github.com/knadh/koanf/providers/env v1.1.0
|
||||
github.com/knadh/koanf/providers/file v1.2.0
|
||||
github.com/knadh/koanf/providers/posflag v1.0.1
|
||||
github.com/knadh/koanf/v2 v2.3.0
|
||||
github.com/lmittmann/tint v1.1.2
|
||||
github.com/mattn/go-isatty v0.0.20
|
||||
github.com/spf13/pflag v1.0.5
|
||||
github.com/tlinden/yadu v0.1.1
|
||||
golang.org/x/sync v0.5.0
|
||||
github.com/spf13/pflag v1.0.10
|
||||
github.com/tlinden/yadu v0.1.3
|
||||
golang.org/x/image v0.32.0
|
||||
golang.org/x/sync v0.17.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.5.1 // indirect
|
||||
github.com/andybalholm/cascadia v1.1.0 // indirect
|
||||
github.com/corona10/goimagehash v1.1.0 // indirect
|
||||
github.com/fatih/color v1.16.0 // indirect
|
||||
github.com/fsnotify/fsnotify v1.6.0 // indirect
|
||||
github.com/knadh/koanf/maps v0.1.1 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
||||
github.com/knadh/koanf/maps v0.1.2 // indirect
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/mitchellh/copystructure v1.2.0 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/mitchellh/reflectwalk v1.0.2 // indirect
|
||||
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect
|
||||
github.com/pelletier/go-toml v1.9.5 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect
|
||||
golang.org/x/sys v0.14.0 // indirect
|
||||
golang.org/x/net v0.38.0 // indirect
|
||||
golang.org/x/sys v0.32.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
|
||||
)
|
||||
|
||||
92
go.sum
92
go.sum
@@ -13,76 +13,76 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
|
||||
github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
|
||||
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
|
||||
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
|
||||
github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww=
|
||||
github.com/jarcoal/httpmock v1.3.1/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg=
|
||||
github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NIs=
|
||||
github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI=
|
||||
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
||||
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/jarcoal/httpmock v1.4.1 h1:0Ju+VCFuARfFlhVXFc2HxlcQkfB+Xq12/EotHko+x2A=
|
||||
github.com/jarcoal/httpmock v1.4.1/go.mod h1:ftW1xULwo+j0R0JJkJIIi7UKigZUXCLLanykgjwBXL0=
|
||||
github.com/knadh/koanf/maps v0.1.2 h1:RBfmAW5CnZT+PJ1CVc1QSJKf4Xu9kxfQgYVQSu8hpbo=
|
||||
github.com/knadh/koanf/maps v0.1.2/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI=
|
||||
github.com/knadh/koanf/parsers/toml v0.1.0 h1:S2hLqS4TgWZYj4/7mI5m1CQQcWurxUz6ODgOub/6LCI=
|
||||
github.com/knadh/koanf/parsers/toml v0.1.0/go.mod h1:yUprhq6eo3GbyVXFFMdbfZSo928ksS+uo0FFqNMnO18=
|
||||
github.com/knadh/koanf/providers/confmap v0.1.0 h1:gOkxhHkemwG4LezxxN8DMOFopOPghxRVp7JbIvdvqzU=
|
||||
github.com/knadh/koanf/providers/confmap v0.1.0/go.mod h1:2uLhxQzJnyHKfxG927awZC7+fyHFdQkd697K4MdLnIU=
|
||||
github.com/knadh/koanf/providers/env v0.1.0 h1:LqKteXqfOWyx5Ab9VfGHmjY9BvRXi+clwyZozgVRiKg=
|
||||
github.com/knadh/koanf/providers/env v0.1.0/go.mod h1:RE8K9GbACJkeEnkl8L/Qcj8p4ZyPXZIQ191HJi44ZaQ=
|
||||
github.com/knadh/koanf/providers/file v0.1.0 h1:fs6U7nrV58d3CFAFh8VTde8TM262ObYf3ODrc//Lp+c=
|
||||
github.com/knadh/koanf/providers/file v0.1.0/go.mod h1:rjJ/nHQl64iYCtAW2QQnF0eSmDEX/YZ/eNFj5yR6BvA=
|
||||
github.com/knadh/koanf/providers/posflag v0.1.0 h1:mKJlLrKPcAP7Ootf4pBZWJ6J+4wHYujwipe7Ie3qW6U=
|
||||
github.com/knadh/koanf/providers/posflag v0.1.0/go.mod h1:SYg03v/t8ISBNrMBRMlojH8OsKowbkXV7giIbBVgbz0=
|
||||
github.com/knadh/koanf/v2 v2.0.1 h1:1dYGITt1I23x8cfx8ZnldtezdyaZtfAuRtIFOiRzK7g=
|
||||
github.com/knadh/koanf/v2 v2.0.1/go.mod h1:ZeiIlIDXTE7w1lMT6UVcNiRAS2/rCeLn/GdLNvY1Dus=
|
||||
github.com/lmittmann/tint v1.0.4 h1:LeYihpJ9hyGvE0w+K2okPTGUdVLfng1+nDNVR4vWISc=
|
||||
github.com/lmittmann/tint v1.0.4/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE=
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/knadh/koanf/providers/confmap v1.0.0 h1:mHKLJTE7iXEys6deO5p6olAiZdG5zwp8Aebir+/EaRE=
|
||||
github.com/knadh/koanf/providers/confmap v1.0.0/go.mod h1:txHYHiI2hAtF0/0sCmcuol4IDcuQbKTybiB1nOcUo1A=
|
||||
github.com/knadh/koanf/providers/env v1.1.0 h1:U2VXPY0f+CsNDkvdsG8GcsnK4ah85WwWyJgef9oQMSc=
|
||||
github.com/knadh/koanf/providers/env v1.1.0/go.mod h1:QhHHHZ87h9JxJAn2czdEl6pdkNnDh/JS1Vtsyt65hTY=
|
||||
github.com/knadh/koanf/providers/file v1.2.0 h1:hrUJ6Y9YOA49aNu/RSYzOTFlqzXSCpmYIDXI7OJU6+U=
|
||||
github.com/knadh/koanf/providers/file v1.2.0/go.mod h1:bp1PM5f83Q+TOUu10J/0ApLBd9uIzg+n9UgthfY+nRA=
|
||||
github.com/knadh/koanf/providers/posflag v1.0.1 h1:EnMxHSrPkYCFnKgBUl5KBgrjed8gVFrcXDzaW4l/C6Y=
|
||||
github.com/knadh/koanf/providers/posflag v1.0.1/go.mod h1:3Wn3+YG3f4ljzRyCUgIwH7G0sZ1pMjCOsNBovrbKmAk=
|
||||
github.com/knadh/koanf/v2 v2.3.0 h1:Qg076dDRFHvqnKG97ZEsi9TAg2/nFTa9hCdcSa1lvlM=
|
||||
github.com/knadh/koanf/v2 v2.3.0/go.mod h1:gRb40VRAbd4iJMYYD5IxZ6hfuopFcXBpc9bbQpZwo28=
|
||||
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/lmittmann/tint v1.1.2 h1:2CQzrL6rslrsyjqLDwD11bZ5OpLBPU+g3G/r5LSfS8w=
|
||||
github.com/lmittmann/tint v1.1.2/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE=
|
||||
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
|
||||
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/maxatome/go-testdeep v1.12.0 h1:Ql7Go8Tg0C1D/uMMX59LAoYK7LffeJQ6X2T04nTH68g=
|
||||
github.com/maxatome/go-testdeep v1.12.0/go.mod h1:lPZc/HAcJMP92l7yI6TRz1aZN5URwUBUAfUNvrclaNM=
|
||||
github.com/maxatome/go-testdeep v1.14.0 h1:rRlLv1+kI8eOI3OaBXZwb3O7xY3exRzdW5QyX48g9wI=
|
||||
github.com/maxatome/go-testdeep v1.14.0/go.mod h1:lPZc/HAcJMP92l7yI6TRz1aZN5URwUBUAfUNvrclaNM=
|
||||
github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
|
||||
github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
|
||||
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
|
||||
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
|
||||
github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
|
||||
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
|
||||
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
|
||||
github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
|
||||
github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
|
||||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
|
||||
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/tlinden/yadu v0.0.0-20240118202225-ec3f0b7fc355 h1:EmgK+IGUz2m42bFKteLY5SYJLn/CyBrz6nkgS22K8Bk=
|
||||
github.com/tlinden/yadu v0.0.0-20240118202225-ec3f0b7fc355/go.mod h1:l3bRmHKL9zGAR6pnBHY2HRPxBecf7L74BoBgOOpTcUA=
|
||||
github.com/tlinden/yadu v0.1.0 h1:qtCi1jxg392qVRLFyrJ2LYu6/PiKSp1LT02EX+mNLME=
|
||||
github.com/tlinden/yadu v0.1.0/go.mod h1:l3bRmHKL9zGAR6pnBHY2HRPxBecf7L74BoBgOOpTcUA=
|
||||
github.com/tlinden/yadu v0.1.1 h1:116oEUy9b4PcMF5wLL2dCFA/sn/praYutOnao07MROw=
|
||||
github.com/tlinden/yadu v0.1.1/go.mod h1:l3bRmHKL9zGAR6pnBHY2HRPxBecf7L74BoBgOOpTcUA=
|
||||
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/tlinden/yadu v0.1.3 h1:5cRCUmj+l5yvlM2irtpFBIJwVV2DPEgYSaWvF19FtcY=
|
||||
github.com/tlinden/yadu v0.1.3/go.mod h1:l3bRmHKL9zGAR6pnBHY2HRPxBecf7L74BoBgOOpTcUA=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/image v0.32.0 h1:6lZQWq75h7L5IWNk0r+SCpUJ6tUVd3v4ZHnbRKLkUDQ=
|
||||
golang.org/x/image v0.32.0/go.mod h1:/R37rrQmKXtO6tYXAjtDLwQgFLHmhW+V6ayXlxzP2Pc=
|
||||
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190606173856-1492cefac77f/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
|
||||
golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
|
||||
golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
|
||||
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
|
||||
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
|
||||
golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
|
||||
golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
4
http.go
4
http.go
@@ -80,7 +80,9 @@ func drainBody(resp *http.Response) {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
resp.Body.Close()
|
||||
if err := resp.Body.Close(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
64
image.go
64
image.go
@@ -20,11 +20,16 @@ package main
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"image/jpeg"
|
||||
"image"
|
||||
_ "image/gif"
|
||||
_ "image/jpeg"
|
||||
_ "image/png"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
_ "golang.org/x/image/webp"
|
||||
|
||||
"github.com/corona10/goimagehash"
|
||||
)
|
||||
|
||||
@@ -33,8 +38,9 @@ const MaxDistance = 3
|
||||
type Image struct {
|
||||
Filename string
|
||||
Hash *goimagehash.ImageHash
|
||||
Data *bytes.Buffer
|
||||
Data *bytes.Reader
|
||||
URI string
|
||||
Mime string
|
||||
}
|
||||
|
||||
// used for logging to avoid printing Data
|
||||
@@ -49,21 +55,52 @@ func (img *Image) LogValue() slog.Value {
|
||||
// holds all images of an ad
|
||||
type Cache []*goimagehash.ImageHash
|
||||
|
||||
func NewImage(buf *bytes.Buffer, filename string, uri string) *Image {
|
||||
// filename comes from the scraper, it contains directory/base w/o suffix
|
||||
func NewImage(buf *bytes.Reader, filename, uri string) (*Image, error) {
|
||||
_, imgconfig, err := image.DecodeConfig(buf)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to decode image: %w", err)
|
||||
}
|
||||
|
||||
_, err = buf.Seek(0, 0)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to seek(0) on image buffer: %w", err)
|
||||
}
|
||||
|
||||
if imgconfig == "jpeg" {
|
||||
// we're using the format as file extension, but have used
|
||||
// "jpg" in the past, so to be backwards compatible, stay with
|
||||
// it.
|
||||
imgconfig = "jpg"
|
||||
}
|
||||
|
||||
if imgconfig == "" {
|
||||
return nil, fmt.Errorf("failed to process image: unknown or unsupported image format (supported: jpg,png,gif,webp)")
|
||||
}
|
||||
|
||||
filename += "." + imgconfig
|
||||
|
||||
img := &Image{
|
||||
Filename: filename,
|
||||
URI: uri,
|
||||
Data: buf,
|
||||
Mime: imgconfig,
|
||||
}
|
||||
|
||||
return img
|
||||
slog.Debug("image MIME", "mime", img.Mime)
|
||||
|
||||
return img, nil
|
||||
}
|
||||
|
||||
// Calculate diff hash of the image
|
||||
func (img *Image) CalcHash() error {
|
||||
jpgdata, err := jpeg.Decode(img.Data)
|
||||
jpgdata, format, err := image.Decode(img.Data)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to decode JPEG image: %w", err)
|
||||
return fmt.Errorf("failed to decode image: %w", err)
|
||||
}
|
||||
|
||||
if format == "" {
|
||||
return fmt.Errorf("failed to decode image: unknown or unsupported image format (supported: jpg,png,gif,webp)")
|
||||
}
|
||||
|
||||
hash1, err := goimagehash.DifferenceHash(jpgdata)
|
||||
@@ -131,12 +168,21 @@ func ReadImages(addir string, dont bool) (Cache, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
img := NewImage(data, filename, "")
|
||||
if err = img.CalcHash(); err != nil {
|
||||
reader := bytes.NewReader(data.Bytes())
|
||||
|
||||
img, err := NewImage(reader, filename, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
slog.Debug("Caching image from file system", "image", img, "hash", img.Hash.ToString())
|
||||
if err := img.CalcHash(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if img.Hash != nil {
|
||||
slog.Debug("Caching image from file system", "image", img, "hash", img.Hash.ToString())
|
||||
}
|
||||
|
||||
cache = append(cache, img.Hash)
|
||||
}
|
||||
}
|
||||
|
||||
122
kleingebaeck.1
122
kleingebaeck.1
@@ -133,7 +133,7 @@
|
||||
.\" ========================================================================
|
||||
.\"
|
||||
.IX Title "KLEINGEBAECK 1"
|
||||
.TH KLEINGEBAECK 1 "2024-01-25" "1" "User Commands"
|
||||
.TH KLEINGEBAECK 1 "2025-02-27" "1" "User Commands"
|
||||
.\" For nroff, turn off justification. Always turn off hyphenation; it makes
|
||||
.\" way too many mistakes in technical documents.
|
||||
.if n .ad l
|
||||
@@ -152,7 +152,7 @@ kleingebaeck \- kleinanzeigen.de backup tool
|
||||
\& \-l \-\-limit <num> Limit the ads to download to <num>, default: load all.
|
||||
\& \-c \-\-config <file> Use config file <file> (default: ~/.kleingebaeck).
|
||||
\& \-\-ignoreerrors Ignore HTTP errors, may lead to incomplete ad backup.
|
||||
\& \-f \-\-force Download images even if they already exist.
|
||||
\& \-f \-\-force Overwrite images and ads even if the already exist.
|
||||
\& \-m \-\-manual Show manual.
|
||||
\& \-h \-\-help Show usage.
|
||||
\& \-V \-\-version Show program version.
|
||||
@@ -195,7 +195,7 @@ Be careful if you want to change the template. The variable is a
|
||||
multiline string surrounded by three double quotes. You can left out
|
||||
certain fields and use any formatting you like. Refer to
|
||||
<https://pkg.go.dev/text/template> for details how to write a
|
||||
template.
|
||||
template. Also read the \s-1TEMPLATES\s0 section below.
|
||||
.PP
|
||||
If you're on windows and want to customize the output directory, put
|
||||
it into single quotes to avoid the backslashes interpreted as escape
|
||||
@@ -204,6 +204,118 @@ chars like this:
|
||||
.Vb 1
|
||||
\& outdir = \*(AqC:\eData\eAds\*(Aq
|
||||
.Ve
|
||||
.SH "TEMPLATES"
|
||||
.IX Header "TEMPLATES"
|
||||
Various parts of the configuration can be modified using templates:
|
||||
the output directory, the ad directory and the ad listing itself.
|
||||
.SS "\s-1OUTPUT DIR TEMPLATE\s0"
|
||||
.IX Subsection "OUTPUT DIR TEMPLATE"
|
||||
The config varialbe \f(CW\*(C`outdir\*(C'\fR or the command line parameter \f(CW\*(C`\-o\*(C'\fR take a
|
||||
template which may contain:
|
||||
.ie n .IP """{{.Year}}""" 4
|
||||
.el .IP "\f(CW{{.Year}}\fR" 4
|
||||
.IX Item "{{.Year}}"
|
||||
.PD 0
|
||||
.ie n .IP """{{.Month}}""" 4
|
||||
.el .IP "\f(CW{{.Month}}\fR" 4
|
||||
.IX Item "{{.Month}}"
|
||||
.ie n .IP """{{.Day}}""" 4
|
||||
.el .IP "\f(CW{{.Day}}\fR" 4
|
||||
.IX Item "{{.Day}}"
|
||||
.PD
|
||||
.PP
|
||||
That way you can create a new output directory for every backup
|
||||
run. For example:
|
||||
.PP
|
||||
.Vb 1
|
||||
\& outdir = "/home/backups/ads\-{{.Year}}\-{{.Month}}\-{{.Day}}"
|
||||
.Ve
|
||||
.PP
|
||||
Or using the command line flag:
|
||||
.PP
|
||||
.Vb 1
|
||||
\& \-o "/home/backups/ads\-{{.Year}}\-{{.Month}}\-{{.Day}}"
|
||||
.Ve
|
||||
.PP
|
||||
The default value is \f(CW\*(C`.\*(C'\fR \- the current directory.
|
||||
.SS "\s-1AD DIRECTORY TEMPLATE\s0"
|
||||
.IX Subsection "AD DIRECTORY TEMPLATE"
|
||||
The ad directory name can be modified using the following ad values:
|
||||
.IP "{{.Price}}" 4
|
||||
.IX Item "{{.Price}}"
|
||||
.PD 0
|
||||
.IP "{{.ID}}" 4
|
||||
.IX Item "{{.ID}}"
|
||||
.IP "{{.Category}}" 4
|
||||
.IX Item "{{.Category}}"
|
||||
.IP "{{.Condition}}" 4
|
||||
.IX Item "{{.Condition}}"
|
||||
.IP "{{.Created}}" 4
|
||||
.IX Item "{{.Created}}"
|
||||
.IP "{{.Slug}}" 4
|
||||
.IX Item "{{.Slug}}"
|
||||
.IP "{{.Text}}" 4
|
||||
.IX Item "{{.Text}}"
|
||||
.PD
|
||||
.PP
|
||||
It can only be configured in the config file. By default only
|
||||
\&\f(CW\*(C`{{.Slug}}\*(C'\fR is being used, this is the title of the ad in url format.
|
||||
.SS "\s-1AD NAME TEMPLATE\s0"
|
||||
.IX Subsection "AD NAME TEMPLATE"
|
||||
The name of the directory per ad can be tuned as well:
|
||||
.ie n .IP """{{.Year}}""" 4
|
||||
.el .IP "\f(CW{{.Year}}\fR" 4
|
||||
.IX Item "{{.Year}}"
|
||||
.PD 0
|
||||
.ie n .IP """{{.Month}}""" 4
|
||||
.el .IP "\f(CW{{.Month}}\fR" 4
|
||||
.IX Item "{{.Month}}"
|
||||
.ie n .IP """{{.Day}}""" 4
|
||||
.el .IP "\f(CW{{.Day}}\fR" 4
|
||||
.IX Item "{{.Day}}"
|
||||
.ie n .IP """{{.Slug}}""" 4
|
||||
.el .IP "\f(CW{{.Slug}}\fR" 4
|
||||
.IX Item "{{.Slug}}"
|
||||
.ie n .IP """{{.Category}}""" 4
|
||||
.el .IP "\f(CW{{.Category}}\fR" 4
|
||||
.IX Item "{{.Category}}"
|
||||
.ie n .IP """{{.ID}}""" 4
|
||||
.el .IP "\f(CW{{.ID}}\fR" 4
|
||||
.IX Item "{{.ID}}"
|
||||
.PD
|
||||
.SS "\s-1AD TEMPLATE\s0"
|
||||
.IX Subsection "AD TEMPLATE"
|
||||
The ad listing itself can be modified as well, using the same
|
||||
variables as the ad name template above.
|
||||
.PP
|
||||
This is the default template:
|
||||
.PP
|
||||
.Vb 8
|
||||
\& Title: {{.Title}}
|
||||
\& Price: {{.Price}}
|
||||
\& Id: {{.ID}}
|
||||
\& Category: {{.Category}}
|
||||
\& Condition: {{.Condition}}
|
||||
\& Type: {{.Type}}
|
||||
\& Created: {{.Created}}
|
||||
\& Expire: {{.Expire}}
|
||||
\&
|
||||
\& {{.Text}}
|
||||
.Ve
|
||||
.PP
|
||||
The config parameter to modify is \f(CW\*(C`template\*(C'\fR. See example.conf in the
|
||||
source repository. Please take care, since this is a multiline
|
||||
string. This is how it shall look if you modify it:
|
||||
.PP
|
||||
.Vb 2
|
||||
\& template="""
|
||||
\& Title: {{.Title}}
|
||||
\&
|
||||
\& {{.Text}}
|
||||
\& """
|
||||
.Ve
|
||||
.PP
|
||||
That is, the content between the two \f(CW"""\fR chars is the template.
|
||||
.SH "SETUP"
|
||||
.IX Header "SETUP"
|
||||
To setup the tool, you need to lookup your userid on
|
||||
@@ -241,7 +353,7 @@ commandline flags take precedence over env!
|
||||
.IX Header "BUGS"
|
||||
In order to report a bug, unexpected behavior, feature requests
|
||||
or to submit a patch, please open an issue on github:
|
||||
<https://github.com/TLINDEN/kleingebaeck/issues>.
|
||||
<https://codeberg.org/scip/kleingebaeck/issues>.
|
||||
.PP
|
||||
Please repeat the failing command with debugging enabled \f(CW\*(C`\-d\*(C'\fR and
|
||||
include the output in the issue.
|
||||
@@ -255,7 +367,7 @@ Also there's currently no parallelization implemented. This will
|
||||
change in the future.
|
||||
.SH "LICENSE"
|
||||
.IX Header "LICENSE"
|
||||
Copyright 2023\-2024 Thomas von Dein
|
||||
Copyright 2023\-2025 Thomas von Dein
|
||||
.PP
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the \s-1GNU\s0 General Public License as published by
|
||||
|
||||
@@ -14,7 +14,7 @@ SYNOPSYS
|
||||
-l --limit <num> Limit the ads to download to <num>, default: load all.
|
||||
-c --config <file> Use config file <file> (default: ~/.kleingebaeck).
|
||||
--ignoreerrors Ignore HTTP errors, may lead to incomplete ad backup.
|
||||
-f --force Download images even if they already exist.
|
||||
-f --force Overwrite images and ads even if the already exist.
|
||||
-m --manual Show manual.
|
||||
-h --help Show usage.
|
||||
-V --version Show program version.
|
||||
@@ -55,6 +55,7 @@ CONFIGURATION
|
||||
multiline string surrounded by three double quotes. You can left out
|
||||
certain fields and use any formatting you like. Refer to
|
||||
<https://pkg.go.dev/text/template> for details how to write a template.
|
||||
Also read the TEMPLATES section below.
|
||||
|
||||
If you're on windows and want to customize the output directory, put it
|
||||
into single quotes to avoid the backslashes interpreted as escape chars
|
||||
@@ -62,6 +63,82 @@ CONFIGURATION
|
||||
|
||||
outdir = 'C:\Data\Ads'
|
||||
|
||||
TEMPLATES
|
||||
Various parts of the configuration can be modified using templates: the
|
||||
output directory, the ad directory and the ad listing itself.
|
||||
|
||||
OUTPUT DIR TEMPLATE
|
||||
The config varialbe "outdir" or the command line parameter "-o" take a
|
||||
template which may contain:
|
||||
|
||||
"{{.Year}}"
|
||||
"{{.Month}}"
|
||||
"{{.Day}}"
|
||||
|
||||
That way you can create a new output directory for every backup run. For
|
||||
example:
|
||||
|
||||
outdir = "/home/backups/ads-{{.Year}}-{{.Month}}-{{.Day}}"
|
||||
|
||||
Or using the command line flag:
|
||||
|
||||
-o "/home/backups/ads-{{.Year}}-{{.Month}}-{{.Day}}"
|
||||
|
||||
The default value is "." - the current directory.
|
||||
|
||||
AD DIRECTORY TEMPLATE
|
||||
The ad directory name can be modified using the following ad values:
|
||||
|
||||
{{.Price}}
|
||||
{{.ID}}
|
||||
{{.Category}}
|
||||
{{.Condition}}
|
||||
{{.Created}}
|
||||
{{.Slug}}
|
||||
{{.Text}}
|
||||
|
||||
It can only be configured in the config file. By default only
|
||||
"{{.Slug}}" is being used, this is the title of the ad in url format.
|
||||
|
||||
AD NAME TEMPLATE
|
||||
The name of the directory per ad can be tuned as well:
|
||||
|
||||
"{{.Year}}"
|
||||
"{{.Month}}"
|
||||
"{{.Day}}"
|
||||
"{{.Slug}}"
|
||||
"{{.Category}}"
|
||||
"{{.ID}}"
|
||||
|
||||
AD TEMPLATE
|
||||
The ad listing itself can be modified as well, using the same variables
|
||||
as the ad name template above.
|
||||
|
||||
This is the default template:
|
||||
|
||||
Title: {{.Title}}
|
||||
Price: {{.Price}}
|
||||
Id: {{.ID}}
|
||||
Category: {{.Category}}
|
||||
Condition: {{.Condition}}
|
||||
Type: {{.Type}}
|
||||
Created: {{.Created}}
|
||||
Expire: {{.Expire}}
|
||||
|
||||
{{.Text}}
|
||||
|
||||
The config parameter to modify is "template". See example.conf in the
|
||||
source repository. Please take care, since this is a multiline string.
|
||||
This is how it shall look if you modify it:
|
||||
|
||||
template="""
|
||||
Title: {{.Title}}
|
||||
|
||||
{{.Text}}
|
||||
"""
|
||||
|
||||
That is, the content between the two """ chars is the template.
|
||||
|
||||
SETUP
|
||||
To setup the tool, you need to lookup your userid on kleinanzeigen.de.
|
||||
Go to your ad overview page while NOT being logged in:
|
||||
@@ -93,7 +170,7 @@ ENVIRONMENT VARIABLES
|
||||
BUGS
|
||||
In order to report a bug, unexpected behavior, feature requests or to
|
||||
submit a patch, please open an issue on github:
|
||||
<https://github.com/TLINDEN/kleingebaeck/issues>.
|
||||
<https://codeberg.org/scip/kleingebaeck/issues>.
|
||||
|
||||
Please repeat the failing command with debugging enabled "-d" and
|
||||
include the output in the issue.
|
||||
@@ -107,7 +184,7 @@ LIMITATIONS
|
||||
in the future.
|
||||
|
||||
LICENSE
|
||||
Copyright 2023-2024 Thomas von Dein
|
||||
Copyright 2023-2025 Thomas von Dein
|
||||
|
||||
This program is free software: you can redistribute it and/or modify it
|
||||
under the terms of the GNU General Public License as published by the
|
||||
|
||||
115
kleingebaeck.pod
115
kleingebaeck.pod
@@ -13,7 +13,7 @@ kleingebaeck - kleinanzeigen.de backup tool
|
||||
-l --limit <num> Limit the ads to download to <num>, default: load all.
|
||||
-c --config <file> Use config file <file> (default: ~/.kleingebaeck).
|
||||
--ignoreerrors Ignore HTTP errors, may lead to incomplete ad backup.
|
||||
-f --force Download images even if they already exist.
|
||||
-f --force Overwrite images and ads even if the already exist.
|
||||
-m --manual Show manual.
|
||||
-h --help Show usage.
|
||||
-V --version Show program version.
|
||||
@@ -55,7 +55,7 @@ Be careful if you want to change the template. The variable is a
|
||||
multiline string surrounded by three double quotes. You can left out
|
||||
certain fields and use any formatting you like. Refer to
|
||||
L<https://pkg.go.dev/text/template> for details how to write a
|
||||
template.
|
||||
template. Also read the TEMPLATES section below.
|
||||
|
||||
If you're on windows and want to customize the output directory, put
|
||||
it into single quotes to avoid the backslashes interpreted as escape
|
||||
@@ -63,6 +63,113 @@ chars like this:
|
||||
|
||||
outdir = 'C:\Data\Ads'
|
||||
|
||||
=head1 TEMPLATES
|
||||
|
||||
Various parts of the configuration can be modified using templates:
|
||||
the output directory, the ad directory and the ad listing itself.
|
||||
|
||||
=head2 OUTPUT DIR TEMPLATE
|
||||
|
||||
The config varialbe C<outdir> or the command line parameter C<-o> take a
|
||||
template which may contain:
|
||||
|
||||
=over
|
||||
|
||||
=item C<{{.Year}}>
|
||||
|
||||
=item C<{{.Month}}>
|
||||
|
||||
=item C<{{.Day}}>
|
||||
|
||||
=back
|
||||
|
||||
That way you can create a new output directory for every backup
|
||||
run. For example:
|
||||
|
||||
outdir = "/home/backups/ads-{{.Year}}-{{.Month}}-{{.Day}}"
|
||||
|
||||
Or using the command line flag:
|
||||
|
||||
-o "/home/backups/ads-{{.Year}}-{{.Month}}-{{.Day}}"
|
||||
|
||||
The default value is C<.> - the current directory.
|
||||
|
||||
=head2 AD DIRECTORY TEMPLATE
|
||||
|
||||
The ad directory name can be modified using the following ad values:
|
||||
|
||||
=over
|
||||
|
||||
=item {{.Price}}
|
||||
|
||||
=item {{.ID}}
|
||||
|
||||
=item {{.Category}}
|
||||
|
||||
=item {{.Condition}}
|
||||
|
||||
=item {{.Created}}
|
||||
|
||||
=item {{.Slug}}
|
||||
|
||||
=item {{.Text}}
|
||||
|
||||
=back
|
||||
|
||||
It can only be configured in the config file. By default only
|
||||
C<{{.Slug}}> is being used, this is the title of the ad in url format.
|
||||
|
||||
=head2 AD NAME TEMPLATE
|
||||
|
||||
The name of the directory per ad can be tuned as well:
|
||||
|
||||
=over
|
||||
|
||||
=item C<{{.Year}}>
|
||||
|
||||
=item C<{{.Month}}>
|
||||
|
||||
=item C<{{.Day}}>
|
||||
|
||||
=item C<{{.Slug}}>
|
||||
|
||||
=item C<{{.Category}}>
|
||||
|
||||
=item C<{{.ID}}>
|
||||
|
||||
|
||||
=back
|
||||
|
||||
=head2 AD TEMPLATE
|
||||
|
||||
The ad listing itself can be modified as well, using the same
|
||||
variables as the ad name template above.
|
||||
|
||||
This is the default template:
|
||||
|
||||
Title: {{.Title}}
|
||||
Price: {{.Price}}
|
||||
Id: {{.ID}}
|
||||
Category: {{.Category}}
|
||||
Condition: {{.Condition}}
|
||||
Type: {{.Type}}
|
||||
Created: {{.Created}}
|
||||
Expire: {{.Expire}}
|
||||
|
||||
{{.Text}}
|
||||
|
||||
The config parameter to modify is C<template>. See example.conf in the
|
||||
source repository. Please take care, since this is a multiline
|
||||
string. This is how it shall look if you modify it:
|
||||
|
||||
template="""
|
||||
Title: {{.Title}}
|
||||
|
||||
{{.Text}}
|
||||
"""
|
||||
|
||||
That is, the content between the two C<"""> chars is the template.
|
||||
|
||||
=head1 SETUP
|
||||
|
||||
To setup the tool, you need to lookup your userid on
|
||||
@@ -100,7 +207,7 @@ commandline flags take precedence over env!
|
||||
|
||||
In order to report a bug, unexpected behavior, feature requests
|
||||
or to submit a patch, please open an issue on github:
|
||||
L<https://github.com/TLINDEN/kleingebaeck/issues>.
|
||||
L<https://codeberg.org/scip/kleingebaeck/issues>.
|
||||
|
||||
Please repeat the failing command with debugging enabled C<-d> and
|
||||
include the output in the issue.
|
||||
@@ -116,7 +223,7 @@ change in the future.
|
||||
|
||||
=head1 LICENSE
|
||||
|
||||
Copyright 2023-2024 Thomas von Dein
|
||||
Copyright 2023-2025 Thomas von Dein
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
|
||||
48
main.go
48
main.go
@@ -18,13 +18,16 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"runtime"
|
||||
"runtime/debug"
|
||||
|
||||
"github.com/inconshreveable/mousetrap"
|
||||
"github.com/lmittmann/tint"
|
||||
"github.com/tlinden/yadu"
|
||||
)
|
||||
@@ -35,6 +38,25 @@ func main() {
|
||||
os.Exit(Main(os.Stdout))
|
||||
}
|
||||
|
||||
func init() {
|
||||
// if we're running on Windows AND if the user double clicked the
|
||||
// exe file from explorer, we tell them and then wait until any
|
||||
// key has been hit, which will make the cmd window disappear and
|
||||
// thus give the user time to read it.
|
||||
if runtime.GOOS == "windows" {
|
||||
if mousetrap.StartedByExplorer() {
|
||||
fmt.Println("Do no double click kleingebaeck.exe!")
|
||||
fmt.Println("Please open a command shell and run it from there.")
|
||||
fmt.Println()
|
||||
fmt.Print("Press any key to quit: ")
|
||||
_, err := bufio.NewReader(os.Stdin).ReadString('\n')
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func Main(output io.Writer) int {
|
||||
logLevel := &slog.LevelVar{}
|
||||
opts := &tint.Options{
|
||||
@@ -64,13 +86,19 @@ func Main(output io.Writer) int {
|
||||
}
|
||||
|
||||
if conf.Showversion {
|
||||
fmt.Fprintf(output, "This is kleingebaeck version %s\n", VERSION)
|
||||
_, err := fmt.Fprintf(output, "This is kleingebaeck version %s\n", VERSION)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
if conf.Showhelp {
|
||||
fmt.Fprintln(output, Usage)
|
||||
_, err := fmt.Fprintln(output, Usage)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
@@ -112,10 +140,11 @@ func Main(output io.Writer) int {
|
||||
slog.Debug("config", "conf", conf)
|
||||
|
||||
// prepare output dir
|
||||
err = Mkdir(conf.Outdir)
|
||||
outdir, err := OutDirName(conf)
|
||||
if err != nil {
|
||||
return Die(err)
|
||||
}
|
||||
conf.Outdir = outdir
|
||||
|
||||
// used for all HTTP requests
|
||||
fetch, err := NewFetcher(conf)
|
||||
@@ -123,6 +152,9 @@ func Main(output io.Writer) int {
|
||||
return Die(err)
|
||||
}
|
||||
|
||||
// setup ad dir registry, needed to check for duplicates
|
||||
DirsVisited = make(map[string]int)
|
||||
|
||||
switch {
|
||||
case len(conf.Adlinks) >= 1:
|
||||
// directly backup ad listing[s]
|
||||
@@ -149,10 +181,16 @@ func Main(output io.Writer) int {
|
||||
adstr = "ad"
|
||||
}
|
||||
|
||||
fmt.Fprintf(output, "Successfully downloaded %d %s with %d images to %s.\n",
|
||||
_, err := fmt.Fprintf(output, "Successfully downloaded %d %s with %d images to %s.\n",
|
||||
conf.StatsCountAds, adstr, conf.StatsCountImages, conf.Outdir)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
} else {
|
||||
fmt.Fprintf(output, "No ads found.")
|
||||
_, err := fmt.Fprintf(output, "No ads found.")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
return 0
|
||||
|
||||
80
main_test.go
80
main_test.go
@@ -93,6 +93,10 @@ const ADTPL string = `DOCTYPE html>
|
||||
<li class="addetailslist--detail">
|
||||
Zustand<span class="addetailslist--detail--value" >
|
||||
{{ .Condition }}</span>
|
||||
Farbe<span class="addetailslist--detail--value" >
|
||||
{{ .Color }}</span>
|
||||
Art<span class="addetailslist--detail--value" >
|
||||
{{ .Type }}</span>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
@@ -182,13 +186,13 @@ var tests = []Tests{
|
||||
{
|
||||
name: "download-all-ads",
|
||||
args: base + " -o t/out -u 1",
|
||||
expect: "Successfully downloaded 6 ads with 12 images to t/out",
|
||||
expect: "Successfully downloaded 7 ads with 16 images to t/out",
|
||||
exitcode: 0,
|
||||
},
|
||||
{
|
||||
name: "download-all-ads-using-config",
|
||||
args: "kleingebaeck -c t/fullconfig.conf",
|
||||
expect: "Successfully downloaded 6 ads with 12 images to t/out",
|
||||
expect: "Successfully downloaded 7 ads with 16 images to t/out",
|
||||
exitcode: 0,
|
||||
},
|
||||
}
|
||||
@@ -251,11 +255,14 @@ type AdConfig struct {
|
||||
Price string
|
||||
Category string
|
||||
Condition string
|
||||
Type string
|
||||
Color string
|
||||
Created string
|
||||
Text string
|
||||
Images []string // files in ./t/
|
||||
}
|
||||
|
||||
// used to generate ad listings returned by httpmock using templates
|
||||
var adsrc = []AdConfig{
|
||||
{
|
||||
Title: "First Ad",
|
||||
@@ -263,7 +270,9 @@ var adsrc = []AdConfig{
|
||||
Category: "Klimbim",
|
||||
Text: "Thing to sale",
|
||||
Slug: "first-ad",
|
||||
Condition: "works",
|
||||
Condition: "Sehr Gut",
|
||||
Color: "Grün",
|
||||
Type: "Ball",
|
||||
Created: "Yesterday",
|
||||
Images: []string{"t/1.jpg", "t/2.jpg"},
|
||||
},
|
||||
@@ -273,7 +282,9 @@ var adsrc = []AdConfig{
|
||||
Category: "Kram",
|
||||
Text: "Thing to sale",
|
||||
Slug: "second-ad",
|
||||
Condition: "works",
|
||||
Condition: "Gut",
|
||||
Color: "Lila",
|
||||
Type: "Schoki",
|
||||
Created: "Yesterday",
|
||||
Images: []string{"t/1.jpg", "t/2.jpg"},
|
||||
},
|
||||
@@ -284,7 +295,9 @@ var adsrc = []AdConfig{
|
||||
Category: "Kuddelmuddel",
|
||||
Text: "Thing to sale",
|
||||
Slug: "third-ad",
|
||||
Condition: "works",
|
||||
Condition: "In Ordnung",
|
||||
Color: "Blau",
|
||||
Type: "Auto",
|
||||
Created: "Yesterday",
|
||||
Images: []string{"t/1.jpg", "t/2.jpg"},
|
||||
},
|
||||
@@ -295,7 +308,9 @@ var adsrc = []AdConfig{
|
||||
Category: "Krempel",
|
||||
Text: "Thing to sale",
|
||||
Slug: "fourth-ad",
|
||||
Condition: "works",
|
||||
Condition: "Neu",
|
||||
Color: "Rot",
|
||||
Type: "Spielzeut",
|
||||
Created: "Yesterday",
|
||||
Images: []string{"t/1.jpg", "t/2.jpg"},
|
||||
},
|
||||
@@ -306,7 +321,9 @@ var adsrc = []AdConfig{
|
||||
Category: "Kladderadatsch",
|
||||
Text: "Thing to sale",
|
||||
Slug: "fifth-ad",
|
||||
Condition: "works",
|
||||
Condition: "Sehr Gut",
|
||||
Color: "Braun",
|
||||
Type: "Parteibuch",
|
||||
Created: "Yesterday",
|
||||
Images: []string{"t/1.jpg", "t/2.jpg"},
|
||||
},
|
||||
@@ -317,10 +334,25 @@ var adsrc = []AdConfig{
|
||||
Category: "Klunker",
|
||||
Text: "Thing to sale",
|
||||
Slug: "sixth-ad",
|
||||
Condition: "works",
|
||||
Condition: "Sehr Gut",
|
||||
Color: "Silber",
|
||||
Type: "Ring",
|
||||
Created: "Yesterday",
|
||||
Images: []string{"t/1.jpg", "t/2.jpg"},
|
||||
},
|
||||
{
|
||||
Title: "Ad with multiple img formats",
|
||||
ID: "7",
|
||||
Price: "5€",
|
||||
Category: "Klunker",
|
||||
Text: "Thing to sale",
|
||||
Slug: "seventh-ad",
|
||||
Condition: "Sehr Gut",
|
||||
Color: "Gelpb",
|
||||
Type: "Schmuck",
|
||||
Created: "Yesterday",
|
||||
Images: []string{"t/1.png", "t/1.gif", "t/1.webp", "t/1.jpg"},
|
||||
},
|
||||
}
|
||||
|
||||
// An Adsource is used to construct a httpmock responder for a
|
||||
@@ -334,14 +366,14 @@ type Adsource struct {
|
||||
}
|
||||
|
||||
// Render a HTML template for an adlisting or an ad
|
||||
func GetTemplate(adconfigs []AdConfig, adconfig AdConfig, htmltemplate string) string {
|
||||
func GetTemplate(adconfigs []AdConfig, adconfig *AdConfig, htmltemplate string) string {
|
||||
tmpl, err := tpl.New("template").Parse(htmltemplate)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
if len(adconfig.ID) == 0 {
|
||||
if adconfig.ID == "" {
|
||||
err = tmpl.Execute(&out, adconfigs)
|
||||
} else {
|
||||
err = tmpl.Execute(&out, adconfig)
|
||||
@@ -363,7 +395,7 @@ func InitValidSources() []Adsource {
|
||||
|
||||
// valid ad listing page 2
|
||||
list2 := []AdConfig{
|
||||
adsrc[3], adsrc[4], adsrc[5],
|
||||
adsrc[3], adsrc[4], adsrc[5], adsrc[6],
|
||||
}
|
||||
|
||||
// valid ad listing page 3, which is empty
|
||||
@@ -376,15 +408,15 @@ func InitValidSources() []Adsource {
|
||||
ads := []Adsource{
|
||||
{
|
||||
uri: fmt.Sprintf("%s%s?userId=1", Baseuri, Listuri),
|
||||
content: GetTemplate(list1, empty, LISTTPL),
|
||||
content: GetTemplate(list1, &empty, LISTTPL),
|
||||
},
|
||||
{
|
||||
uri: fmt.Sprintf("%s%s?userId=1&pageNum=2", Baseuri, Listuri),
|
||||
content: GetTemplate(list2, empty, LISTTPL),
|
||||
content: GetTemplate(list2, &empty, LISTTPL),
|
||||
},
|
||||
{
|
||||
uri: fmt.Sprintf("%s%s?userId=1&pageNum=3", Baseuri, Listuri),
|
||||
content: GetTemplate(list3, empty, LISTTPL),
|
||||
content: GetTemplate(list3, &empty, LISTTPL),
|
||||
},
|
||||
}
|
||||
|
||||
@@ -392,7 +424,7 @@ func InitValidSources() []Adsource {
|
||||
for _, ad := range adsrc {
|
||||
ads = append(ads, Adsource{
|
||||
uri: fmt.Sprintf("%s/s-anzeige/%s/%s", Baseuri, ad.Slug, ad.ID),
|
||||
content: GetTemplate(nil, ad, ADTPL),
|
||||
content: GetTemplate(nil, &ad, ADTPL),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -405,28 +437,28 @@ func InitInvalidSources() []Adsource {
|
||||
{
|
||||
// valid ad page but without content
|
||||
uri: fmt.Sprintf("%s/s-anzeige/empty/1", Baseuri),
|
||||
content: GetTemplate(nil, empty, EMPTYPAGE),
|
||||
content: GetTemplate(nil, &empty, EMPTYPAGE),
|
||||
},
|
||||
{
|
||||
// some random foreign webpage
|
||||
uri: INVALIDURI,
|
||||
content: GetTemplate(nil, empty, "<html>foo</html>"),
|
||||
content: GetTemplate(nil, &empty, "<html>foo</html>"),
|
||||
},
|
||||
{
|
||||
// some invalid page path
|
||||
uri: fmt.Sprintf("%s/anzeige/name/1", Baseuri),
|
||||
content: GetTemplate(nil, empty, "<html></html>"),
|
||||
content: GetTemplate(nil, &empty, "<html></html>"),
|
||||
},
|
||||
{
|
||||
// some none-ad page
|
||||
uri: fmt.Sprintf("%s/anzeige/name/1/foo/bar", Baseuri),
|
||||
content: GetTemplate(nil, empty, "<html>HTTP 404: /eine-anzeige/ does not exist!</html>"),
|
||||
content: GetTemplate(nil, &empty, "<html>HTTP 404: /eine-anzeige/ does not exist!</html>"),
|
||||
status: 404,
|
||||
},
|
||||
{
|
||||
// valid ad page but 503
|
||||
uri: fmt.Sprintf("%s/s-anzeige/503/1", Baseuri),
|
||||
content: GetTemplate(nil, empty, "<html>HTTP 503: service unavailable</html>"),
|
||||
content: GetTemplate(nil, &empty, "<html>HTTP 503: service unavailable</html>"),
|
||||
status: 503,
|
||||
},
|
||||
}
|
||||
@@ -459,13 +491,13 @@ func SetIntercept(ads []Adsource) {
|
||||
}
|
||||
|
||||
// we just use 2 images, put this here
|
||||
for _, image := range []string{"t/1.jpg", "t/2.jpg"} {
|
||||
for _, image := range []string{"t/1.jpg", "t/2.jpg", "t/1.png", "t/1.gif", "t/1.webp"} {
|
||||
httpmock.RegisterResponder("GET", image,
|
||||
httpmock.NewBytesResponder(200, GetImage(image)).HeaderAdd(headers))
|
||||
}
|
||||
}
|
||||
|
||||
func VerifyAd(advertisement AdConfig) error {
|
||||
func VerifyAd(advertisement *AdConfig) error {
|
||||
body := advertisement.Title + advertisement.Price + advertisement.ID + "Kleinanzeigen => " +
|
||||
advertisement.Category + advertisement.Condition + advertisement.Created
|
||||
|
||||
@@ -525,8 +557,8 @@ func TestMain(t *testing.T) {
|
||||
|
||||
// verify if downloaded ads match
|
||||
for _, ad := range adsrc {
|
||||
if err := VerifyAd(ad); err != nil {
|
||||
t.Errorf(err.Error())
|
||||
if err := VerifyAd(&ad); err != nil {
|
||||
t.Error(err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
15
mkrel.sh
15
mkrel.sh
@@ -22,7 +22,12 @@ freebsd/amd64
|
||||
linux/amd64
|
||||
netbsd/amd64
|
||||
openbsd/amd64
|
||||
windows/amd64"
|
||||
windows/amd64
|
||||
freebsd/arm64
|
||||
linux/arm64
|
||||
netbsd/arm64
|
||||
openbsd/arm64
|
||||
windows/arm64"
|
||||
|
||||
tool="$1"
|
||||
version="$2"
|
||||
@@ -40,15 +45,21 @@ for D in $DIST; do
|
||||
os=${D/\/*/}
|
||||
arch=${D/*\//}
|
||||
binfile="releases/${tool}-${os}-${arch}-${version}"
|
||||
pie=""
|
||||
|
||||
if test "$os" = "windows"; then
|
||||
binfile="${binfile}.exe"
|
||||
fi
|
||||
|
||||
if test "$D" = "linux/amd64"; then
|
||||
pie="-buildmode=pie"
|
||||
fi
|
||||
|
||||
tardir="${tool}-${os}-${arch}-${version}"
|
||||
tarfile="releases/${tool}-${os}-${arch}-${version}.tar.gz"
|
||||
set -x
|
||||
GOOS=${os} GOARCH=${arch} go build -tags osusergo,netgo -ldflags "-extldflags=-static" -o ${binfile}
|
||||
GOOS=${os} GOARCH=${arch} go build -tags osusergo,netgo -ldflags "-extldflags=-static -w" --trimpath $pie -o ${binfile}
|
||||
strip --strip-all ${binfile}
|
||||
mkdir -p ${tardir}
|
||||
cp ${binfile} README.md LICENSE ${tardir}/
|
||||
echo 'tool = kleingebaeck
|
||||
|
||||
65
scrape.go
65
scrape.go
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
Copyright © 2023-2024 Thomas von Dein
|
||||
Copyright © 2023-2025 Thomas von Dein
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
@@ -50,7 +50,11 @@ func ScrapeUser(fetch *Fetcher) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer body.Close()
|
||||
defer func() {
|
||||
if err := body.Close(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}()
|
||||
|
||||
err = goq.NewDecoder(body).Decode(&index)
|
||||
if err != nil {
|
||||
@@ -88,7 +92,13 @@ func ScrapeUser(fetch *Fetcher) error {
|
||||
|
||||
// scrape an ad. uri is the full uri of the ad, dir is the basedir
|
||||
func ScrapeAd(fetch *Fetcher, uri string) error {
|
||||
advertisement := &Ad{}
|
||||
now := time.Now()
|
||||
|
||||
advertisement := &Ad{
|
||||
Year: now.Format("2006"),
|
||||
Month: now.Format("01"),
|
||||
Day: now.Format("02"),
|
||||
}
|
||||
|
||||
// extract slug and id from uri
|
||||
uriparts := strings.Split(uri, "/")
|
||||
@@ -106,7 +116,11 @@ func ScrapeAd(fetch *Fetcher, uri string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer body.Close()
|
||||
defer func() {
|
||||
if err := body.Close(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}()
|
||||
|
||||
// extract ad contents with goquery/goq
|
||||
err = goq.NewDecoder(body).Decode(&advertisement)
|
||||
@@ -124,18 +138,35 @@ func ScrapeAd(fetch *Fetcher, uri string) error {
|
||||
return fmt.Errorf("could not extract ad data from page, got empty struct")
|
||||
}
|
||||
|
||||
advertisement.DecodeAttributes()
|
||||
advertisement.CalculateExpire()
|
||||
|
||||
// write listing
|
||||
addir, err := WriteAd(fetch.Config, advertisement)
|
||||
// prepare ad dir name
|
||||
addir, err := AdDirName(fetch.Config, advertisement)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
proceed := CheckAdVisited(fetch.Config, addir)
|
||||
if !proceed {
|
||||
return nil
|
||||
}
|
||||
|
||||
// write listing
|
||||
err = WriteAd(fetch.Config, advertisement, addir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// tell the user
|
||||
slog.Debug("extracted ad listing", "ad", advertisement)
|
||||
|
||||
// stats
|
||||
fetch.Config.IncrAds()
|
||||
|
||||
// register for later checks
|
||||
DirsVisited[addir] = 1
|
||||
|
||||
return ScrapeImages(fetch, advertisement, addir)
|
||||
}
|
||||
|
||||
@@ -154,7 +185,9 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error {
|
||||
|
||||
for _, imguri := range advertisement.Images {
|
||||
imguri := imguri
|
||||
file := filepath.Join(adpath, fmt.Sprintf("%d.jpg", img))
|
||||
|
||||
// we append the suffix later in NewImage() based on image format
|
||||
basefilename := filepath.Join(adpath, fmt.Sprintf("%d", img))
|
||||
|
||||
egroup.Go(func() error {
|
||||
// wait a little
|
||||
@@ -168,14 +201,19 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error {
|
||||
}
|
||||
|
||||
buf := new(bytes.Buffer)
|
||||
|
||||
_, err = buf.ReadFrom(body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read from image buffer: %w", err)
|
||||
}
|
||||
|
||||
buf2 := buf.Bytes() // needed for image writing
|
||||
reader := bytes.NewReader(buf.Bytes())
|
||||
|
||||
image, err := NewImage(reader, basefilename, imguri)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
image := NewImage(buf, file, imguri)
|
||||
err = image.CalcHash()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -189,12 +227,17 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error {
|
||||
}
|
||||
}
|
||||
|
||||
err = WriteImage(file, buf2)
|
||||
_, err = reader.Seek(0, 0)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to seek(0) on image reader: %w", err)
|
||||
}
|
||||
|
||||
err = WriteImage(image.Filename, reader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
slog.Debug("wrote image", "image", image, "size", len(buf2), "throttle", throttle)
|
||||
slog.Debug("wrote image", "image", image, "size", buf.Len(), "throttle", throttle)
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
88
store.go
88
store.go
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
Copyright © 2023-2024 Thomas von Dein
|
||||
Copyright © 2023-2025 Thomas von Dein
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
@@ -26,8 +26,36 @@ import (
|
||||
"runtime"
|
||||
"strings"
|
||||
tpl "text/template"
|
||||
"time"
|
||||
)
|
||||
|
||||
type OutdirData struct {
|
||||
Year, Day, Month string
|
||||
}
|
||||
|
||||
func OutDirName(conf *Config) (string, error) {
|
||||
tmpl, err := tpl.New("outdir").Parse(conf.Outdir)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to parse outdir template: %w", err)
|
||||
}
|
||||
|
||||
buf := bytes.Buffer{}
|
||||
|
||||
now := time.Now()
|
||||
data := OutdirData{
|
||||
Year: now.Format("2006"),
|
||||
Month: now.Format("01"),
|
||||
Day: now.Format("02"),
|
||||
}
|
||||
|
||||
err = tmpl.Execute(&buf, data)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to execute outdir template: %w", err)
|
||||
}
|
||||
|
||||
return buf.String(), nil
|
||||
}
|
||||
|
||||
func AdDirName(conf *Config, advertisement *Ad) (string, error) {
|
||||
tmpl, err := tpl.New("adname").Parse(conf.Adnametemplate)
|
||||
if err != nil {
|
||||
@@ -44,19 +72,13 @@ func AdDirName(conf *Config, advertisement *Ad) (string, error) {
|
||||
return buf.String(), nil
|
||||
}
|
||||
|
||||
func WriteAd(conf *Config, advertisement *Ad) (string, error) {
|
||||
// prepare ad dir name
|
||||
addir, err := AdDirName(conf, advertisement)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
func WriteAd(conf *Config, advertisement *Ad, addir string) error {
|
||||
// prepare output dir
|
||||
dir := filepath.Join(conf.Outdir, addir)
|
||||
|
||||
err = Mkdir(dir)
|
||||
err := Mkdir(dir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return err
|
||||
}
|
||||
|
||||
// write ad file
|
||||
@@ -64,9 +86,13 @@ func WriteAd(conf *Config, advertisement *Ad) (string, error) {
|
||||
|
||||
listingfd, err := os.Create(listingfile)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create Adlisting.txt: %w", err)
|
||||
return fmt.Errorf("failed to create Adlisting.txt: %w", err)
|
||||
}
|
||||
defer listingfd.Close()
|
||||
defer func() {
|
||||
if err := listingfd.Close(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}()
|
||||
|
||||
if runtime.GOOS == WIN {
|
||||
advertisement.Text = strings.ReplaceAll(advertisement.Text, "<br/>", "\r\n")
|
||||
@@ -76,27 +102,31 @@ func WriteAd(conf *Config, advertisement *Ad) (string, error) {
|
||||
|
||||
tmpl, err := tpl.New("adlisting").Parse(conf.Template)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to parse adlisting template: %w", err)
|
||||
return fmt.Errorf("failed to parse adlisting template: %w", err)
|
||||
}
|
||||
|
||||
err = tmpl.Execute(listingfd, advertisement)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to execute adlisting template: %w", err)
|
||||
return fmt.Errorf("failed to execute adlisting template: %w", err)
|
||||
}
|
||||
|
||||
slog.Info("wrote ad listing", "listingfile", listingfile)
|
||||
|
||||
return addir, nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func WriteImage(filename string, buf []byte) error {
|
||||
func WriteImage(filename string, reader *bytes.Reader) error {
|
||||
file, err := os.Create(filename)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open image file: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
defer func() {
|
||||
if err := file.Close(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}()
|
||||
|
||||
_, err = file.Write(buf)
|
||||
_, err = reader.WriteTo(file)
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to write to image file: %w", err)
|
||||
@@ -127,9 +157,29 @@ func ReadImage(filename string) (*bytes.Buffer, error) {
|
||||
|
||||
func fileExists(filename string) bool {
|
||||
info, err := os.Stat(filename)
|
||||
if os.IsNotExist(err) {
|
||||
|
||||
if err != nil {
|
||||
// return false on any error
|
||||
return false
|
||||
}
|
||||
|
||||
return !info.IsDir()
|
||||
}
|
||||
|
||||
// check if an addir has already been processed by current run and
|
||||
// decide what to do
|
||||
func CheckAdVisited(conf *Config, adname string) bool {
|
||||
if Exists(DirsVisited, adname) {
|
||||
if conf.ForceDownload {
|
||||
slog.Warn("an ad with the same name has already been downloaded, overwriting", "addir", adname)
|
||||
return true
|
||||
}
|
||||
|
||||
// don't overwrite
|
||||
slog.Warn("an ad with the same name has already been downloaded, skipping (use -f to overwrite)", "addir", adname)
|
||||
return false
|
||||
}
|
||||
|
||||
// overwrite
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -28,10 +29,10 @@ import (
|
||||
func TestWriteImage(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
buf := []byte{1, 2, 3, 4, 5, 6, 7, 8}
|
||||
reader := bytes.NewReader([]byte{1, 2, 3, 4, 5, 6, 7, 8})
|
||||
file := "t/out/t.jpg"
|
||||
|
||||
err := WriteImage(file, buf)
|
||||
err := WriteImage(file, reader)
|
||||
if err != nil {
|
||||
t.Errorf("Could not write mock image to %s: %s", file, err.Error())
|
||||
}
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
#!/bin/sh -x
|
||||
base="../kleinanzeigen"
|
||||
|
||||
rm -rf $base
|
||||
mkdir -p $base
|
||||
|
||||
echo "Generating /s-bestandsliste.html"
|
||||
|
||||
13
util.go
13
util.go
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
Copyright © 2023 Thomas von Dein
|
||||
Copyright © 2023-2024 Thomas von Dein
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
@@ -32,7 +32,7 @@ import (
|
||||
|
||||
func Mkdir(dir string) error {
|
||||
if _, err := os.Stat(dir); errors.Is(err, os.ErrNotExist) {
|
||||
err := os.Mkdir(dir, os.ModePerm)
|
||||
err := os.MkdirAll(dir, os.ModePerm)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create directory %s: %w", dir, err)
|
||||
}
|
||||
@@ -74,3 +74,12 @@ func IsNoTty() bool {
|
||||
func GetThrottleTime() time.Duration {
|
||||
return time.Duration(rand.Intn(MaxThrottle-MinThrottle+1)+MinThrottle) * time.Millisecond
|
||||
}
|
||||
|
||||
// look if a key in a map exists, generic variant
|
||||
func Exists[K comparable, V any](m map[K]V, v K) bool {
|
||||
if _, ok := m[v]; ok {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user