Compare commits

...

90 Commits
v0.3.2 ... main

Author SHA1 Message Date
T. von Dein
b44c1ba1dc update x/image (#124) 2025-11-05 10:13:52 +01:00
b6350344be fix image links 2025-11-05 08:56:20 +01:00
e5ff2bbc5c fix changelog link 2025-11-05 08:45:12 +01:00
T. von Dein
230bbf3d53 migrate to codeberg (#123) 2025-11-05 08:35:00 +01:00
dependabot[bot]
b3b1aac63e Bump github.com/spf13/pflag from 1.0.9 to 1.0.10 (#170) 2025-10-02 22:42:45 +02:00
dependabot[bot]
ef68a8f36b Bump golang.org/x/image from 0.30.0 to 0.31.0 (#169) 2025-10-02 22:42:25 +02:00
dependabot[bot]
734d94291b Bump docker/login-action from 3.5.0 to 3.6.0 (#167) 2025-10-02 22:42:05 +02:00
dependabot[bot]
f90158c1e1 Bump github.com/knadh/koanf/v2 from 2.2.2 to 2.3.0 (#171) 2025-10-02 22:39:43 +02:00
dependabot[bot]
09184861e3 Bump golang.org/x/sync from 0.16.0 to 0.17.0 (#168) 2025-10-02 22:38:37 +02:00
dependabot[bot]
2a520c648d Bump actions/setup-go from 5 to 6 (#166) 2025-10-02 22:38:17 +02:00
287b9995fe update go to 1.24 2025-10-02 22:35:13 +02:00
T.v.Dein
0b53e95e5e Enhancements and various Fixes (#165)
* fix expire data calculation, fixes #162
* add shipping price, if any. fixes #164
* add gh-dash config
* add logo source
* fix #163: used invalid key for type attribute, fixed switch
2025-09-03 22:18:22 +02:00
dependabot[bot]
e95bdadb12 Bump github.com/jarcoal/httpmock from 1.4.0 to 1.4.1 (#160) 2025-09-03 22:15:07 +02:00
dependabot[bot]
9959ad2468 Bump actions/checkout from 3 to 5 (#159) 2025-09-03 22:13:32 +02:00
dependabot[bot]
8da83085c8 Bump golang.org/x/image from 0.26.0 to 0.30.0 (#158) 2025-09-03 22:12:51 +02:00
dependabot[bot]
fdbec2f393 Bump github.com/spf13/pflag from 1.0.6 to 1.0.9 (#157) 2025-09-03 22:12:20 +02:00
dependabot[bot]
25bf602bba Bump github.com/go-viper/mapstructure/v2 from 2.3.0 to 2.4.0 (#156) 2025-09-03 22:11:29 +02:00
dependabot[bot]
5bfabe8fd8 Bump github.com/knadh/koanf/v2 from 2.2.1 to 2.2.2 (#153) 2025-09-03 22:11:03 +02:00
dependabot[bot]
063aa8ba86 Bump docker/login-action from 3.4.0 to 3.5.0 (#161) 2025-09-03 22:10:04 +02:00
T.v.Dein
0654150d83 bump version (#152) 2025-07-29 12:16:50 +02:00
dependabot[bot]
519d6f350a Bump github.com/go-viper/mapstructure/v2 from 2.2.1 to 2.3.0 (#147) 2025-07-29 12:15:31 +02:00
dependabot[bot]
3c071d0447 Bump golang.org/x/sync from 0.14.0 to 0.15.0 (#150) 2025-07-29 12:15:19 +02:00
dependabot[bot]
cd9d79eda7 Bump github.com/lmittmann/tint from 1.1.1 to 1.1.2 (#148) 2025-07-29 12:08:27 +02:00
dependabot[bot]
89b7f551d8 Bump github.com/knadh/koanf/providers/file from 1.1.2 to 1.2.0 (#151) 2025-07-29 12:05:49 +02:00
dependabot[bot]
85bfb0be00 Bump github.com/knadh/koanf/v2 from 2.2.0 to 2.2.1 (#149) 2025-07-29 12:05:33 +02:00
T.v.Dein
212c338ec9 update dependencies and user agent version (#146) 2025-06-10 16:13:32 +02:00
dd45dc996b ifix go version 2025-05-04 20:40:29 +02:00
1d5aa50423 fix go versions 2025-05-04 12:36:37 +02:00
T.v.Dein
15b2e7f2a7 update go, dependencies, catch Fprint and Close errors (#138)
Co-authored-by: Thomas von Dein <tom@vondein.org>
2025-05-04 12:05:59 +02:00
T.v.Dein
4030d04b06 Add year,month,day support to Adnametemplate (#123)
* add year, month and day to adnametemplate as well
2025-02-27 17:58:05 +01:00
eff0af0b34 pie only on linux 2025-02-19 18:08:15 +01:00
34b1ad9d1e remove symbols and crap from released binaries 2025-02-19 18:01:05 +01:00
T.v.Dein
6675c4d232 Fix/timeformat (#122)
* Fix #121: confused day with month thanks to time.Format
* Add outdir template variable example
2025-02-10 22:20:25 +01:00
T.v.Dein
46be48af38 Generic attributes (#120)
* fix #117: use a generic attribute parser, still support fixed attrs
2025-02-10 18:20:54 +01:00
T.v.Dein
09948a6b39 add color detail as well (#119)
Co-authored-by: Thomas von Dein <tom@vondein.org>
2025-02-06 20:13:08 +01:00
T.v.Dein
bc01391872 Fix ad condition parsing (#118)
* fix #117: use details slice and pre-set to properly extract condition
* also added the type part of the detail content (original de: "Art")

---------

Co-authored-by: Thomas von Dein <tom@vondein.org>
2025-02-06 13:48:20 +01:00
cd3d00adbe add changelog builder, update release builder 2025-02-05 17:54:47 +01:00
528ecdd43d updated deps 2025-02-01 18:04:25 +01:00
dependabot[bot]
5cb928518d Bump docker/build-push-action from 6.10.0 to 6.13.0
Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 6.10.0 to 6.13.0.
- [Release notes](https://github.com/docker/build-push-action/releases)
- [Commits](48aba3b46d...ca877d9245)

---
updated-dependencies:
- dependency-name: docker/build-push-action
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-02-01 18:03:43 +01:00
dependabot[bot]
d8c7409c7a Bump github.com/lmittmann/tint from 1.0.6 to 1.0.7
Bumps [github.com/lmittmann/tint](https://github.com/lmittmann/tint) from 1.0.6 to 1.0.7.
- [Release notes](https://github.com/lmittmann/tint/releases)
- [Commits](https://github.com/lmittmann/tint/compare/v1.0.6...v1.0.7)

---
updated-dependencies:
- dependency-name: github.com/lmittmann/tint
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-02-01 18:03:33 +01:00
dependabot[bot]
1cd6eb5134 Bump github.com/spf13/pflag from 1.0.5 to 1.0.6
Bumps [github.com/spf13/pflag](https://github.com/spf13/pflag) from 1.0.5 to 1.0.6.
- [Release notes](https://github.com/spf13/pflag/releases)
- [Commits](https://github.com/spf13/pflag/compare/v1.0.5...v1.0.6)

---
updated-dependencies:
- dependency-name: github.com/spf13/pflag
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-02-01 18:03:11 +01:00
9e2983a85c build release binaries using ci workflow 2025-01-18 10:53:41 +01:00
6eddd08e4a added warning about security update 2025-01-17 12:22:20 +01:00
34dfc25e87 bump version 2025-01-17 12:00:21 +01:00
dependabot[bot]
0bc6a0ae59 Bump github.com/lmittmann/tint from 1.0.5 to 1.0.6
Bumps [github.com/lmittmann/tint](https://github.com/lmittmann/tint) from 1.0.5 to 1.0.6.
- [Release notes](https://github.com/lmittmann/tint/releases)
- [Commits](https://github.com/lmittmann/tint/compare/v1.0.5...v1.0.6)

---
updated-dependencies:
- dependency-name: github.com/lmittmann/tint
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-01-06 18:44:09 +01:00
dependabot[bot]
14c554563a Bump golang.org/x/sync from 0.9.0 to 0.10.0
Bumps [golang.org/x/sync](https://github.com/golang/sync) from 0.9.0 to 0.10.0.
- [Commits](https://github.com/golang/sync/compare/v0.9.0...v0.10.0)

---
updated-dependencies:
- dependency-name: golang.org/x/sync
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-01-06 18:43:57 +01:00
475a9a2fd3 add test files in other formats 2025-01-06 18:42:19 +01:00
fbd9a5a621 make clear that this is not the complete filename 2025-01-06 18:42:19 +01:00
7014c97dee added image formats test 2025-01-06 18:42:19 +01:00
91edfeb19a support hashing with all formats 2025-01-06 18:42:19 +01:00
3b3435515c check seek err 2025-01-06 18:42:19 +01:00
2239a83f76 properly check image format for storing and distance hashing 2025-01-06 18:42:19 +01:00
31b27beee5 fix error handling when trying to open files 2024-12-13 11:23:24 +01:00
dependabot[bot]
a4be51f498 Bump docker/build-push-action from 6.9.0 to 6.10.0
Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 6.9.0 to 6.10.0.
- [Release notes](https://github.com/docker/build-push-action/releases)
- [Commits](4f58ea7922...48aba3b46d)

---
updated-dependencies:
- dependency-name: docker/build-push-action
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-12-13 11:11:31 +01:00
dependabot[bot]
6b5af984cc Bump golang.org/x/sync from 0.8.0 to 0.9.0
Bumps [golang.org/x/sync](https://github.com/golang/sync) from 0.8.0 to 0.9.0.
- [Commits](https://github.com/golang/sync/compare/v0.8.0...v0.9.0)

---
updated-dependencies:
- dependency-name: golang.org/x/sync
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-12-13 11:09:28 +01:00
dependabot[bot]
f5d3853388 Bump github.com/knadh/koanf/v2 from 2.1.1 to 2.1.2
Bumps [github.com/knadh/koanf/v2](https://github.com/knadh/koanf) from 2.1.1 to 2.1.2.
- [Release notes](https://github.com/knadh/koanf/releases)
- [Commits](https://github.com/knadh/koanf/compare/v2.1.1...v2.1.2)

---
updated-dependencies:
- dependency-name: github.com/knadh/koanf/v2
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-12-13 11:08:36 +01:00
07ebd7afad bump version 2024-11-01 12:21:04 +01:00
dependabot[bot]
5e82881b69 Bump github.com/knadh/koanf/providers/file from 1.1.0 to 1.1.2
Bumps [github.com/knadh/koanf/providers/file](https://github.com/knadh/koanf) from 1.1.0 to 1.1.2.
- [Release notes](https://github.com/knadh/koanf/releases)
- [Commits](https://github.com/knadh/koanf/compare/v1.1.0...providers/file/v1.1.2)

---
updated-dependencies:
- dependency-name: github.com/knadh/koanf/providers/file
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-11-01 12:20:45 +01:00
c9a75e3f91 bump version 2024-10-02 10:52:24 +02:00
dependabot[bot]
2fd2028cbe Bump github.com/knadh/koanf/providers/env from 0.1.0 to 1.0.0
Bumps [github.com/knadh/koanf/providers/env](https://github.com/knadh/koanf) from 0.1.0 to 1.0.0.
- [Release notes](https://github.com/knadh/koanf/releases)
- [Commits](https://github.com/knadh/koanf/compare/v0.1.0...v1.0.0)

---
updated-dependencies:
- dependency-name: github.com/knadh/koanf/providers/env
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-10-02 10:49:44 +02:00
dependabot[bot]
766f35d1d5 Bump docker/build-push-action from 6.7.0 to 6.9.0
Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 6.7.0 to 6.9.0.
- [Release notes](https://github.com/docker/build-push-action/releases)
- [Commits](5cd11c3a4c...4f58ea7922)

---
updated-dependencies:
- dependency-name: docker/build-push-action
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-10-02 10:49:17 +02:00
e4edde082b bump version 2024-09-25 15:43:15 +02:00
dependabot[bot]
6b95a01591 Bump github.com/lmittmann/tint from 1.0.4 to 1.0.5
Bumps [github.com/lmittmann/tint](https://github.com/lmittmann/tint) from 1.0.4 to 1.0.5.
- [Release notes](https://github.com/lmittmann/tint/releases)
- [Commits](https://github.com/lmittmann/tint/compare/v1.0.4...v1.0.5)

---
updated-dependencies:
- dependency-name: github.com/lmittmann/tint
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-09-25 15:41:03 +02:00
dependabot[bot]
2f66758c9f Bump github.com/tlinden/yadu from 0.1.2 to 0.1.3
Bumps [github.com/tlinden/yadu](https://github.com/tlinden/yadu) from 0.1.2 to 0.1.3.
- [Release notes](https://github.com/tlinden/yadu/releases)
- [Commits](https://github.com/tlinden/yadu/compare/v0.1.2...v0.1.3)

---
updated-dependencies:
- dependency-name: github.com/tlinden/yadu
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-09-25 15:40:20 +02:00
dependabot[bot]
f21b47e14f Bump golang.org/x/sync from 0.5.0 to 0.8.0
Bumps [golang.org/x/sync](https://github.com/golang/sync) from 0.5.0 to 0.8.0.
- [Commits](https://github.com/golang/sync/compare/v0.5.0...v0.8.0)

---
updated-dependencies:
- dependency-name: golang.org/x/sync
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-09-25 15:37:23 +02:00
dependabot[bot]
14f1c89f05 Bump github.com/knadh/koanf/v2 from 2.0.1 to 2.1.1
Bumps [github.com/knadh/koanf/v2](https://github.com/knadh/koanf) from 2.0.1 to 2.1.1.
- [Release notes](https://github.com/knadh/koanf/releases)
- [Commits](https://github.com/knadh/koanf/compare/v2.0.1...v2.1.1)

---
updated-dependencies:
- dependency-name: github.com/knadh/koanf/v2
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-09-25 15:36:55 +02:00
dependabot[bot]
b7530843c5 Bump github.com/knadh/koanf/providers/file from 0.1.0 to 1.1.0
Bumps [github.com/knadh/koanf/providers/file](https://github.com/knadh/koanf) from 0.1.0 to 1.1.0.
- [Release notes](https://github.com/knadh/koanf/releases)
- [Commits](https://github.com/knadh/koanf/compare/v0.1.0...v1.1.0)

---
updated-dependencies:
- dependency-name: github.com/knadh/koanf/providers/file
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-09-25 15:23:50 +02:00
dependabot[bot]
44c5e40466 Bump docker/login-action from 1.10.0 to 3.3.0
Bumps [docker/login-action](https://github.com/docker/login-action) from 1.10.0 to 3.3.0.
- [Release notes](https://github.com/docker/login-action/releases)
- [Commits](f054a8b539...9780b0c442)

---
updated-dependencies:
- dependency-name: docker/login-action
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-09-25 15:22:39 +02:00
dependabot[bot]
7b4dd50ebf Bump docker/build-push-action from 2.5.0 to 6.7.0
Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 2.5.0 to 6.7.0.
- [Release notes](https://github.com/docker/build-push-action/releases)
- [Commits](ad44023a93...5cd11c3a4c)

---
updated-dependencies:
- dependency-name: docker/build-push-action
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-09-25 15:21:43 +02:00
dependabot[bot]
5a6fe0f2fe Bump golang.org/x/net from 0.0.0-20220722155237-a158d28d115b to 0.23.0
Bumps [golang.org/x/net](https://github.com/golang/net) from 0.0.0-20220722155237-a158d28d115b to 0.23.0.
- [Commits](https://github.com/golang/net/commits/v0.23.0)

---
updated-dependencies:
- dependency-name: golang.org/x/net
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-09-25 15:18:58 +02:00
d7f5fbbe58 update other actions as well 2024-09-25 15:13:41 +02:00
ee550b1ed7 use normal Error() func 2024-09-25 15:08:34 +02:00
e35815f27f Update golang-ci linter 2024-09-25 15:05:44 +02:00
9138770d6e added dependabot config 2024-09-25 14:56:47 +02:00
834fbcd9c5 check return of ReadString() 2024-04-26 13:42:28 +02:00
4c10ae89f8 bump version 2024-04-26 13:42:28 +02:00
df6fc47ca3 fix #88: respond accordingly when user double clicks kleingebaeck.exe 2024-04-26 13:42:28 +02:00
5956a68e72 minor critics corrected 2024-04-26 13:42:28 +02:00
eee0167574 added 2024-04-26 13:42:28 +02:00
a3b2748479 fix #87: document image distance hash 2024-04-26 13:42:28 +02:00
8cc5a9e3ed missed commits 2024-02-12 13:36:29 +01:00
d2bcd7b505 fix #80: using os.MkdirAll():
Recursively create ad dir including output dir. The output dir itself
is not being created separately anymore. That way, no directory will
be created if no ads could be downloaded.
2024-02-12 13:32:25 +01:00
c59c2e2931 fix #81: add arm64 build support 2024-02-12 13:32:25 +01:00
2288806105 fix #77: use processed ad dir for duplicate checking, not slug 2024-02-10 15:15:43 +01:00
5a2c277f0e fix #71 and #73: add support for outdir template and enhance docs 2024-02-10 14:44:09 +01:00
612ed2aa79 fix #74: warn if about to write to already visited ad, overwrite if -f 2024-02-10 14:44:09 +01:00
ed78731b3c check seek error 2024-01-27 17:34:44 +01:00
a84f0e1436 get rid of duplicate bytes.Buffer, use bytes.Reader instead, #39 2024-01-27 17:34:44 +01:00
d8d5be5c7d fix #58: add missing dashes to self issue template 2024-01-27 17:34:44 +01:00
bcf920c91e correct #39 add --ignoreerrors flag 2024-01-27 17:34:44 +01:00
38 changed files with 1289 additions and 309 deletions

96
.gh-dash.yml Normal file
View File

@@ -0,0 +1,96 @@
prSections:
- title: Responsible PRs
filters: repo:tlinden/kleingebaeck is:open NOT dependabot
layout:
repoName:
hidden: true
- title: Responsible Dependabot PRs
filters: repo:tlinden/kleingebaeck is:open dependabot
layout:
repoName:
hidden: true
issuesSections:
- title: Responsible Issues
filters: is:open repo:tlinden/kleingebaeck -author:@me
layout:
repoName:
hidden: true
- title: Note-to-Self Issues
filters: is:open repo:tlinden/kleingebaeck author:@me
layout:
creator:
hidden: true
repoName:
hidden: true
defaults:
preview:
open: false
width: 100
keybindings:
universal:
- key: "shift+down"
builtin: pageDown
- key: "shift+up"
builtin: pageUp
prs:
- key: g
name: gitu
command: >
cd {{.RepoPath}} && /home/scip/bin/gitu
- key: M
name: squash-merge
command: gh pr merge --rebase --squash --admin --repo {{.RepoName}} {{.PrNumber}}
- key: i
name: show ci checks
command: gh pr checks --repo {{.RepoName}} {{.PrNumber}} | glow -p
- key: e
name: edit pr
command: ~/.config/gh-dash/edit-gh-pr {{.RepoName}} {{.PrNumber}}
- key: E
name: open repo in emacs
command: emacsclient {{.RepoPath}} &
issues:
- key: v
name: view
command: gh issue view --repo {{.RepoName}} {{.IssueNumber}} | glow -p
- key: l
name: add label
command: gh issue --repo {{.RepoName}} edit {{.IssueNumber}} --add-label $(gum choose bug enhancement question dependencies wontfix)
- key: L
name: remove label
command: gh issue --repo {{.RepoName}} edit {{.IssueNumber}} --remove-label $(gum choose bug enhancement question dependencies wontfix)
- key: E
name: open repo in emacs
command: emacsclient {{.RepoPath}} &
theme:
ui:
sectionsShowCount: true
table:
compact: false
showSeparator: true
colors:
text:
primary: "#E2E1ED"
secondary: "#6770cb"
inverted: "#242347"
faint: "#b0793b"
warning: "#E0AF68"
success: "#3DF294"
background:
selected: "#1B1B33"
border:
primary: "#383B5B"
secondary: "#39386B"
faint: "#8d3e0b"
repoPaths:
:owner/:repo: ~/dev/:repo
pager:
diff: delta

View File

@@ -5,3 +5,4 @@ title: "[bug-report]"
labels: bug
assignees: TLINDEN
---

BIN
.github/assets/kleingebaecklogo.xcf vendored Normal file

Binary file not shown.

10
.github/dependabot.yml vendored Normal file
View File

@@ -0,0 +1,10 @@
version: 2
updates:
- package-ecosystem: "gomod"
directory: "/"
schedule:
interval: "monthly"
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "monthly"

View File

@@ -1,47 +0,0 @@
name: build-and-test
on: [push, pull_request]
jobs:
build:
strategy:
matrix:
version: [1.21]
os: [ubuntu-latest, windows-latest, macos-latest]
name: Build
runs-on: ${{ matrix.os }}
steps:
- name: Set up Go
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.version }}
id: go
- name: checkout
uses: actions/checkout@v3
- name: build
run: go build
- name: test
run: make test
- name: Update coverage report
uses: ncruces/go-coverage-report@main
with:
report: true
chart: true
amend: true
if: |
matrix.os == 'ubuntu-latest' &&
github.event_name == 'push'
continue-on-error: true
golangci:
name: lint
runs-on: ubuntu-latest
steps:
- uses: actions/setup-go@v3
with:
go-version: 1.21
- uses: actions/checkout@v3
- name: golangci-lint
uses: golangci/golangci-lint-action@v3

View File

@@ -1,34 +0,0 @@
name: build-push-image
on:
push:
tags:
- 'v*'
jobs:
build-and-push-image:
runs-on: ubuntu-latest
permissions:
packages: write
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: Log in to the Container registry
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
with:
registry: https://ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push Docker image
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
with:
push: true
tags: ghcr.io/tlinden/kleingebaeck:${{ github.ref_name}}
- name: Build and push latest Docker image
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
with:
push: true
tags: ghcr.io/tlinden/kleingebaeck:latest

69
.goreleaser.yaml Normal file
View File

@@ -0,0 +1,69 @@
# vim: set ts=2 sw=2 tw=0 fo=cnqoj
version: 2
before:
hooks:
- go mod tidy
gitea_urls:
api: https://codeberg.org/api/v1
download: https://codeberg.org
builds:
- env:
- CGO_ENABLED=0
goos:
- linux
- windows
- darwin
- freebsd
archives:
- formats: [tar.gz]
# this name template makes the OS and Arch compatible with the results of `uname`.
name_template: >-
{{ .ProjectName }}_
{{- title .Os }}_
{{- if eq .Arch "amd64" }}x86_64
{{- else if eq .Arch "386" }}i386
{{- else }}{{ .Arch }}{{ end }}
{{- if .Arm }}v{{ .Arm }}{{ end }}_{{ .Tag }}
# use zip for windows archives
format_overrides:
- goos: windows
formats: [zip]
- goos: linux
formats: [tar.gz,binary]
files:
- src: "*.md"
strip_parent: true
- src: "docs/*"
strip_parent: true
- src: Makefile.dist
dst: Makefile
wrap_in_directory: true
changelog:
sort: asc
filters:
exclude:
- "^docs:"
- "^test:"
groups:
- title: Improved
regexp: '^.*?(feat|add|new)(\([[:word:]]+\))??!?:.+$'
order: 0
- title: Fixed
regexp: '^.*?(bug|fix)(\([[:word:]]+\))??!?:.+$'
order: 1
- title: Changed
order: 999
release:
header: "# Release Notes"
footer: >-
---
Full Changelog: [{{ .PreviousTag }}...{{ .Tag }}](https://codeberg.org/scip/kleingebaeck/compare/{{ .PreviousTag }}...{{ .Tag }})

36
.woodpecker/build.yaml Normal file
View File

@@ -0,0 +1,36 @@
matrix:
platform:
- linux/amd64
goversion:
- 1.24
labels:
platform: ${platform}
steps:
build:
when:
event: [push]
image: golang:${goversion}
commands:
- go get
- go build
linter:
when:
event: [push]
image: golang:${goversion}
commands:
- curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.5.0
- golangci-lint --version
- golangci-lint run ./...
depends_on: [build]
test:
when:
event: [push]
image: golang:${goversion}
commands:
- go get
- go test -v -cover
depends_on: [build,linter]

32
.woodpecker/image.yaml Normal file
View File

@@ -0,0 +1,32 @@
# https://woodpecker-ci.org/plugins/docker-buildx
# enable Package unit and go to /scip/-/packages after building to link to proj
variables:
- &repo codeberg.org/${CI_REPO_OWNER}/kleingebaeck
steps:
dryrun:
image: docker.io/woodpeckerci/plugin-docker-buildx:latest
settings:
dockerfile: Dockerfile
platforms: linux/amd64
dry_run: true
repo: *repo
tags: latest
when:
event: [pull_request]
publish:
image: docker.io/woodpeckerci/plugin-docker-buildx:latest
settings:
dockerfile: Dockerfile
platforms: linux/amd64
repo: *repo
registry: codeberg.org
tags: latest,${CI_COMMIT_SHA:0:8},${CI_COMMIT_TAG}
username: ${CI_REPO_OWNER}
password:
from_secret: REGISTRY_TOKEN
when:
event: [tag]
branch: main

15
.woodpecker/release.yaml Normal file
View File

@@ -0,0 +1,15 @@
# build release
labels:
platform: linux/amd64
steps:
goreleaser:
image: goreleaser/goreleaser
when:
event: [tag]
environment:
GITEA_TOKEN:
from_secret: DEPLOY_TOKEN
commands:
- goreleaser release --clean --verbose

View File

@@ -1,4 +1,4 @@
FROM golang:1.21-alpine as builder
FROM golang:1.24-alpine as builder
RUN apk update
RUN apk upgrade

View File

@@ -63,6 +63,7 @@ lint:
lint-full:
golangci-lint run --enable-all --exclude-use-default --disable exhaustivestruct,exhaustruct,depguard,interfacer,deadcode,golint,structcheck,scopelint,varcheck,ifshort,maligned,nosnakecase,godot,funlen,gofumpt,cyclop,noctx,gochecknoglobals,paralleltest
gocritic check -enableAll *.go
testfuzzy: clean
go test -fuzz ./... $(ARGS)
@@ -81,8 +82,8 @@ goupdate:
buildall:
./mkrel.sh $(tool) $(VERSION)
release: buildall
gh release create v$(VERSION) --generate-notes releases/*
release:
gh release create v$(VERSION) --generate-notes
show-versions: buildlocal
@echo "### kleingebaeck version:"

20
Makefile.dist Normal file
View File

@@ -0,0 +1,20 @@
# -*-make-*-
.PHONY: install all
tool = rpn
PREFIX = /usr/local
UID = root
GID = 0
all:
@echo "Type 'sudo make install' to install the tool."
@echo "To change prefix, type 'sudo make install PREFIX=/opt'"
install:
install -d -o $(UID) -g $(GID) $(PREFIX)/bin
install -d -o $(UID) -g $(GID) $(PREFIX)/man/man1
install -d -o $(UID) -g $(GID) $(PREFIX)/share/doc
install -o $(UID) -g $(GID) -m 555 $(tool) $(PREFIX)/sbin/
install -o $(UID) -g $(GID) -m 444 $(tool).1 $(PREFIX)/man/man1/
install -o $(UID) -g $(GID) -m 444 *.md $(PREFIX)/share/doc/

View File

@@ -1,13 +1,12 @@
## Kleingebäck - kleinanzeigen.de Backup
![Kleingebaeck Logo](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/kleingebaecklogo-small.png)
![Kleingebaeck Logo](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/kleingebaecklogo-small.png)
[![Go Report Card](https://goreportcard.com/badge/github.com/tlinden/kleingebaeck)](https://goreportcard.com/report/github.com/tlinden/kleingebaeck)
[![Actions](https://github.com/tlinden/kleingebaeck/actions/workflows/ci.yaml/badge.svg)](https://github.com/tlinden/kleingebaeck/actions)
[![Go Coverage](https://github.com/tlinden/kleingebaeck/wiki/coverage.svg)](https://raw.githack.com/wiki/tlinden/kleingebaeck/coverage.html)
[![Go Report Card](https://goreportcard.com/badge/codeberg.org/scip/kleingebaeck)](https://goreportcard.com/report/codeberg.org/scip/kleingebaeck)
[![status-badge](https://ci.codeberg.org/api/badges/15530/status.svg)](https://ci.codeberg.org/repos/15530)
![GitHub License](https://img.shields.io/github/license/tlinden/kleingebaeck)
[![GitHub release](https://img.shields.io/github/v/release/tlinden/kleingebaeck?color=%2300a719)](https://github.com/TLINDEN/kleingebaeck/releases/latest)
[![English](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/english.png)](https://github.com/tlinden/kleingebaeck/blob/main/README.md)
[![GitHub release](https://img.shields.io/github/v/release/tlinden/kleingebaeck?color=%2300a719)](https://codeberg.org/scip/kleingebaeck/releases)
[![English](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/english.png)](https://codeberg.org/scip/kleingebaeck/raw/branch/main/README.md)
Mit diesem Tool kann man seine Anzeigen bei https://kleinanzeigen.de sichern.
@@ -17,37 +16,46 @@ Anzeige gespeichert werden. In dem Verzeichnis wird eine Datei
`Adlisting.txt` erstellt, in der sich die Inhalte der Anzeige wie
Titel, Preis, Text etc befinden. Bilder werden natürlich auch heruntergeladen.
## ACHTUNG - SICHERHEITS-UPDATE
Fertige vorcompilierte Programme älter als Version `v0.3.12` sind von
Schwachstellen in der Behandlung von HTTP und Zertifikaten
betroffen. Falls Du eine ältere Kleingebäck-Version im Einsatz hast,
bitte update auf Version `v0.3.12` oder höher. Bitte lies auch die [Release Notes für
v0.3.12](https://codeberg.org/scip/kleingebaeck/releases/tag/v0.3.12)
für mehr Details.
## Screenshots
Das ist die Hauptseite meines kleinanzeigen.de Accounts:
![Index](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/kleinanzeigen-index.png)
![Index](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/kleinanzeigen-index.png)
Sichern ich meine Anzeigen:
![Download](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/kleinanzeigen-download.png)
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/kleinanzeigen-download.png)
Backupverzeichnis nach dem Download:
![Download](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/kleinanzeigen-backup.png)
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/kleinanzeigen-backup.png)
Verzeichnis einer Anzeige:
![Download](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/kleinanzeigen-ad.png)
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/kleinanzeigen-ad.png)
**Das gleiche unter Windows:**
Anzeigen Sichern:
![Download](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/cmd-windows.jpg)
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/cmd-windows.jpg)
Backupverzeichnis nach dem Download
![Download](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/liste-windows.jpg)
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/liste-windows.jpg)
Und eine Anzeige:
![Download](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/adlisting-windows.jpg)
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/adlisting-windows.jpg)
## Installation
@@ -57,7 +65,7 @@ für seine Plattform herunter und kann direkt loslegen.
### Installation des vorcompilierten Programms
Auf der Seite [des letzten Releases](https://github.com/TLINDEN/kleingebaeck/releases/latest) findet man das Program für sein Betriebssystem und die Plattform (z.b. Windows + Intel)
Auf der Seite [des letzten Releases](https://codeberg.org/scip/kleingebaeck/releases) findet man das Program für sein Betriebssystem und die Plattform (z.b. Windows + Intel)
Es gibt 2 Varianten:
@@ -96,7 +104,7 @@ hilfreich, aber nicht unbedingt erforderlich.
Um das Programm zu compilieren, muss man folgende Schritte ausführen:
```shell
git clone https://github.com/TLINDEN/kleingebaeck.git
git clone https://codeberg.org/scip/kleingebaeck.git
cd kleingebaeck
go mod tidy
make # (oder make)
@@ -137,7 +145,7 @@ gemountet.
Die Optionen `-u XXX -v` sind kleingebäck Optionen. Ersetze `XXX`
durch Deine tatsächliche kleinanzeigen.de Userid.
Eine Liste verfügbarer Images findet man [hier](https://github.com/tlinden/kleingebaeck/pkgs/container/kleingebaeck/versions?filters%5Bversion_type%5D=tagged)
Eine Liste verfügbarer Images findet man [hier](https://codeberg.org/scip/kleingebaeck/pkgs/container/kleingebaeck/versions?filters%5Bversion_type%5D=tagged)
## Kommandozeilen Optionen:
@@ -222,10 +230,53 @@ Sowie alle Bilder.
Das Format kann man mit der Variable `template` in der Konfiguration
ändern. Die `example.conf` enthält ein Beispiel für das Standard Template.
## Verhalten des Tools
Es gibt einige Dinge über das Verhalten von kleingebäck, über die Du
Bescheid wissen solltest:
- alle HTML Seiten und Bilder werden immer heruntergeladen
- es wird ein (konfigurierbarer) Useragent verwendet
- HTTP Cookies werden beachtet
- bei Fehlern wird dreimal mit unterschiedlichem Abstand erneut
versucht
- Bilder Downloads laufen parallelisiert mit leicht unterschiedlichen
zeitlichen Abständen ab
- Gleich aussehende Bilder werden nicht überschrieben
Der letzte Punkt muss genauer erläutert werden:
Wenn man bei Kleinanzeigen.de eine Anzeige einstellt und Bilder
postet, werden diese dort in ihrer Grösse reduziert (durch Kompression
und Verkleinerung der Bilder usw.). Diese reduzierten Bilder werden
dann von kleingebäck heruntergeladen. Falls Du Deine original Bilder
behalten hast, kannst Du diese danach in das Backupverzeichnis
kopieren. Bei einem erneuten kleingebäck-Lauf werden diese Bilder dann
nicht überschrieben.
Wir verwenden dafür einen Algorythmus namens [distance
hashing](https://github.com/corona10/goimagehash). Dieser Algorithmus
prüft die Ähnlichkeit von Bildern. Diese können in ihrer Auflösung,
Kompression, Farbtiefe und vielem mehr manipuliert worden sein und
trotzdem als das "gleiche Bild" erkannt werden (wohlgemerkt nicht "das
selbe": die Dateien sind durchaus unterschiedlich!). Bis zu einer
Distance von 5 überschreiben wir keine Bilder, weil wir dann davon
ausgehen, dass das lokal Vorhandene das Original ist.
Bitte beachte aber, dass dies KEIN Cachingmechanismus ist: die Bilder
werden trotzdem immer alle heruntergeladen. Das muss so sein, da wir
uns nicht die Dateinamen anschauen können, da kleinanzeigen.de diese
nämlich zu Zahlen umbenennt. Und die Dateinamen können sich auch
ändern, wenn der User in der Anzeige die Bilder umarrangiert hat.
Du kannst dieses Verhalten mit der Option **--force** ausschalten. Du
kannst ausserdem mit der Option **--ignoreerrors** auch alle Fehler
ignorieren, die beim Bilderdownload auftreten könnten.
## Documentation
Die Dokumentation kann man
[online](https://github.com/TLINDEN/kleingebaeck/blob/main/kleingebaeck.pod)
[online](https://codeberg.org/scip/kleingebaeck/raw/branch/main/kleingebaeck.pod)
oder lokal lesen mit: `kleingebaeck --manual`. Hat man das Tool mit
dem Tarball installiert, funktioniert auch `man kleingebaeck`.
@@ -241,7 +292,7 @@ das doch der beste Weg, die Anfrage zu übersehen und zu vergessen.
Um einen Fehler, ein unerwartetes Verhalten, eine Feature Request oder
einen Patch zu übermitteln, eröffne daher bitte einen Issue unter:
https://github.com/TLINDEN/kleingebaeck/issues. Danke!
https://codeberg.org/scip/kleingebaeck/issues. Danke!
Bitte gebe den fehlgeschlagenen Befehl an, rufe es auch mit Debugging
`-d` auf.

View File

@@ -1,13 +1,12 @@
## Kleingebäck - kleinanzeigen.de Backup
![Kleingebaeck Logo](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/kleingebaecklogo-small.png)
![Kleingebaeck Logo](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/kleingebaecklogo-small.png)
[![Go Report Card](https://goreportcard.com/badge/github.com/tlinden/kleingebaeck)](https://goreportcard.com/report/github.com/tlinden/kleingebaeck)
[![Actions](https://github.com/tlinden/kleingebaeck/actions/workflows/ci.yaml/badge.svg)](https://github.com/tlinden/kleingebaeck/actions)
[![Go Coverage](https://github.com/tlinden/kleingebaeck/wiki/coverage.svg)](https://raw.githack.com/wiki/tlinden/kleingebaeck/coverage.html)
[![Go Report Card](https://goreportcard.com/badge/codeberg.org/scip/kleingebaeck)](https://goreportcard.com/report/codeberg.org/scip/kleingebaeck)
[![status-badge](https://ci.codeberg.org/api/badges/15530/status.svg)](https://ci.codeberg.org/repos/15530)
![GitHub License](https://img.shields.io/github/license/tlinden/kleingebaeck)
[![GitHub release](https://img.shields.io/github/v/release/tlinden/kleingebaeck?color=%2300a719)](https://github.com/TLINDEN/kleingebaeck/releases/latest)
[![German](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/german.png)](https://github.com/tlinden/kleingebaeck/blob/main/README-de.md)
[![GitHub release](https://img.shields.io/github/v/release/tlinden/kleingebaeck?color=%2300a719)](https://codeberg.org/scip/kleingebaeck/releases)
[![German](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/german.png)](https://codeberg.org/scip/kleingebaeck/raw/branch/main/README-de.md)
[Die deutsche Version des READMEs findet Ihr hier](README-de.md).
@@ -18,37 +17,46 @@ directory, each ad into its own subdirectory. The backup will contain
a textfile `Adlisting.txt` which contains the ad contents as the
title, body, price etc. All images will be downloaded as well.
## CAUTION - SECURITY UPDATE
Binary releases prior to version `v0.3.11` are affected by
vulnerabilities in HTTP and certificate handling. If you are using
such a binary, please update to `v0.3.12` or higher. Please also refer
to the [Release Notes of
v0.3.12](https://codeberg.org/scip/kleingebaeck/releases/tag/v0.3.12)
for more details.
## Screenshots
This is the index of my kleinanzeigen.de Account:
![Index](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/kleinanzeigen-index.png)
![Index](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/kleinanzeigen-index.png)
Here I download my ads on the commandline:
![Download](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/kleinanzeigen-download.png)
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/kleinanzeigen-download.png)
And this is the backup directory after download:
![Download](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/kleinanzeigen-backup.png)
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/kleinanzeigen-backup.png)
Here's a directory for one ad:
![Download](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/kleinanzeigen-ad.png)
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/kleinanzeigen-ad.png)
**The same thing under windows:**
Downloading ads:
![Download](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/cmd-windows.jpg)
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/cmd-windows.jpg)
Backup directory after download:
![Download](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/liste-windows.jpg)
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/liste-windows.jpg)
And one ad listing directory:
![Download](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/adlisting-windows.jpg)
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/adlisting-windows.jpg)
## Installation
@@ -59,7 +67,7 @@ releases page and you're good to go.
### Installation using a pre-compiled binary
Go to the [latest release
page](https://github.com/TLINDEN/kleingebaeck/releases/latest) and
page](https://codeberg.org/scip/kleingebaeck/releases) and
look for your OS and platform. There are two options to install the binary:
1. Directly download the binary for your platform,
@@ -129,7 +137,7 @@ docker run. And the local directory `myads` will be mapped to
The options `-u XXX -v` are kleingebaeck options, replace `XXX` with
your actual kleinanzeigen.de user id.
A list of available images is [here](https://github.com/tlinden/kleingebaeck/pkgs/container/kleingebaeck/versions?filters%5Bversion_type%5D=tagged)
A list of available images is [here](https://codeberg.org/scip/kleingebaeck/pkgs/container/kleingebaeck/versions?filters%5Bversion_type%5D=tagged)
## Commandline options:
@@ -192,9 +200,11 @@ somewhat like this:
```default
Title: A book I sell
Price: 99 € VB
Shipping: 6,90 €
Id: 1919191919
Category: Sachbücher
Condition: Sehr Gut
Type: Buch
Created: 10.12.2023
This is the description text.
@@ -207,9 +217,51 @@ variable. The supplied sample config contains the default template.
All images will be stored in the same directory.
## Tool Behavior
There are a bunch of things you might want to know about the behavior
of the kleingebäck tool:
- all HTML pages and IMAGEs are always being downloaded
- we use a (customizable) user agent
- we respect HTTP cookies
- in the case of an error, the tool does 3 retries, the time it waits
between tries is longer for each retry
- image download is parallized using small time differences to look
more natural
- same images are not being overwritten on subsequent download
The latter needs to be elaborated a bit more:
If you publish an ad on kleinanzeigen.de and post images, those images
will be reduced in size by the site (by compressing and down sizing
them). This reduced images will be downloaded by kleingebäck. However,
you may still own the original images and may want to put them into
that backup directory so that you have all things for one ad together.
You can easily do that, because kleingebäck won't overwrite those
original images. It uses something called a distance hash using
[goimagehash](https://github.com/corona10/goimagehash). This
algorithmus checks the similarity of images. If an image has been
resized it is still very similar to the original one. We accept a
maximum of a distance of 5, everything above leads to overwrite.
This works with resizes, cropped and otherwise manipulated images as
long as the image still shows the original contents good enough.
Also note, that this is NOT a caching mechanism: the images will be
downloaded anyway during each run. We also can't look at the file
names because kleinanzeigen.de renames all images to numbers. And
those might even change if the user re-arranges the images.
You can override this behavior using the **--force** option. Another
option, **--ignoreerrors**, can be used to ignore all kinds of image
errors.
## Documentation
You can read the documentation [online](https://github.com/TLINDEN/kleingebaeck/blob/main/kleingebaeck.pod) or locally once you have installed kleingebaeck with: `kleingebaeck --manual`.
You can read the documentation [online](https://codeberg.org/scip/kleingebaeck/raw/branch/main/kleingebaeck.pod) or locally once you have installed kleingebaeck with: `kleingebaeck --manual`.
## Kleingebäck?
@@ -225,7 +277,7 @@ that's the best way for me to forget to do something.
In order to report a bug, unexpected behavior, feature requests or to
submit a patch, please open an issue on github:
https://github.com/TLINDEN/kleingebaeck/issues.
https://codeberg.org/scip/kleingebaeck/issues.
Please repeat the failing command with debugging enabled `-d` and
include the output in the issue.

17
SECURITY.md Normal file
View File

@@ -0,0 +1,17 @@
# Security Policy
## Supported Versions
Only the latest release is supported. If you find an issue (any
issue!), please check with the latest release first.
## Reporting a Vulnerability
I don't agree with the "responsible disclosure" process most projects
(and companies) work these days.
So, if you find a vulnerability of any kind, please just open an
[issue](https://codeberg.org/scip/kleingebaeck/issues). Please add
all details required to reproduce the vulnerability. You won't be chased.
That's just all about it.

91
ad.go
View File

@@ -1,5 +1,5 @@
/*
Copyright © 2023 Thomas von Dein
Copyright © 2023-2025 Thomas von Dein
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -18,6 +18,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"bufio"
"log/slog"
"strings"
"time"
@@ -31,14 +32,23 @@ type Ad struct {
Title string `goquery:"h1"`
Slug string
ID string
Condition string `goquery:".addetailslist--detail--value,text"`
Details string `goquery:".addetailslist--detail,text"`
Attributes map[string]string // processed afterwards
Condition string // post processed from details for backward compatibility
Type string // post processed from details for backward compatibility
Color string // post processed from details for backward compatibility
Material string // post processed from details for backward compatibility
Category string
CategoryTree []string `goquery:".breadcrump-link,text"`
Price string `goquery:"h2#viewad-price"`
Created string `goquery:"#viewad-extra-info,text"`
Text string `goquery:"p#viewad-description-text,html"`
Images []string `goquery:".galleryimage-element img,[src]"`
Shipping string `goquery:".boxedarticle--details--shipping,text"` // not always filled
Expire string
// runtime computed
Year, Day, Month string
}
// Used by slog to pretty print an ad
@@ -50,9 +60,10 @@ func (ad *Ad) LogValue() slog.Value {
slog.Int("imagecount", len(ad.Images)),
slog.Int("bodysize", len(ad.Text)),
slog.String("categorytree", strings.Join(ad.CategoryTree, "+")),
slog.String("condition", ad.Condition),
slog.String("created", ad.Created),
slog.String("expire", ad.Expire),
slog.String("shipping", ad.Shipping),
slog.String("details", ad.Details),
)
}
@@ -73,10 +84,80 @@ func (ad *Ad) Incomplete() bool {
}
func (ad *Ad) CalculateExpire() {
if len(ad.Created) > 0 {
if ad.Created != "" {
ts, err := time.Parse("02.01.2006", ad.Created)
if err == nil {
ad.Expire = ts.AddDate(0, ExpireMonths, ExpireDays).Format("02.01.2006")
ad.Expire = ts.AddDate(0, 0, ExpireDays).Format("02.01.2006")
}
}
}
/*
Decode attributes like color or condition. See
https://codeberg.org/scip/kleingebaeck/issues/117
for more details. In short: the HTML delivered by
kleinanzeigen.de has no css attribute for the keys
so we cannot extract key=>value mappings of the
ad details but have to parse them manually.
The ad.Details member contains this after goq run:
Art
Weitere Kinderzimmermöbel
Farbe
Holz
Zustand
In Ordnung
We parse this into ad.Attributes and fill in some
static members for backward compatibility reasons.
*/
func (ad *Ad) DecodeAttributes() {
rd := strings.NewReader(ad.Details)
scanner := bufio.NewScanner(rd)
isattr := true
attr := ""
attrmap := map[string]string{}
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" {
continue
}
if isattr {
attr = line
} else {
attrmap[attr] = line
}
isattr = !isattr
}
ad.Attributes = attrmap
if Exists(ad.Attributes, "Zustand") {
ad.Condition = ad.Attributes["Zustand"]
}
if Exists(ad.Attributes, "Farbe") {
ad.Color = ad.Attributes["Farbe"]
}
if Exists(ad.Attributes, "Art") {
ad.Type = ad.Attributes["Art"]
}
if Exists(ad.Attributes, "Material") {
ad.Material = ad.Attributes["Material"]
}
slog.Debug("parsed attributes", "attributes", ad.Attributes)
ad.Shipping = strings.Replace(ad.Shipping, "+ Versand ab ", "", 1)
}

View File

@@ -1,5 +1,5 @@
/*
Copyright © 2023-2024 Thomas von Dein
Copyright © 2023-2025 Thomas von Dein
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -34,24 +34,34 @@ import (
)
const (
VERSION string = "0.3.2"
VERSION string = "0.3.24"
Baseuri string = "https://www.kleinanzeigen.de"
Listuri string = "/s-bestandsliste.html"
Defaultdir string = "."
DefaultTemplate string = "Title: {{.Title}}\nPrice: {{.Price}}\nId: {{.ID}}\n" +
"Category: {{.Category}}\nCondition: {{.Condition}}\n" +
/*
Also possible: loop through .Attributes:
DefaultTemplate string = "Title: {{.Title}}\nPrice: {{.Price}}\nId: {{.ID}}\n" +
"Category: {{.Category}}\n{{ range $key,$val := .Attributes }}{{ $key }}: {{ $val }}\n{{ end }}" +
"Created: {{.Created}}\nExpire: {{.Expire}}\n\n{{.Text}}\n"
*/
DefaultTemplate string = "Title: {{.Title}}\nPrice: {{.Price}}\nShipping: {{.Shipping}}\nId: {{.ID}}\n" +
"Category: {{.Category}}\nCondition: {{.Condition}}\nType: {{.Type}}\nColor: {{.Color}}\n" +
"Created: {{.Created}}\nExpire: {{.Expire}}\n\n{{.Text}}\n"
DefaultTemplateWin string = "Title: {{.Title}}\r\nPrice: {{.Price}}\r\nId: {{.ID}}\r\n" +
"Category: {{.Category}}\r\nCondition: {{.Condition}}\r\n" +
DefaultTemplateWin string = "Title: {{.Title}}\r\nPrice: {{.Price}}\r\nShipping: {{.Shipping}}\r\nId: {{.ID}}\r\n" +
"Category: {{.Category}}\r\nCondition: {{.Condition}}\r\nType: {{.Type}}\r\nColor: {{.Color}}\r\n" +
"Created: {{.Created}}\r\nExpires: {{.Expire}}\r\n\r\n{{.Text}}\r\n"
DefaultUserAgent string = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " +
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36"
DefaultAdNameTemplate string = "{{.Slug}}"
DefaultOutdirTemplate string = "."
// for image download throttling
MinThrottle int = 2
MaxThrottle int = 20
@@ -59,12 +69,18 @@ const (
// we extract the slug from the uri
SlugURIPartNum int = 6
ExpireMonths int = 2
ExpireDays int = 1
// We have to calculate the ad expiry because the real value can
// only be seen by logged in users. The initial ad lifetime is 120
// days. It can be extended by the user 8 days before expire by 60
// days. But this is unknown to us, so we'll stick with our 120
// days. They may be wrong for older ads. Don't rely on it!
ExpireDays int = 120
WIN string = "windows"
)
var DirsVisited map[string]int
const Usage string = `This is kleingebaeck, the kleinanzeigen.de backup tool.
Usage: kleingebaeck [-dvVhmoclu] [<ad-listing-url>,...]
@@ -77,7 +93,7 @@ Options:
-l --limit <num> Limit the ads to download to <num>, default: load all.
-c --config <file> Use config file <file> (default: ~/.kleingebaeck).
--ignoreerrors Ignore HTTP errors, may lead to incomplete ad backup.
-f --force Download images even if they already exist.
-f --force Overwrite images and ads even if the already exist.
-m --manual Show manual.
-h --help Show usage.
-V --version Show program version.
@@ -126,7 +142,7 @@ func InitConfig(output io.Writer) (*Config, error) {
// Load default values using the confmap provider.
if err := kloader.Load(confmap.Provider(map[string]interface{}{
"template": template,
"outdir": ".",
"outdir": DefaultOutdirTemplate,
"loglevel": "notice",
"userid": 0,
"adnametemplate": DefaultAdNameTemplate,
@@ -138,7 +154,11 @@ func InitConfig(output io.Writer) (*Config, error) {
// setup custom usage
flagset := flag.NewFlagSet("config", flag.ContinueOnError)
flagset.Usage = func() {
fmt.Fprintln(output, Usage)
_, err := fmt.Fprintln(output, Usage)
if err != nil {
panic(err)
}
os.Exit(0)
}
@@ -153,6 +173,7 @@ func InitConfig(output io.Writer) (*Config, error) {
flagset.BoolP("help", "h", false, "show usage")
flagset.BoolP("manual", "m", false, "show manual")
flagset.BoolP("force", "f", false, "force")
flagset.BoolP("ignoreerrors", "", false, "ignore image download HTTP errors")
if err := flagset.Parse(os.Args[1:]); err != nil {
return nil, fmt.Errorf("failed to parse program arguments: %w", err)
@@ -178,13 +199,22 @@ func InitConfig(output io.Writer) (*Config, error) {
// Load the config file[s]
for _, cfgfile := range configfiles {
if path, err := os.Stat(cfgfile); !os.IsNotExist(err) {
if !path.IsDir() {
if err := kloader.Load(file.Provider(cfgfile), toml.Parser()); err != nil {
return nil, fmt.Errorf("error loading config file: %w", err)
}
path, err := os.Stat(cfgfile)
if err != nil {
// ignore non-existent files, but bail out on any other errors
if !os.IsNotExist(err) {
return nil, fmt.Errorf("failed to stat config file: %w", err)
}
} // else: we ignore the file if it doesn't exists
continue
}
if !path.IsDir() {
if err := kloader.Load(file.Provider(cfgfile), toml.Parser()); err != nil {
return nil, fmt.Errorf("error loading config file: %w", err)
}
}
}
// env overrides config file

View File

@@ -12,19 +12,37 @@ user = 00000000
loglevel = "verbose"
# directory where to store downloaded ads. kleingebaeck will try to
# create it. must be a quoted string.
outdir = "test"
# create it. must be a quoted string. You can also include a couple of
# template variables, e.g:
# outdir = "test-{{.Year}}-{{.Month}}-{{.Day}}"
outdir = "test"
# template for stored adlistings. To enable it, remove the comment
# chars up until the last #"""
#template="""
#Title: {{.Title}}
#Price: {{.Price}}
#Id: {{.Id}}
#Category: {{.Category}}
#Condition: {{.Condition}}
#Created: {{.Created}}
# template for stored adlistings.
template="""
Title: {{.Title}}
Price: {{.Price}}
Shipping: {{.Shipping}}
Id: {{.Id}}
Category: {{.Category}}
Condition: {{.Condition}}
Type: {{.Type}}
Created: {{.Created}}
#{{.Text}}
# """
{{.Text}}
"""
# Ads may contain more attributes than just the Condition. To print
# all attributes, loop over all of them:
template="""
Title: {{.Title}}
Price: {{.Price}}
Id: {{.Id}}
Category: {{.Category}}
{{ range $key,$val := .Attributes }}{{ $key }}: {{ $val }}
{{ end }}
Type: {{.Type}}
Created: {{.Created}}
{{.Text}}
"""

View File

@@ -52,7 +52,7 @@ func NewFetcher(conf *Config) (*Fetcher, error) {
}
func (f *Fetcher) Get(uri string) (io.ReadCloser, error) {
req, err := http.NewRequest(http.MethodGet, uri, nil)
req, err := http.NewRequest(http.MethodGet, uri, http.NoBody)
if err != nil {
return nil, fmt.Errorf("failed to create a new HTTP request obj: %w", err)
}

42
go.mod
View File

@@ -1,39 +1,41 @@
module kleingebaeck
go 1.21
go 1.24.0
toolchain go1.24.5
require (
astuart.co/goq v1.0.0
github.com/jarcoal/httpmock v1.3.1
github.com/corona10/goimagehash v1.1.0
github.com/inconshreveable/mousetrap v1.1.0
github.com/jarcoal/httpmock v1.4.1
github.com/knadh/koanf/parsers/toml v0.1.0
github.com/knadh/koanf/providers/confmap v0.1.0
github.com/knadh/koanf/providers/env v0.1.0
github.com/knadh/koanf/providers/file v0.1.0
github.com/knadh/koanf/providers/posflag v0.1.0
github.com/knadh/koanf/v2 v2.0.1
github.com/lmittmann/tint v1.0.4
github.com/knadh/koanf/providers/confmap v1.0.0
github.com/knadh/koanf/providers/env v1.1.0
github.com/knadh/koanf/providers/file v1.2.0
github.com/knadh/koanf/providers/posflag v1.0.1
github.com/knadh/koanf/v2 v2.3.0
github.com/lmittmann/tint v1.1.2
github.com/mattn/go-isatty v0.0.20
github.com/spf13/pflag v1.0.5
github.com/tlinden/yadu v0.1.1
golang.org/x/sync v0.5.0
github.com/spf13/pflag v1.0.10
github.com/tlinden/yadu v0.1.3
golang.org/x/image v0.32.0
golang.org/x/sync v0.17.0
)
require (
github.com/PuerkitoBio/goquery v1.5.1 // indirect
github.com/andybalholm/cascadia v1.1.0 // indirect
github.com/corona10/goimagehash v1.1.0 // indirect
github.com/fatih/color v1.16.0 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/knadh/koanf/maps v0.1.1 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
github.com/knadh/koanf/maps v0.1.2 // indirect
github.com/mattn/go-colorable v0.1.14 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect
github.com/pkg/errors v0.9.1 // indirect
golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect
golang.org/x/sys v0.14.0 // indirect
golang.org/x/net v0.38.0 // indirect
golang.org/x/sys v0.32.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

92
go.sum
View File

@@ -13,76 +13,76 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww=
github.com/jarcoal/httpmock v1.3.1/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg=
github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NIs=
github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI=
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/jarcoal/httpmock v1.4.1 h1:0Ju+VCFuARfFlhVXFc2HxlcQkfB+Xq12/EotHko+x2A=
github.com/jarcoal/httpmock v1.4.1/go.mod h1:ftW1xULwo+j0R0JJkJIIi7UKigZUXCLLanykgjwBXL0=
github.com/knadh/koanf/maps v0.1.2 h1:RBfmAW5CnZT+PJ1CVc1QSJKf4Xu9kxfQgYVQSu8hpbo=
github.com/knadh/koanf/maps v0.1.2/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI=
github.com/knadh/koanf/parsers/toml v0.1.0 h1:S2hLqS4TgWZYj4/7mI5m1CQQcWurxUz6ODgOub/6LCI=
github.com/knadh/koanf/parsers/toml v0.1.0/go.mod h1:yUprhq6eo3GbyVXFFMdbfZSo928ksS+uo0FFqNMnO18=
github.com/knadh/koanf/providers/confmap v0.1.0 h1:gOkxhHkemwG4LezxxN8DMOFopOPghxRVp7JbIvdvqzU=
github.com/knadh/koanf/providers/confmap v0.1.0/go.mod h1:2uLhxQzJnyHKfxG927awZC7+fyHFdQkd697K4MdLnIU=
github.com/knadh/koanf/providers/env v0.1.0 h1:LqKteXqfOWyx5Ab9VfGHmjY9BvRXi+clwyZozgVRiKg=
github.com/knadh/koanf/providers/env v0.1.0/go.mod h1:RE8K9GbACJkeEnkl8L/Qcj8p4ZyPXZIQ191HJi44ZaQ=
github.com/knadh/koanf/providers/file v0.1.0 h1:fs6U7nrV58d3CFAFh8VTde8TM262ObYf3ODrc//Lp+c=
github.com/knadh/koanf/providers/file v0.1.0/go.mod h1:rjJ/nHQl64iYCtAW2QQnF0eSmDEX/YZ/eNFj5yR6BvA=
github.com/knadh/koanf/providers/posflag v0.1.0 h1:mKJlLrKPcAP7Ootf4pBZWJ6J+4wHYujwipe7Ie3qW6U=
github.com/knadh/koanf/providers/posflag v0.1.0/go.mod h1:SYg03v/t8ISBNrMBRMlojH8OsKowbkXV7giIbBVgbz0=
github.com/knadh/koanf/v2 v2.0.1 h1:1dYGITt1I23x8cfx8ZnldtezdyaZtfAuRtIFOiRzK7g=
github.com/knadh/koanf/v2 v2.0.1/go.mod h1:ZeiIlIDXTE7w1lMT6UVcNiRAS2/rCeLn/GdLNvY1Dus=
github.com/lmittmann/tint v1.0.4 h1:LeYihpJ9hyGvE0w+K2okPTGUdVLfng1+nDNVR4vWISc=
github.com/lmittmann/tint v1.0.4/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/knadh/koanf/providers/confmap v1.0.0 h1:mHKLJTE7iXEys6deO5p6olAiZdG5zwp8Aebir+/EaRE=
github.com/knadh/koanf/providers/confmap v1.0.0/go.mod h1:txHYHiI2hAtF0/0sCmcuol4IDcuQbKTybiB1nOcUo1A=
github.com/knadh/koanf/providers/env v1.1.0 h1:U2VXPY0f+CsNDkvdsG8GcsnK4ah85WwWyJgef9oQMSc=
github.com/knadh/koanf/providers/env v1.1.0/go.mod h1:QhHHHZ87h9JxJAn2czdEl6pdkNnDh/JS1Vtsyt65hTY=
github.com/knadh/koanf/providers/file v1.2.0 h1:hrUJ6Y9YOA49aNu/RSYzOTFlqzXSCpmYIDXI7OJU6+U=
github.com/knadh/koanf/providers/file v1.2.0/go.mod h1:bp1PM5f83Q+TOUu10J/0ApLBd9uIzg+n9UgthfY+nRA=
github.com/knadh/koanf/providers/posflag v1.0.1 h1:EnMxHSrPkYCFnKgBUl5KBgrjed8gVFrcXDzaW4l/C6Y=
github.com/knadh/koanf/providers/posflag v1.0.1/go.mod h1:3Wn3+YG3f4ljzRyCUgIwH7G0sZ1pMjCOsNBovrbKmAk=
github.com/knadh/koanf/v2 v2.3.0 h1:Qg076dDRFHvqnKG97ZEsi9TAg2/nFTa9hCdcSa1lvlM=
github.com/knadh/koanf/v2 v2.3.0/go.mod h1:gRb40VRAbd4iJMYYD5IxZ6hfuopFcXBpc9bbQpZwo28=
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/lmittmann/tint v1.1.2 h1:2CQzrL6rslrsyjqLDwD11bZ5OpLBPU+g3G/r5LSfS8w=
github.com/lmittmann/tint v1.1.2/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE=
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/maxatome/go-testdeep v1.12.0 h1:Ql7Go8Tg0C1D/uMMX59LAoYK7LffeJQ6X2T04nTH68g=
github.com/maxatome/go-testdeep v1.12.0/go.mod h1:lPZc/HAcJMP92l7yI6TRz1aZN5URwUBUAfUNvrclaNM=
github.com/maxatome/go-testdeep v1.14.0 h1:rRlLv1+kI8eOI3OaBXZwb3O7xY3exRzdW5QyX48g9wI=
github.com/maxatome/go-testdeep v1.14.0/go.mod h1:lPZc/HAcJMP92l7yI6TRz1aZN5URwUBUAfUNvrclaNM=
github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/tlinden/yadu v0.0.0-20240118202225-ec3f0b7fc355 h1:EmgK+IGUz2m42bFKteLY5SYJLn/CyBrz6nkgS22K8Bk=
github.com/tlinden/yadu v0.0.0-20240118202225-ec3f0b7fc355/go.mod h1:l3bRmHKL9zGAR6pnBHY2HRPxBecf7L74BoBgOOpTcUA=
github.com/tlinden/yadu v0.1.0 h1:qtCi1jxg392qVRLFyrJ2LYu6/PiKSp1LT02EX+mNLME=
github.com/tlinden/yadu v0.1.0/go.mod h1:l3bRmHKL9zGAR6pnBHY2HRPxBecf7L74BoBgOOpTcUA=
github.com/tlinden/yadu v0.1.1 h1:116oEUy9b4PcMF5wLL2dCFA/sn/praYutOnao07MROw=
github.com/tlinden/yadu v0.1.1/go.mod h1:l3bRmHKL9zGAR6pnBHY2HRPxBecf7L74BoBgOOpTcUA=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/tlinden/yadu v0.1.3 h1:5cRCUmj+l5yvlM2irtpFBIJwVV2DPEgYSaWvF19FtcY=
github.com/tlinden/yadu v0.1.3/go.mod h1:l3bRmHKL9zGAR6pnBHY2HRPxBecf7L74BoBgOOpTcUA=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/image v0.32.0 h1:6lZQWq75h7L5IWNk0r+SCpUJ6tUVd3v4ZHnbRKLkUDQ=
golang.org/x/image v0.32.0/go.mod h1:/R37rrQmKXtO6tYXAjtDLwQgFLHmhW+V6ayXlxzP2Pc=
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190606173856-1492cefac77f/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -80,7 +80,9 @@ func drainBody(resp *http.Response) {
panic(err)
}
resp.Body.Close()
if err := resp.Body.Close(); err != nil {
panic(err)
}
}
}
}

View File

@@ -20,11 +20,16 @@ package main
import (
"bytes"
"fmt"
"image/jpeg"
"image"
_ "image/gif"
_ "image/jpeg"
_ "image/png"
"log/slog"
"os"
"path/filepath"
_ "golang.org/x/image/webp"
"github.com/corona10/goimagehash"
)
@@ -33,8 +38,9 @@ const MaxDistance = 3
type Image struct {
Filename string
Hash *goimagehash.ImageHash
Data *bytes.Buffer
Data *bytes.Reader
URI string
Mime string
}
// used for logging to avoid printing Data
@@ -49,21 +55,52 @@ func (img *Image) LogValue() slog.Value {
// holds all images of an ad
type Cache []*goimagehash.ImageHash
func NewImage(buf *bytes.Buffer, filename string, uri string) *Image {
// filename comes from the scraper, it contains directory/base w/o suffix
func NewImage(buf *bytes.Reader, filename, uri string) (*Image, error) {
_, imgconfig, err := image.DecodeConfig(buf)
if err != nil {
return nil, fmt.Errorf("failed to decode image: %w", err)
}
_, err = buf.Seek(0, 0)
if err != nil {
return nil, fmt.Errorf("failed to seek(0) on image buffer: %w", err)
}
if imgconfig == "jpeg" {
// we're using the format as file extension, but have used
// "jpg" in the past, so to be backwards compatible, stay with
// it.
imgconfig = "jpg"
}
if imgconfig == "" {
return nil, fmt.Errorf("failed to process image: unknown or unsupported image format (supported: jpg,png,gif,webp)")
}
filename += "." + imgconfig
img := &Image{
Filename: filename,
URI: uri,
Data: buf,
Mime: imgconfig,
}
return img
slog.Debug("image MIME", "mime", img.Mime)
return img, nil
}
// Calculate diff hash of the image
func (img *Image) CalcHash() error {
jpgdata, err := jpeg.Decode(img.Data)
jpgdata, format, err := image.Decode(img.Data)
if err != nil {
return fmt.Errorf("failed to decode JPEG image: %w", err)
return fmt.Errorf("failed to decode image: %w", err)
}
if format == "" {
return fmt.Errorf("failed to decode image: unknown or unsupported image format (supported: jpg,png,gif,webp)")
}
hash1, err := goimagehash.DifferenceHash(jpgdata)
@@ -131,12 +168,21 @@ func ReadImages(addir string, dont bool) (Cache, error) {
return nil, err
}
img := NewImage(data, filename, "")
if err = img.CalcHash(); err != nil {
reader := bytes.NewReader(data.Bytes())
img, err := NewImage(reader, filename, "")
if err != nil {
return nil, err
}
slog.Debug("Caching image from file system", "image", img, "hash", img.Hash.ToString())
if err := img.CalcHash(); err != nil {
return nil, err
}
if img.Hash != nil {
slog.Debug("Caching image from file system", "image", img, "hash", img.Hash.ToString())
}
cache = append(cache, img.Hash)
}
}

View File

@@ -133,7 +133,7 @@
.\" ========================================================================
.\"
.IX Title "KLEINGEBAECK 1"
.TH KLEINGEBAECK 1 "2024-01-25" "1" "User Commands"
.TH KLEINGEBAECK 1 "2025-02-27" "1" "User Commands"
.\" For nroff, turn off justification. Always turn off hyphenation; it makes
.\" way too many mistakes in technical documents.
.if n .ad l
@@ -152,7 +152,7 @@ kleingebaeck \- kleinanzeigen.de backup tool
\& \-l \-\-limit <num> Limit the ads to download to <num>, default: load all.
\& \-c \-\-config <file> Use config file <file> (default: ~/.kleingebaeck).
\& \-\-ignoreerrors Ignore HTTP errors, may lead to incomplete ad backup.
\& \-f \-\-force Download images even if they already exist.
\& \-f \-\-force Overwrite images and ads even if the already exist.
\& \-m \-\-manual Show manual.
\& \-h \-\-help Show usage.
\& \-V \-\-version Show program version.
@@ -195,7 +195,7 @@ Be careful if you want to change the template. The variable is a
multiline string surrounded by three double quotes. You can left out
certain fields and use any formatting you like. Refer to
<https://pkg.go.dev/text/template> for details how to write a
template.
template. Also read the \s-1TEMPLATES\s0 section below.
.PP
If you're on windows and want to customize the output directory, put
it into single quotes to avoid the backslashes interpreted as escape
@@ -204,6 +204,118 @@ chars like this:
.Vb 1
\& outdir = \*(AqC:\eData\eAds\*(Aq
.Ve
.SH "TEMPLATES"
.IX Header "TEMPLATES"
Various parts of the configuration can be modified using templates:
the output directory, the ad directory and the ad listing itself.
.SS "\s-1OUTPUT DIR TEMPLATE\s0"
.IX Subsection "OUTPUT DIR TEMPLATE"
The config varialbe \f(CW\*(C`outdir\*(C'\fR or the command line parameter \f(CW\*(C`\-o\*(C'\fR take a
template which may contain:
.ie n .IP """{{.Year}}""" 4
.el .IP "\f(CW{{.Year}}\fR" 4
.IX Item "{{.Year}}"
.PD 0
.ie n .IP """{{.Month}}""" 4
.el .IP "\f(CW{{.Month}}\fR" 4
.IX Item "{{.Month}}"
.ie n .IP """{{.Day}}""" 4
.el .IP "\f(CW{{.Day}}\fR" 4
.IX Item "{{.Day}}"
.PD
.PP
That way you can create a new output directory for every backup
run. For example:
.PP
.Vb 1
\& outdir = "/home/backups/ads\-{{.Year}}\-{{.Month}}\-{{.Day}}"
.Ve
.PP
Or using the command line flag:
.PP
.Vb 1
\& \-o "/home/backups/ads\-{{.Year}}\-{{.Month}}\-{{.Day}}"
.Ve
.PP
The default value is \f(CW\*(C`.\*(C'\fR \- the current directory.
.SS "\s-1AD DIRECTORY TEMPLATE\s0"
.IX Subsection "AD DIRECTORY TEMPLATE"
The ad directory name can be modified using the following ad values:
.IP "{{.Price}}" 4
.IX Item "{{.Price}}"
.PD 0
.IP "{{.ID}}" 4
.IX Item "{{.ID}}"
.IP "{{.Category}}" 4
.IX Item "{{.Category}}"
.IP "{{.Condition}}" 4
.IX Item "{{.Condition}}"
.IP "{{.Created}}" 4
.IX Item "{{.Created}}"
.IP "{{.Slug}}" 4
.IX Item "{{.Slug}}"
.IP "{{.Text}}" 4
.IX Item "{{.Text}}"
.PD
.PP
It can only be configured in the config file. By default only
\&\f(CW\*(C`{{.Slug}}\*(C'\fR is being used, this is the title of the ad in url format.
.SS "\s-1AD NAME TEMPLATE\s0"
.IX Subsection "AD NAME TEMPLATE"
The name of the directory per ad can be tuned as well:
.ie n .IP """{{.Year}}""" 4
.el .IP "\f(CW{{.Year}}\fR" 4
.IX Item "{{.Year}}"
.PD 0
.ie n .IP """{{.Month}}""" 4
.el .IP "\f(CW{{.Month}}\fR" 4
.IX Item "{{.Month}}"
.ie n .IP """{{.Day}}""" 4
.el .IP "\f(CW{{.Day}}\fR" 4
.IX Item "{{.Day}}"
.ie n .IP """{{.Slug}}""" 4
.el .IP "\f(CW{{.Slug}}\fR" 4
.IX Item "{{.Slug}}"
.ie n .IP """{{.Category}}""" 4
.el .IP "\f(CW{{.Category}}\fR" 4
.IX Item "{{.Category}}"
.ie n .IP """{{.ID}}""" 4
.el .IP "\f(CW{{.ID}}\fR" 4
.IX Item "{{.ID}}"
.PD
.SS "\s-1AD TEMPLATE\s0"
.IX Subsection "AD TEMPLATE"
The ad listing itself can be modified as well, using the same
variables as the ad name template above.
.PP
This is the default template:
.PP
.Vb 8
\& Title: {{.Title}}
\& Price: {{.Price}}
\& Id: {{.ID}}
\& Category: {{.Category}}
\& Condition: {{.Condition}}
\& Type: {{.Type}}
\& Created: {{.Created}}
\& Expire: {{.Expire}}
\&
\& {{.Text}}
.Ve
.PP
The config parameter to modify is \f(CW\*(C`template\*(C'\fR. See example.conf in the
source repository. Please take care, since this is a multiline
string. This is how it shall look if you modify it:
.PP
.Vb 2
\& template="""
\& Title: {{.Title}}
\&
\& {{.Text}}
\& """
.Ve
.PP
That is, the content between the two \f(CW"""\fR chars is the template.
.SH "SETUP"
.IX Header "SETUP"
To setup the tool, you need to lookup your userid on
@@ -241,7 +353,7 @@ commandline flags take precedence over env!
.IX Header "BUGS"
In order to report a bug, unexpected behavior, feature requests
or to submit a patch, please open an issue on github:
<https://github.com/TLINDEN/kleingebaeck/issues>.
<https://codeberg.org/scip/kleingebaeck/issues>.
.PP
Please repeat the failing command with debugging enabled \f(CW\*(C`\-d\*(C'\fR and
include the output in the issue.
@@ -255,7 +367,7 @@ Also there's currently no parallelization implemented. This will
change in the future.
.SH "LICENSE"
.IX Header "LICENSE"
Copyright 2023\-2024 Thomas von Dein
Copyright 2023\-2025 Thomas von Dein
.PP
This program is free software: you can redistribute it and/or modify
it under the terms of the \s-1GNU\s0 General Public License as published by

View File

@@ -14,7 +14,7 @@ SYNOPSYS
-l --limit <num> Limit the ads to download to <num>, default: load all.
-c --config <file> Use config file <file> (default: ~/.kleingebaeck).
--ignoreerrors Ignore HTTP errors, may lead to incomplete ad backup.
-f --force Download images even if they already exist.
-f --force Overwrite images and ads even if the already exist.
-m --manual Show manual.
-h --help Show usage.
-V --version Show program version.
@@ -55,6 +55,7 @@ CONFIGURATION
multiline string surrounded by three double quotes. You can left out
certain fields and use any formatting you like. Refer to
<https://pkg.go.dev/text/template> for details how to write a template.
Also read the TEMPLATES section below.
If you're on windows and want to customize the output directory, put it
into single quotes to avoid the backslashes interpreted as escape chars
@@ -62,6 +63,82 @@ CONFIGURATION
outdir = 'C:\Data\Ads'
TEMPLATES
Various parts of the configuration can be modified using templates: the
output directory, the ad directory and the ad listing itself.
OUTPUT DIR TEMPLATE
The config varialbe "outdir" or the command line parameter "-o" take a
template which may contain:
"{{.Year}}"
"{{.Month}}"
"{{.Day}}"
That way you can create a new output directory for every backup run. For
example:
outdir = "/home/backups/ads-{{.Year}}-{{.Month}}-{{.Day}}"
Or using the command line flag:
-o "/home/backups/ads-{{.Year}}-{{.Month}}-{{.Day}}"
The default value is "." - the current directory.
AD DIRECTORY TEMPLATE
The ad directory name can be modified using the following ad values:
{{.Price}}
{{.ID}}
{{.Category}}
{{.Condition}}
{{.Created}}
{{.Slug}}
{{.Text}}
It can only be configured in the config file. By default only
"{{.Slug}}" is being used, this is the title of the ad in url format.
AD NAME TEMPLATE
The name of the directory per ad can be tuned as well:
"{{.Year}}"
"{{.Month}}"
"{{.Day}}"
"{{.Slug}}"
"{{.Category}}"
"{{.ID}}"
AD TEMPLATE
The ad listing itself can be modified as well, using the same variables
as the ad name template above.
This is the default template:
Title: {{.Title}}
Price: {{.Price}}
Id: {{.ID}}
Category: {{.Category}}
Condition: {{.Condition}}
Type: {{.Type}}
Created: {{.Created}}
Expire: {{.Expire}}
{{.Text}}
The config parameter to modify is "template". See example.conf in the
source repository. Please take care, since this is a multiline string.
This is how it shall look if you modify it:
template="""
Title: {{.Title}}
{{.Text}}
"""
That is, the content between the two """ chars is the template.
SETUP
To setup the tool, you need to lookup your userid on kleinanzeigen.de.
Go to your ad overview page while NOT being logged in:
@@ -93,7 +170,7 @@ ENVIRONMENT VARIABLES
BUGS
In order to report a bug, unexpected behavior, feature requests or to
submit a patch, please open an issue on github:
<https://github.com/TLINDEN/kleingebaeck/issues>.
<https://codeberg.org/scip/kleingebaeck/issues>.
Please repeat the failing command with debugging enabled "-d" and
include the output in the issue.
@@ -107,7 +184,7 @@ LIMITATIONS
in the future.
LICENSE
Copyright 2023-2024 Thomas von Dein
Copyright 2023-2025 Thomas von Dein
This program is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by the

View File

@@ -13,7 +13,7 @@ kleingebaeck - kleinanzeigen.de backup tool
-l --limit <num> Limit the ads to download to <num>, default: load all.
-c --config <file> Use config file <file> (default: ~/.kleingebaeck).
--ignoreerrors Ignore HTTP errors, may lead to incomplete ad backup.
-f --force Download images even if they already exist.
-f --force Overwrite images and ads even if the already exist.
-m --manual Show manual.
-h --help Show usage.
-V --version Show program version.
@@ -55,7 +55,7 @@ Be careful if you want to change the template. The variable is a
multiline string surrounded by three double quotes. You can left out
certain fields and use any formatting you like. Refer to
L<https://pkg.go.dev/text/template> for details how to write a
template.
template. Also read the TEMPLATES section below.
If you're on windows and want to customize the output directory, put
it into single quotes to avoid the backslashes interpreted as escape
@@ -63,6 +63,113 @@ chars like this:
outdir = 'C:\Data\Ads'
=head1 TEMPLATES
Various parts of the configuration can be modified using templates:
the output directory, the ad directory and the ad listing itself.
=head2 OUTPUT DIR TEMPLATE
The config varialbe C<outdir> or the command line parameter C<-o> take a
template which may contain:
=over
=item C<{{.Year}}>
=item C<{{.Month}}>
=item C<{{.Day}}>
=back
That way you can create a new output directory for every backup
run. For example:
outdir = "/home/backups/ads-{{.Year}}-{{.Month}}-{{.Day}}"
Or using the command line flag:
-o "/home/backups/ads-{{.Year}}-{{.Month}}-{{.Day}}"
The default value is C<.> - the current directory.
=head2 AD DIRECTORY TEMPLATE
The ad directory name can be modified using the following ad values:
=over
=item {{.Price}}
=item {{.ID}}
=item {{.Category}}
=item {{.Condition}}
=item {{.Created}}
=item {{.Slug}}
=item {{.Text}}
=back
It can only be configured in the config file. By default only
C<{{.Slug}}> is being used, this is the title of the ad in url format.
=head2 AD NAME TEMPLATE
The name of the directory per ad can be tuned as well:
=over
=item C<{{.Year}}>
=item C<{{.Month}}>
=item C<{{.Day}}>
=item C<{{.Slug}}>
=item C<{{.Category}}>
=item C<{{.ID}}>
=back
=head2 AD TEMPLATE
The ad listing itself can be modified as well, using the same
variables as the ad name template above.
This is the default template:
Title: {{.Title}}
Price: {{.Price}}
Id: {{.ID}}
Category: {{.Category}}
Condition: {{.Condition}}
Type: {{.Type}}
Created: {{.Created}}
Expire: {{.Expire}}
{{.Text}}
The config parameter to modify is C<template>. See example.conf in the
source repository. Please take care, since this is a multiline
string. This is how it shall look if you modify it:
template="""
Title: {{.Title}}
{{.Text}}
"""
That is, the content between the two C<"""> chars is the template.
=head1 SETUP
To setup the tool, you need to lookup your userid on
@@ -100,7 +207,7 @@ commandline flags take precedence over env!
In order to report a bug, unexpected behavior, feature requests
or to submit a patch, please open an issue on github:
L<https://github.com/TLINDEN/kleingebaeck/issues>.
L<https://codeberg.org/scip/kleingebaeck/issues>.
Please repeat the failing command with debugging enabled C<-d> and
include the output in the issue.
@@ -116,7 +223,7 @@ change in the future.
=head1 LICENSE
Copyright 2023-2024 Thomas von Dein
Copyright 2023-2025 Thomas von Dein
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by

48
main.go
View File

@@ -18,13 +18,16 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"bufio"
"errors"
"fmt"
"io"
"log/slog"
"os"
"runtime"
"runtime/debug"
"github.com/inconshreveable/mousetrap"
"github.com/lmittmann/tint"
"github.com/tlinden/yadu"
)
@@ -35,6 +38,25 @@ func main() {
os.Exit(Main(os.Stdout))
}
func init() {
// if we're running on Windows AND if the user double clicked the
// exe file from explorer, we tell them and then wait until any
// key has been hit, which will make the cmd window disappear and
// thus give the user time to read it.
if runtime.GOOS == "windows" {
if mousetrap.StartedByExplorer() {
fmt.Println("Do no double click kleingebaeck.exe!")
fmt.Println("Please open a command shell and run it from there.")
fmt.Println()
fmt.Print("Press any key to quit: ")
_, err := bufio.NewReader(os.Stdin).ReadString('\n')
if err != nil {
panic(err)
}
}
}
}
func Main(output io.Writer) int {
logLevel := &slog.LevelVar{}
opts := &tint.Options{
@@ -64,13 +86,19 @@ func Main(output io.Writer) int {
}
if conf.Showversion {
fmt.Fprintf(output, "This is kleingebaeck version %s\n", VERSION)
_, err := fmt.Fprintf(output, "This is kleingebaeck version %s\n", VERSION)
if err != nil {
panic(err)
}
return 0
}
if conf.Showhelp {
fmt.Fprintln(output, Usage)
_, err := fmt.Fprintln(output, Usage)
if err != nil {
panic(err)
}
return 0
}
@@ -112,10 +140,11 @@ func Main(output io.Writer) int {
slog.Debug("config", "conf", conf)
// prepare output dir
err = Mkdir(conf.Outdir)
outdir, err := OutDirName(conf)
if err != nil {
return Die(err)
}
conf.Outdir = outdir
// used for all HTTP requests
fetch, err := NewFetcher(conf)
@@ -123,6 +152,9 @@ func Main(output io.Writer) int {
return Die(err)
}
// setup ad dir registry, needed to check for duplicates
DirsVisited = make(map[string]int)
switch {
case len(conf.Adlinks) >= 1:
// directly backup ad listing[s]
@@ -149,10 +181,16 @@ func Main(output io.Writer) int {
adstr = "ad"
}
fmt.Fprintf(output, "Successfully downloaded %d %s with %d images to %s.\n",
_, err := fmt.Fprintf(output, "Successfully downloaded %d %s with %d images to %s.\n",
conf.StatsCountAds, adstr, conf.StatsCountImages, conf.Outdir)
if err != nil {
panic(err)
}
} else {
fmt.Fprintf(output, "No ads found.")
_, err := fmt.Fprintf(output, "No ads found.")
if err != nil {
panic(err)
}
}
return 0

View File

@@ -93,6 +93,10 @@ const ADTPL string = `DOCTYPE html>
<li class="addetailslist--detail">
Zustand<span class="addetailslist--detail--value" >
{{ .Condition }}</span>
Farbe<span class="addetailslist--detail--value" >
{{ .Color }}</span>
Art<span class="addetailslist--detail--value" >
{{ .Type }}</span>
</li>
</ul>
</div>
@@ -182,13 +186,13 @@ var tests = []Tests{
{
name: "download-all-ads",
args: base + " -o t/out -u 1",
expect: "Successfully downloaded 6 ads with 12 images to t/out",
expect: "Successfully downloaded 7 ads with 16 images to t/out",
exitcode: 0,
},
{
name: "download-all-ads-using-config",
args: "kleingebaeck -c t/fullconfig.conf",
expect: "Successfully downloaded 6 ads with 12 images to t/out",
expect: "Successfully downloaded 7 ads with 16 images to t/out",
exitcode: 0,
},
}
@@ -251,11 +255,14 @@ type AdConfig struct {
Price string
Category string
Condition string
Type string
Color string
Created string
Text string
Images []string // files in ./t/
}
// used to generate ad listings returned by httpmock using templates
var adsrc = []AdConfig{
{
Title: "First Ad",
@@ -263,7 +270,9 @@ var adsrc = []AdConfig{
Category: "Klimbim",
Text: "Thing to sale",
Slug: "first-ad",
Condition: "works",
Condition: "Sehr Gut",
Color: "Grün",
Type: "Ball",
Created: "Yesterday",
Images: []string{"t/1.jpg", "t/2.jpg"},
},
@@ -273,7 +282,9 @@ var adsrc = []AdConfig{
Category: "Kram",
Text: "Thing to sale",
Slug: "second-ad",
Condition: "works",
Condition: "Gut",
Color: "Lila",
Type: "Schoki",
Created: "Yesterday",
Images: []string{"t/1.jpg", "t/2.jpg"},
},
@@ -284,7 +295,9 @@ var adsrc = []AdConfig{
Category: "Kuddelmuddel",
Text: "Thing to sale",
Slug: "third-ad",
Condition: "works",
Condition: "In Ordnung",
Color: "Blau",
Type: "Auto",
Created: "Yesterday",
Images: []string{"t/1.jpg", "t/2.jpg"},
},
@@ -295,7 +308,9 @@ var adsrc = []AdConfig{
Category: "Krempel",
Text: "Thing to sale",
Slug: "fourth-ad",
Condition: "works",
Condition: "Neu",
Color: "Rot",
Type: "Spielzeut",
Created: "Yesterday",
Images: []string{"t/1.jpg", "t/2.jpg"},
},
@@ -306,7 +321,9 @@ var adsrc = []AdConfig{
Category: "Kladderadatsch",
Text: "Thing to sale",
Slug: "fifth-ad",
Condition: "works",
Condition: "Sehr Gut",
Color: "Braun",
Type: "Parteibuch",
Created: "Yesterday",
Images: []string{"t/1.jpg", "t/2.jpg"},
},
@@ -317,10 +334,25 @@ var adsrc = []AdConfig{
Category: "Klunker",
Text: "Thing to sale",
Slug: "sixth-ad",
Condition: "works",
Condition: "Sehr Gut",
Color: "Silber",
Type: "Ring",
Created: "Yesterday",
Images: []string{"t/1.jpg", "t/2.jpg"},
},
{
Title: "Ad with multiple img formats",
ID: "7",
Price: "5€",
Category: "Klunker",
Text: "Thing to sale",
Slug: "seventh-ad",
Condition: "Sehr Gut",
Color: "Gelpb",
Type: "Schmuck",
Created: "Yesterday",
Images: []string{"t/1.png", "t/1.gif", "t/1.webp", "t/1.jpg"},
},
}
// An Adsource is used to construct a httpmock responder for a
@@ -334,14 +366,14 @@ type Adsource struct {
}
// Render a HTML template for an adlisting or an ad
func GetTemplate(adconfigs []AdConfig, adconfig AdConfig, htmltemplate string) string {
func GetTemplate(adconfigs []AdConfig, adconfig *AdConfig, htmltemplate string) string {
tmpl, err := tpl.New("template").Parse(htmltemplate)
if err != nil {
panic(err)
}
var out bytes.Buffer
if len(adconfig.ID) == 0 {
if adconfig.ID == "" {
err = tmpl.Execute(&out, adconfigs)
} else {
err = tmpl.Execute(&out, adconfig)
@@ -363,7 +395,7 @@ func InitValidSources() []Adsource {
// valid ad listing page 2
list2 := []AdConfig{
adsrc[3], adsrc[4], adsrc[5],
adsrc[3], adsrc[4], adsrc[5], adsrc[6],
}
// valid ad listing page 3, which is empty
@@ -376,15 +408,15 @@ func InitValidSources() []Adsource {
ads := []Adsource{
{
uri: fmt.Sprintf("%s%s?userId=1", Baseuri, Listuri),
content: GetTemplate(list1, empty, LISTTPL),
content: GetTemplate(list1, &empty, LISTTPL),
},
{
uri: fmt.Sprintf("%s%s?userId=1&pageNum=2", Baseuri, Listuri),
content: GetTemplate(list2, empty, LISTTPL),
content: GetTemplate(list2, &empty, LISTTPL),
},
{
uri: fmt.Sprintf("%s%s?userId=1&pageNum=3", Baseuri, Listuri),
content: GetTemplate(list3, empty, LISTTPL),
content: GetTemplate(list3, &empty, LISTTPL),
},
}
@@ -392,7 +424,7 @@ func InitValidSources() []Adsource {
for _, ad := range adsrc {
ads = append(ads, Adsource{
uri: fmt.Sprintf("%s/s-anzeige/%s/%s", Baseuri, ad.Slug, ad.ID),
content: GetTemplate(nil, ad, ADTPL),
content: GetTemplate(nil, &ad, ADTPL),
})
}
@@ -405,28 +437,28 @@ func InitInvalidSources() []Adsource {
{
// valid ad page but without content
uri: fmt.Sprintf("%s/s-anzeige/empty/1", Baseuri),
content: GetTemplate(nil, empty, EMPTYPAGE),
content: GetTemplate(nil, &empty, EMPTYPAGE),
},
{
// some random foreign webpage
uri: INVALIDURI,
content: GetTemplate(nil, empty, "<html>foo</html>"),
content: GetTemplate(nil, &empty, "<html>foo</html>"),
},
{
// some invalid page path
uri: fmt.Sprintf("%s/anzeige/name/1", Baseuri),
content: GetTemplate(nil, empty, "<html></html>"),
content: GetTemplate(nil, &empty, "<html></html>"),
},
{
// some none-ad page
uri: fmt.Sprintf("%s/anzeige/name/1/foo/bar", Baseuri),
content: GetTemplate(nil, empty, "<html>HTTP 404: /eine-anzeige/ does not exist!</html>"),
content: GetTemplate(nil, &empty, "<html>HTTP 404: /eine-anzeige/ does not exist!</html>"),
status: 404,
},
{
// valid ad page but 503
uri: fmt.Sprintf("%s/s-anzeige/503/1", Baseuri),
content: GetTemplate(nil, empty, "<html>HTTP 503: service unavailable</html>"),
content: GetTemplate(nil, &empty, "<html>HTTP 503: service unavailable</html>"),
status: 503,
},
}
@@ -459,13 +491,13 @@ func SetIntercept(ads []Adsource) {
}
// we just use 2 images, put this here
for _, image := range []string{"t/1.jpg", "t/2.jpg"} {
for _, image := range []string{"t/1.jpg", "t/2.jpg", "t/1.png", "t/1.gif", "t/1.webp"} {
httpmock.RegisterResponder("GET", image,
httpmock.NewBytesResponder(200, GetImage(image)).HeaderAdd(headers))
}
}
func VerifyAd(advertisement AdConfig) error {
func VerifyAd(advertisement *AdConfig) error {
body := advertisement.Title + advertisement.Price + advertisement.ID + "Kleinanzeigen => " +
advertisement.Category + advertisement.Condition + advertisement.Created
@@ -525,8 +557,8 @@ func TestMain(t *testing.T) {
// verify if downloaded ads match
for _, ad := range adsrc {
if err := VerifyAd(ad); err != nil {
t.Errorf(err.Error())
if err := VerifyAd(&ad); err != nil {
t.Error(err.Error())
}
}
}

View File

@@ -22,7 +22,12 @@ freebsd/amd64
linux/amd64
netbsd/amd64
openbsd/amd64
windows/amd64"
windows/amd64
freebsd/arm64
linux/arm64
netbsd/arm64
openbsd/arm64
windows/arm64"
tool="$1"
version="$2"
@@ -40,15 +45,21 @@ for D in $DIST; do
os=${D/\/*/}
arch=${D/*\//}
binfile="releases/${tool}-${os}-${arch}-${version}"
pie=""
if test "$os" = "windows"; then
binfile="${binfile}.exe"
fi
if test "$D" = "linux/amd64"; then
pie="-buildmode=pie"
fi
tardir="${tool}-${os}-${arch}-${version}"
tarfile="releases/${tool}-${os}-${arch}-${version}.tar.gz"
set -x
GOOS=${os} GOARCH=${arch} go build -tags osusergo,netgo -ldflags "-extldflags=-static" -o ${binfile}
GOOS=${os} GOARCH=${arch} go build -tags osusergo,netgo -ldflags "-extldflags=-static -w" --trimpath $pie -o ${binfile}
strip --strip-all ${binfile}
mkdir -p ${tardir}
cp ${binfile} README.md LICENSE ${tardir}/
echo 'tool = kleingebaeck

View File

@@ -1,5 +1,5 @@
/*
Copyright © 2023-2024 Thomas von Dein
Copyright © 2023-2025 Thomas von Dein
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -50,7 +50,11 @@ func ScrapeUser(fetch *Fetcher) error {
if err != nil {
return err
}
defer body.Close()
defer func() {
if err := body.Close(); err != nil {
panic(err)
}
}()
err = goq.NewDecoder(body).Decode(&index)
if err != nil {
@@ -88,7 +92,13 @@ func ScrapeUser(fetch *Fetcher) error {
// scrape an ad. uri is the full uri of the ad, dir is the basedir
func ScrapeAd(fetch *Fetcher, uri string) error {
advertisement := &Ad{}
now := time.Now()
advertisement := &Ad{
Year: now.Format("2006"),
Month: now.Format("01"),
Day: now.Format("02"),
}
// extract slug and id from uri
uriparts := strings.Split(uri, "/")
@@ -106,7 +116,11 @@ func ScrapeAd(fetch *Fetcher, uri string) error {
if err != nil {
return err
}
defer body.Close()
defer func() {
if err := body.Close(); err != nil {
panic(err)
}
}()
// extract ad contents with goquery/goq
err = goq.NewDecoder(body).Decode(&advertisement)
@@ -124,18 +138,35 @@ func ScrapeAd(fetch *Fetcher, uri string) error {
return fmt.Errorf("could not extract ad data from page, got empty struct")
}
advertisement.DecodeAttributes()
advertisement.CalculateExpire()
// write listing
addir, err := WriteAd(fetch.Config, advertisement)
// prepare ad dir name
addir, err := AdDirName(fetch.Config, advertisement)
if err != nil {
return err
}
proceed := CheckAdVisited(fetch.Config, addir)
if !proceed {
return nil
}
// write listing
err = WriteAd(fetch.Config, advertisement, addir)
if err != nil {
return err
}
// tell the user
slog.Debug("extracted ad listing", "ad", advertisement)
// stats
fetch.Config.IncrAds()
// register for later checks
DirsVisited[addir] = 1
return ScrapeImages(fetch, advertisement, addir)
}
@@ -154,7 +185,9 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error {
for _, imguri := range advertisement.Images {
imguri := imguri
file := filepath.Join(adpath, fmt.Sprintf("%d.jpg", img))
// we append the suffix later in NewImage() based on image format
basefilename := filepath.Join(adpath, fmt.Sprintf("%d", img))
egroup.Go(func() error {
// wait a little
@@ -168,14 +201,19 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error {
}
buf := new(bytes.Buffer)
_, err = buf.ReadFrom(body)
if err != nil {
return fmt.Errorf("failed to read from image buffer: %w", err)
}
buf2 := buf.Bytes() // needed for image writing
reader := bytes.NewReader(buf.Bytes())
image, err := NewImage(reader, basefilename, imguri)
if err != nil {
return err
}
image := NewImage(buf, file, imguri)
err = image.CalcHash()
if err != nil {
return err
@@ -189,12 +227,17 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error {
}
}
err = WriteImage(file, buf2)
_, err = reader.Seek(0, 0)
if err != nil {
return fmt.Errorf("failed to seek(0) on image reader: %w", err)
}
err = WriteImage(image.Filename, reader)
if err != nil {
return err
}
slog.Debug("wrote image", "image", image, "size", len(buf2), "throttle", throttle)
slog.Debug("wrote image", "image", image, "size", buf.Len(), "throttle", throttle)
return nil
})

View File

@@ -1,5 +1,5 @@
/*
Copyright © 2023-2024 Thomas von Dein
Copyright © 2023-2025 Thomas von Dein
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -26,8 +26,36 @@ import (
"runtime"
"strings"
tpl "text/template"
"time"
)
type OutdirData struct {
Year, Day, Month string
}
func OutDirName(conf *Config) (string, error) {
tmpl, err := tpl.New("outdir").Parse(conf.Outdir)
if err != nil {
return "", fmt.Errorf("failed to parse outdir template: %w", err)
}
buf := bytes.Buffer{}
now := time.Now()
data := OutdirData{
Year: now.Format("2006"),
Month: now.Format("01"),
Day: now.Format("02"),
}
err = tmpl.Execute(&buf, data)
if err != nil {
return "", fmt.Errorf("failed to execute outdir template: %w", err)
}
return buf.String(), nil
}
func AdDirName(conf *Config, advertisement *Ad) (string, error) {
tmpl, err := tpl.New("adname").Parse(conf.Adnametemplate)
if err != nil {
@@ -44,19 +72,13 @@ func AdDirName(conf *Config, advertisement *Ad) (string, error) {
return buf.String(), nil
}
func WriteAd(conf *Config, advertisement *Ad) (string, error) {
// prepare ad dir name
addir, err := AdDirName(conf, advertisement)
if err != nil {
return "", err
}
func WriteAd(conf *Config, advertisement *Ad, addir string) error {
// prepare output dir
dir := filepath.Join(conf.Outdir, addir)
err = Mkdir(dir)
err := Mkdir(dir)
if err != nil {
return "", err
return err
}
// write ad file
@@ -64,9 +86,13 @@ func WriteAd(conf *Config, advertisement *Ad) (string, error) {
listingfd, err := os.Create(listingfile)
if err != nil {
return "", fmt.Errorf("failed to create Adlisting.txt: %w", err)
return fmt.Errorf("failed to create Adlisting.txt: %w", err)
}
defer listingfd.Close()
defer func() {
if err := listingfd.Close(); err != nil {
panic(err)
}
}()
if runtime.GOOS == WIN {
advertisement.Text = strings.ReplaceAll(advertisement.Text, "<br/>", "\r\n")
@@ -76,27 +102,31 @@ func WriteAd(conf *Config, advertisement *Ad) (string, error) {
tmpl, err := tpl.New("adlisting").Parse(conf.Template)
if err != nil {
return "", fmt.Errorf("failed to parse adlisting template: %w", err)
return fmt.Errorf("failed to parse adlisting template: %w", err)
}
err = tmpl.Execute(listingfd, advertisement)
if err != nil {
return "", fmt.Errorf("failed to execute adlisting template: %w", err)
return fmt.Errorf("failed to execute adlisting template: %w", err)
}
slog.Info("wrote ad listing", "listingfile", listingfile)
return addir, nil
return nil
}
func WriteImage(filename string, buf []byte) error {
func WriteImage(filename string, reader *bytes.Reader) error {
file, err := os.Create(filename)
if err != nil {
return fmt.Errorf("failed to open image file: %w", err)
}
defer file.Close()
defer func() {
if err := file.Close(); err != nil {
panic(err)
}
}()
_, err = file.Write(buf)
_, err = reader.WriteTo(file)
if err != nil {
return fmt.Errorf("failed to write to image file: %w", err)
@@ -127,9 +157,29 @@ func ReadImage(filename string) (*bytes.Buffer, error) {
func fileExists(filename string) bool {
info, err := os.Stat(filename)
if os.IsNotExist(err) {
if err != nil {
// return false on any error
return false
}
return !info.IsDir()
}
// check if an addir has already been processed by current run and
// decide what to do
func CheckAdVisited(conf *Config, adname string) bool {
if Exists(DirsVisited, adname) {
if conf.ForceDownload {
slog.Warn("an ad with the same name has already been downloaded, overwriting", "addir", adname)
return true
}
// don't overwrite
slog.Warn("an ad with the same name has already been downloaded, skipping (use -f to overwrite)", "addir", adname)
return false
}
// overwrite
return true
}

View File

@@ -18,6 +18,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"bytes"
"testing"
)
@@ -28,10 +29,10 @@ import (
func TestWriteImage(t *testing.T) {
t.Parallel()
buf := []byte{1, 2, 3, 4, 5, 6, 7, 8}
reader := bytes.NewReader([]byte{1, 2, 3, 4, 5, 6, 7, 8})
file := "t/out/t.jpg"
err := WriteImage(file, buf)
err := WriteImage(file, reader)
if err != nil {
t.Errorf("Could not write mock image to %s: %s", file, err.Error())
}

BIN
t/1.gif Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 62 B

BIN
t/1.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.2 KiB

BIN
t/1.webp Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.3 KiB

View File

@@ -1,5 +1,7 @@
#!/bin/sh -x
base="../kleinanzeigen"
rm -rf $base
mkdir -p $base
echo "Generating /s-bestandsliste.html"

13
util.go
View File

@@ -1,5 +1,5 @@
/*
Copyright © 2023 Thomas von Dein
Copyright © 2023-2024 Thomas von Dein
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -32,7 +32,7 @@ import (
func Mkdir(dir string) error {
if _, err := os.Stat(dir); errors.Is(err, os.ErrNotExist) {
err := os.Mkdir(dir, os.ModePerm)
err := os.MkdirAll(dir, os.ModePerm)
if err != nil {
return fmt.Errorf("failed to create directory %s: %w", dir, err)
}
@@ -74,3 +74,12 @@ func IsNoTty() bool {
func GetThrottleTime() time.Duration {
return time.Duration(rand.Intn(MaxThrottle-MinThrottle+1)+MinThrottle) * time.Millisecond
}
// look if a key in a map exists, generic variant
func Exists[K comparable, V any](m map[K]V, v K) bool {
if _, ok := m[v]; ok {
return true
}
return false
}