Compare commits

...

13 Commits

Author SHA1 Message Date
16b47aa7b0 fix docker 2026-10-12 21:27:27 +02:00
562292de2b dump version 2026-10-12 21:27:22 +02:00
7f2f61ca9e fix tests 2026-10-12 21:23:28 +02:00
4e77573223 satisfy linter 2026-10-12 14:30:28 +02:00
79892bc55b upd linter 2026-10-12 14:16:04 +02:00
75087b47bf fix #125: could not extract data from subsequent pages 2026-10-12 14:08:02 +02:00
3c9124a3e6 reorder for better readability 2026-10-12 14:07:51 +02:00
d20a499519 update go 2026-10-12 14:07:35 +02:00
0d5ee0b485 pin to older release, see woodpecker-plugins/docker-buildx#306 2026-01-07 10:39:38 +01:00
T. von Dein
b44c1ba1dc update x/image (#124) 2025-11-05 10:13:52 +01:00
b6350344be fix image links 2025-11-05 08:56:20 +01:00
e5ff2bbc5c fix changelog link 2025-11-05 08:45:12 +01:00
T. von Dein
230bbf3d53 migrate to codeberg (#123) 2025-11-05 08:35:00 +01:00
28 changed files with 462 additions and 278 deletions

View File

@@ -1,47 +0,0 @@
name: build-and-test
on: [push]
jobs:
build:
strategy:
matrix:
version: [1.24]
os: [ubuntu-latest, windows-latest, macos-latest]
name: Build
runs-on: ${{ matrix.os }}
steps:
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version: ${{ matrix.version }}
id: go
- name: checkout
uses: actions/checkout@v5
- name: build
run: go build
- name: test
run: make test
- name: Update coverage report
uses: ncruces/go-coverage-report@main
with:
report: true
chart: true
amend: true
if: |
matrix.os == 'ubuntu-latest' &&
github.event_name == 'push'
continue-on-error: true
golangci:
name: lint
runs-on: ubuntu-latest
steps:
- uses: actions/setup-go@v6
with:
go-version: 1.24
- uses: actions/checkout@v5
- name: golangci-lint
uses: golangci/golangci-lint-action@v8

View File

@@ -1,34 +0,0 @@
name: build-push-image
on:
push:
tags:
- 'v*'
jobs:
build-and-push-image:
runs-on: ubuntu-latest
permissions:
packages: write
steps:
- name: Checkout repository
uses: actions/checkout@v5
- name: Log in to the Container registry
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef
with:
registry: https://ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push Docker image
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83
with:
push: true
tags: ghcr.io/tlinden/kleingebaeck:${{ github.ref_name}}
- name: Build and push latest Docker image
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83
with:
push: true
tags: ghcr.io/tlinden/kleingebaeck:latest

View File

@@ -1,87 +0,0 @@
name: build-release
on:
push:
tags:
- "v*.*.*"
jobs:
release:
name: Build Release Assets
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v5
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version: 1.24.5
- name: Build the executables
run: ./mkrel.sh kleingebaeck ${{ github.ref_name}}
- name: List the executables
run: ls -l ./releases
- name: Upload the binaries
uses: svenstaro/upload-release-action@v2
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
tag: ${{ github.ref_name }}
file: ./releases/*
file_glob: true
- name: Build Changelog
id: github_release
uses: mikepenz/release-changelog-builder-action@v5
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
mode: "PR"
configurationJson: |
{
"template": "#{{CHANGELOG}}\n\n**Full Changelog**: #{{RELEASE_DIFF}}",
"pr_template": "- #{{TITLE}} (##{{NUMBER}}) by #{{AUTHOR}}\n#{{BODY}}",
"empty_template": "- no changes",
"categories": [
{
"title": "## New Features",
"labels": ["add", "feature"]
},
{
"title": "## Bug Fixes",
"labels": ["fix", "bug", "revert"]
},
{
"title": "## Documentation Enhancements",
"labels": ["doc"]
},
{
"title": "## Refactoring Efforts",
"labels": ["refactor"]
},
{
"title": "## Miscellaneus Changes",
"labels": []
}
],
"ignore_labels": [
"duplicate", "good first issue", "help wanted", "invalid", "question", "wontfix"
],
"label_extractor": [
{
"pattern": "(.) (.+)",
"target": "$1"
},
{
"pattern": "(.) (.+)",
"target": "$1",
"on_property": "title"
}
]
}
- name: Create Release
uses: softprops/action-gh-release@v2
with:
body: ${{steps.github_release.outputs.changelog}}

106
.golangci.yml Normal file
View File

@@ -0,0 +1,106 @@
version: "2"
linters:
enable:
- errcheck
- govet
- ineffassign
- staticcheck
- unused
- arangolint
- asasalint
- asciicheck
- bidichk
- bodyclose
- canonicalheader
- clickhouselint
- containedctx
#- contextcheck
- copyloopvar
#- cyclop
- decorder
- dogsled
- dupword
- durationcheck
- embeddedstructfieldcheck
- errchkjson
- errname
#- errorlint
- exhaustive
- exptostd
- fatcontext
- funcorder
#- funlen
- ginkgolinter
- gocheckcompilerdirectives
#- gochecknoinits
- gochecksumtype
#- gocognit
- gocritic
#- gocyclo
- goheader
- gomoddirectives
- gomodguard_v2
- goprintffuncname
#- gosec
- gosmopolitan
- grouper
- iface
- importas
- inamedparam
- interfacebloat
- intrange
- iotamixing
- lll
- loggercheck
- makezero
- misspell
- modernize
- nakedret
#- nestif
- nilerr
- nilnesserr
- nlreturn
- nonamedreturns
- nosprintfhostport
- perfsprint
- prealloc
- promlinter
- protogetter
- reassign
- recvcheck
- rowserrcheck
- sloglint
- spancheck
- sqlclosecheck
- tagalign
- testableexamples
- testifylint
- testpackage
- thelper
- tparallel
- unconvert
- unparam
- unqueryvet
- usestdlibvars
- usetesting
- varnamelen
- wastedassign
- whitespace
- wsl_v5
- zerologlint
settings:
varnamelen:
ignore-names:
- err
- wg
- mu
- ts
- to
- es
- op
- fd
- id
- fn

69
.goreleaser.yaml Normal file
View File

@@ -0,0 +1,69 @@
# vim: set ts=2 sw=2 tw=0 fo=cnqoj
version: 2
before:
hooks:
- go mod tidy
gitea_urls:
api: https://codeberg.org/api/v1
download: https://codeberg.org
builds:
- env:
- CGO_ENABLED=0
goos:
- linux
- windows
- darwin
- freebsd
archives:
- formats: [tar.gz]
# this name template makes the OS and Arch compatible with the results of `uname`.
name_template: >-
{{ .ProjectName }}_
{{- title .Os }}_
{{- if eq .Arch "amd64" }}x86_64
{{- else if eq .Arch "386" }}i386
{{- else }}{{ .Arch }}{{ end }}
{{- if .Arm }}v{{ .Arm }}{{ end }}_{{ .Tag }}
# use zip for windows archives
format_overrides:
- goos: windows
formats: [zip]
- goos: linux
formats: [tar.gz,binary]
files:
- src: "*.md"
strip_parent: true
- src: "docs/*"
strip_parent: true
- src: Makefile.dist
dst: Makefile
wrap_in_directory: true
changelog:
sort: asc
filters:
exclude:
- "^docs:"
- "^test:"
groups:
- title: Improved
regexp: '^.*?(feat|add|new)(\([[:word:]]+\))??!?:.+$'
order: 0
- title: Fixed
regexp: '^.*?(bug|fix)(\([[:word:]]+\))??!?:.+$'
order: 1
- title: Changed
order: 999
release:
header: "# Release Notes"
footer: >-
---
Full Changelog: [{{ .PreviousTag }}...{{ .Tag }}](https://codeberg.org/scip/kleingebaeck/compare/{{ .PreviousTag }}...{{ .Tag }})

38
.woodpecker/build.yaml Normal file
View File

@@ -0,0 +1,38 @@
matrix:
platform:
- linux/amd64
goversion:
- 1.26
lintversion:
- v2.12.2
labels:
platform: ${platform}
steps:
build:
when:
event: [push]
image: golang:${goversion}
commands:
- go get
- go build
linter:
when:
event: [push]
image: golang:${goversion}
commands:
- curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin ${lintversion}
- golangci-lint --version
- golangci-lint run ./...
depends_on: [build]
test:
when:
event: [push]
image: golang:${goversion}
commands:
- go get
- go test -v -cover
depends_on: [build,linter]

32
.woodpecker/image.yaml Normal file
View File

@@ -0,0 +1,32 @@
# https://woodpecker-ci.org/plugins/docker-buildx
# enable Package unit and go to /scip/-/packages after building to link to proj
variables:
- &repo codeberg.org/${CI_REPO_OWNER}/kleingebaeck
steps:
dryrun:
image: docker.io/woodpeckerci/plugin-docker-buildx:6.0.3
settings:
dockerfile: Dockerfile
platforms: linux/amd64
dry_run: true
repo: *repo
tags: latest
when:
event: [pull_request]
publish:
image: docker.io/woodpeckerci/plugin-docker-buildx:6.0.3
settings:
dockerfile: Dockerfile
platforms: linux/amd64
repo: *repo
registry: codeberg.org
tags: latest,${CI_COMMIT_SHA:0:8},${CI_COMMIT_TAG}
username: ${CI_REPO_OWNER}
password:
from_secret: REGISTRY_TOKEN
when:
event: [tag]
branch: main

15
.woodpecker/release.yaml Normal file
View File

@@ -0,0 +1,15 @@
# build release
labels:
platform: linux/amd64
steps:
goreleaser:
image: goreleaser/goreleaser
when:
event: [tag]
environment:
GITEA_TOKEN:
from_secret: DEPLOY_TOKEN
commands:
- goreleaser release --clean --verbose

View File

@@ -1,4 +1,4 @@
FROM golang:1.24-alpine as builder
FROM golang:1.26-alpine as builder
RUN apk update
RUN apk upgrade

20
Makefile.dist Normal file
View File

@@ -0,0 +1,20 @@
# -*-make-*-
.PHONY: install all
tool = rpn
PREFIX = /usr/local
UID = root
GID = 0
all:
@echo "Type 'sudo make install' to install the tool."
@echo "To change prefix, type 'sudo make install PREFIX=/opt'"
install:
install -d -o $(UID) -g $(GID) $(PREFIX)/bin
install -d -o $(UID) -g $(GID) $(PREFIX)/man/man1
install -d -o $(UID) -g $(GID) $(PREFIX)/share/doc
install -o $(UID) -g $(GID) -m 555 $(tool) $(PREFIX)/sbin/
install -o $(UID) -g $(GID) -m 444 $(tool).1 $(PREFIX)/man/man1/
install -o $(UID) -g $(GID) -m 444 *.md $(PREFIX)/share/doc/

View File

@@ -1,13 +1,12 @@
## Kleingebäck - kleinanzeigen.de Backup
![Kleingebaeck Logo](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/kleingebaecklogo-small.png)
![Kleingebaeck Logo](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/kleingebaecklogo-small.png)
[![Go Report Card](https://goreportcard.com/badge/github.com/tlinden/kleingebaeck)](https://goreportcard.com/report/github.com/tlinden/kleingebaeck)
[![Actions](https://github.com/tlinden/kleingebaeck/actions/workflows/ci.yaml/badge.svg)](https://github.com/tlinden/kleingebaeck/actions)
[![Go Coverage](https://github.com/tlinden/kleingebaeck/wiki/coverage.svg)](https://raw.githack.com/wiki/tlinden/kleingebaeck/coverage.html)
[![Go Report Card](https://goreportcard.com/badge/codeberg.org/scip/kleingebaeck)](https://goreportcard.com/report/codeberg.org/scip/kleingebaeck)
[![status-badge](https://ci.codeberg.org/api/badges/15530/status.svg)](https://ci.codeberg.org/repos/15530)
![GitHub License](https://img.shields.io/github/license/tlinden/kleingebaeck)
[![GitHub release](https://img.shields.io/github/v/release/tlinden/kleingebaeck?color=%2300a719)](https://github.com/TLINDEN/kleingebaeck/releases/latest)
[![English](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/english.png)](https://github.com/tlinden/kleingebaeck/blob/main/README.md)
[![GitHub release](https://img.shields.io/github/v/release/tlinden/kleingebaeck?color=%2300a719)](https://codeberg.org/scip/kleingebaeck/releases)
[![English](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/english.png)](https://codeberg.org/scip/kleingebaeck/raw/branch/main/README.md)
Mit diesem Tool kann man seine Anzeigen bei https://kleinanzeigen.de sichern.
@@ -23,40 +22,40 @@ Fertige vorcompilierte Programme älter als Version `v0.3.12` sind von
Schwachstellen in der Behandlung von HTTP und Zertifikaten
betroffen. Falls Du eine ältere Kleingebäck-Version im Einsatz hast,
bitte update auf Version `v0.3.12` oder höher. Bitte lies auch die [Release Notes für
v0.3.12](https://github.com/TLINDEN/kleingebaeck/releases/tag/v0.3.12)
v0.3.12](https://codeberg.org/scip/kleingebaeck/releases/tag/v0.3.12)
für mehr Details.
## Screenshots
Das ist die Hauptseite meines kleinanzeigen.de Accounts:
![Index](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/kleinanzeigen-index.png)
![Index](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/kleinanzeigen-index.png)
Sichern ich meine Anzeigen:
![Download](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/kleinanzeigen-download.png)
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/kleinanzeigen-download.png)
Backupverzeichnis nach dem Download:
![Download](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/kleinanzeigen-backup.png)
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/kleinanzeigen-backup.png)
Verzeichnis einer Anzeige:
![Download](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/kleinanzeigen-ad.png)
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/kleinanzeigen-ad.png)
**Das gleiche unter Windows:**
Anzeigen Sichern:
![Download](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/cmd-windows.jpg)
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/cmd-windows.jpg)
Backupverzeichnis nach dem Download
![Download](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/liste-windows.jpg)
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/liste-windows.jpg)
Und eine Anzeige:
![Download](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/adlisting-windows.jpg)
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/adlisting-windows.jpg)
## Installation
@@ -66,7 +65,7 @@ für seine Plattform herunter und kann direkt loslegen.
### Installation des vorcompilierten Programms
Auf der Seite [des letzten Releases](https://github.com/TLINDEN/kleingebaeck/releases/latest) findet man das Program für sein Betriebssystem und die Plattform (z.b. Windows + Intel)
Auf der Seite [des letzten Releases](https://codeberg.org/scip/kleingebaeck/releases) findet man das Program für sein Betriebssystem und die Plattform (z.b. Windows + Intel)
Es gibt 2 Varianten:
@@ -105,7 +104,7 @@ hilfreich, aber nicht unbedingt erforderlich.
Um das Programm zu compilieren, muss man folgende Schritte ausführen:
```shell
git clone https://github.com/TLINDEN/kleingebaeck.git
git clone https://codeberg.org/scip/kleingebaeck.git
cd kleingebaeck
go mod tidy
make # (oder make)
@@ -146,7 +145,7 @@ gemountet.
Die Optionen `-u XXX -v` sind kleingebäck Optionen. Ersetze `XXX`
durch Deine tatsächliche kleinanzeigen.de Userid.
Eine Liste verfügbarer Images findet man [hier](https://github.com/tlinden/kleingebaeck/pkgs/container/kleingebaeck/versions?filters%5Bversion_type%5D=tagged)
Eine Liste verfügbarer Images findet man [hier](https://codeberg.org/scip/kleingebaeck/pkgs/container/kleingebaeck/versions?filters%5Bversion_type%5D=tagged)
## Kommandozeilen Optionen:
@@ -277,7 +276,7 @@ ignorieren, die beim Bilderdownload auftreten könnten.
## Documentation
Die Dokumentation kann man
[online](https://github.com/TLINDEN/kleingebaeck/blob/main/kleingebaeck.pod)
[online](https://codeberg.org/scip/kleingebaeck/raw/branch/main/kleingebaeck.pod)
oder lokal lesen mit: `kleingebaeck --manual`. Hat man das Tool mit
dem Tarball installiert, funktioniert auch `man kleingebaeck`.
@@ -293,7 +292,7 @@ das doch der beste Weg, die Anfrage zu übersehen und zu vergessen.
Um einen Fehler, ein unerwartetes Verhalten, eine Feature Request oder
einen Patch zu übermitteln, eröffne daher bitte einen Issue unter:
https://github.com/TLINDEN/kleingebaeck/issues. Danke!
https://codeberg.org/scip/kleingebaeck/issues. Danke!
Bitte gebe den fehlgeschlagenen Befehl an, rufe es auch mit Debugging
`-d` auf.

View File

@@ -1,13 +1,12 @@
## Kleingebäck - kleinanzeigen.de Backup
![Kleingebaeck Logo](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/kleingebaecklogo-small.png)
![Kleingebaeck Logo](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/kleingebaecklogo-small.png)
[![Go Report Card](https://goreportcard.com/badge/github.com/tlinden/kleingebaeck)](https://goreportcard.com/report/github.com/tlinden/kleingebaeck)
[![Actions](https://github.com/tlinden/kleingebaeck/actions/workflows/ci.yaml/badge.svg)](https://github.com/tlinden/kleingebaeck/actions)
[![Go Coverage](https://github.com/tlinden/kleingebaeck/wiki/coverage.svg)](https://raw.githack.com/wiki/tlinden/kleingebaeck/coverage.html)
[![Go Report Card](https://goreportcard.com/badge/codeberg.org/scip/kleingebaeck)](https://goreportcard.com/report/codeberg.org/scip/kleingebaeck)
[![status-badge](https://ci.codeberg.org/api/badges/15530/status.svg)](https://ci.codeberg.org/repos/15530)
![GitHub License](https://img.shields.io/github/license/tlinden/kleingebaeck)
[![GitHub release](https://img.shields.io/github/v/release/tlinden/kleingebaeck?color=%2300a719)](https://github.com/TLINDEN/kleingebaeck/releases/latest)
[![German](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/german.png)](https://github.com/tlinden/kleingebaeck/blob/main/README-de.md)
[![GitHub release](https://img.shields.io/github/v/release/tlinden/kleingebaeck?color=%2300a719)](https://codeberg.org/scip/kleingebaeck/releases)
[![German](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/german.png)](https://codeberg.org/scip/kleingebaeck/raw/branch/main/README-de.md)
[Die deutsche Version des READMEs findet Ihr hier](README-de.md).
@@ -24,40 +23,40 @@ Binary releases prior to version `v0.3.11` are affected by
vulnerabilities in HTTP and certificate handling. If you are using
such a binary, please update to `v0.3.12` or higher. Please also refer
to the [Release Notes of
v0.3.12](https://github.com/TLINDEN/kleingebaeck/releases/tag/v0.3.12)
v0.3.12](https://codeberg.org/scip/kleingebaeck/releases/tag/v0.3.12)
for more details.
## Screenshots
This is the index of my kleinanzeigen.de Account:
![Index](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/kleinanzeigen-index.png)
![Index](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/kleinanzeigen-index.png)
Here I download my ads on the commandline:
![Download](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/kleinanzeigen-download.png)
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/kleinanzeigen-download.png)
And this is the backup directory after download:
![Download](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/kleinanzeigen-backup.png)
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/kleinanzeigen-backup.png)
Here's a directory for one ad:
![Download](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/kleinanzeigen-ad.png)
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/kleinanzeigen-ad.png)
**The same thing under windows:**
Downloading ads:
![Download](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/cmd-windows.jpg)
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/cmd-windows.jpg)
Backup directory after download:
![Download](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/liste-windows.jpg)
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/liste-windows.jpg)
And one ad listing directory:
![Download](https://github.com/TLINDEN/kleingebaeck/blob/main/.github/assets/adlisting-windows.jpg)
![Download](https://codeberg.org/scip/kleingebaeck/raw/branch/main/.github/assets/adlisting-windows.jpg)
## Installation
@@ -68,7 +67,7 @@ releases page and you're good to go.
### Installation using a pre-compiled binary
Go to the [latest release
page](https://github.com/TLINDEN/kleingebaeck/releases/latest) and
page](https://codeberg.org/scip/kleingebaeck/releases) and
look for your OS and platform. There are two options to install the binary:
1. Directly download the binary for your platform,
@@ -138,7 +137,7 @@ docker run. And the local directory `myads` will be mapped to
The options `-u XXX -v` are kleingebaeck options, replace `XXX` with
your actual kleinanzeigen.de user id.
A list of available images is [here](https://github.com/tlinden/kleingebaeck/pkgs/container/kleingebaeck/versions?filters%5Bversion_type%5D=tagged)
A list of available images is [here](https://codeberg.org/scip/kleingebaeck/pkgs/container/kleingebaeck/versions?filters%5Bversion_type%5D=tagged)
## Commandline options:
@@ -262,7 +261,7 @@ errors.
## Documentation
You can read the documentation [online](https://github.com/TLINDEN/kleingebaeck/blob/main/kleingebaeck.pod) or locally once you have installed kleingebaeck with: `kleingebaeck --manual`.
You can read the documentation [online](https://codeberg.org/scip/kleingebaeck/raw/branch/main/kleingebaeck.pod) or locally once you have installed kleingebaeck with: `kleingebaeck --manual`.
## Kleingebäck?
@@ -278,7 +277,7 @@ that's the best way for me to forget to do something.
In order to report a bug, unexpected behavior, feature requests or to
submit a patch, please open an issue on github:
https://github.com/TLINDEN/kleingebaeck/issues.
https://codeberg.org/scip/kleingebaeck/issues.
Please repeat the failing command with debugging enabled `-d` and
include the output in the issue.

View File

@@ -11,7 +11,7 @@ I don't agree with the "responsible disclosure" process most projects
(and companies) work these days.
So, if you find a vulnerability of any kind, please just open an
[issue](https://github.com/TLINDEN/kleingebaeck/issues). Please add
[issue](https://codeberg.org/scip/kleingebaeck/issues). Please add
all details required to reproduce the vulnerability. You won't be chased.
That's just all about it.

22
ad.go
View File

@@ -29,16 +29,8 @@ type Index struct {
}
type Ad struct {
Title string `goquery:"h1"`
Slug string
ID string
Details string `goquery:".addetailslist--detail,text"`
Attributes map[string]string // processed afterwards
Condition string // post processed from details for backward compatibility
Type string // post processed from details for backward compatibility
Color string // post processed from details for backward compatibility
Material string // post processed from details for backward compatibility
Category string
Title string `goquery:"h1"`
Details string `goquery:".addetailslist--detail,text"`
CategoryTree []string `goquery:".breadcrump-link,text"`
Price string `goquery:"h2#viewad-price"`
Created string `goquery:"#viewad-extra-info,text"`
@@ -46,6 +38,14 @@ type Ad struct {
Images []string `goquery:".galleryimage-element img,[src]"`
Shipping string `goquery:".boxedarticle--details--shipping,text"` // not always filled
Expire string
Slug string
ID string
Attributes map[string]string // processed afterwards
Condition string // post processed from details for backward compatibility
Type string // post processed from details for backward compatibility
Color string // post processed from details for backward compatibility
Material string // post processed from details for backward compatibility
Category string
// runtime computed
Year, Day, Month string
@@ -94,7 +94,7 @@ func (ad *Ad) CalculateExpire() {
/*
Decode attributes like color or condition. See
https://github.com/TLINDEN/kleingebaeck/issues/117
https://codeberg.org/scip/kleingebaeck/issues/117
for more details. In short: the HTML delivered by
kleinanzeigen.de has no css attribute for the keys
so we cannot extract key=>value mappings of the

View File

@@ -34,7 +34,7 @@ import (
)
const (
VERSION string = "0.3.23"
VERSION string = "0.3.25"
Baseuri string = "https://www.kleinanzeigen.de"
Listuri string = "/s-bestandsliste.html"
Defaultdir string = "."
@@ -140,7 +140,7 @@ func InitConfig(output io.Writer) (*Config, error) {
}
// Load default values using the confmap provider.
if err := kloader.Load(confmap.Provider(map[string]interface{}{
if err := kloader.Load(confmap.Provider(map[string]any{
"template": template,
"outdir": DefaultOutdirTemplate,
"loglevel": "notice",
@@ -200,7 +200,6 @@ func InitConfig(output io.Writer) (*Config, error) {
// Load the config file[s]
for _, cfgfile := range configfiles {
path, err := os.Stat(cfgfile)
if err != nil {
// ignore non-existent files, but bail out on any other errors
if !os.IsNotExist(err) {

View File

@@ -24,7 +24,11 @@ import (
"log/slog"
"net/http"
"net/http/cookiejar"
"net/http/httputil"
"net/url"
"os"
"strings"
"sync"
)
// convenient wrapper to fetch some web content
@@ -34,12 +38,25 @@ type Fetcher struct {
Cookies []*http.Cookie
}
// Pos is a counter for debug output file names
type Pos struct {
sync.Mutex
pos int
}
var pos *Pos
func NewFetcher(conf *Config) (*Fetcher, error) {
jar, err := cookiejar.New(nil)
if err != nil {
return nil, fmt.Errorf("failed to create a cookie jar obj: %w", err)
}
if pos == nil {
pos = &Pos{}
}
return &Fetcher{
Client: &http.Client{
Transport: &loggingTransport{}, // implemented in http.go
@@ -82,9 +99,43 @@ func (f *Fetcher) Get(uri string) (io.ReadCloser, error) {
slog.Debug("got cookies?", "cookies", res.Cookies())
f.Cookies = res.Cookies()
if err := f.Dump(res); err != nil {
return nil, err
}
return res.Body, nil
}
func (f *Fetcher) Dump(res *http.Response) error {
if f.Config.Debug && strings.Contains(res.Header.Get("Content-Type"), "text/html") {
// => dump content
pos.Lock()
defer pos.Unlock()
pos.pos++
dump, err := httputil.DumpResponse(res, true)
if err != nil {
return err
}
file := fmt.Sprintf("out%d.html", pos.pos)
fd, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
return err
}
if _, err := fd.Write(dump); err != nil {
return err
}
slog.Debug("dump content", "file", file)
}
return nil
}
// fetch an image
func (f *Fetcher) Getimage(uri string) (io.ReadCloser, error) {
slog.Debug("fetching ad image", "uri", uri)

6
go.mod
View File

@@ -1,8 +1,6 @@
module kleingebaeck
go 1.24.0
toolchain go1.24.5
go 1.26.0
require (
astuart.co/goq v1.0.0
@@ -19,7 +17,7 @@ require (
github.com/mattn/go-isatty v0.0.20
github.com/spf13/pflag v1.0.10
github.com/tlinden/yadu v0.1.3
golang.org/x/image v0.31.0
golang.org/x/image v0.32.0
golang.org/x/sync v0.17.0
)

2
go.sum
View File

@@ -68,6 +68,8 @@ github.com/tlinden/yadu v0.1.3/go.mod h1:l3bRmHKL9zGAR6pnBHY2HRPxBecf7L74BoBgOOp
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/image v0.31.0 h1:mLChjE2MV6g1S7oqbXC0/UcKijjm5fnJLUYKIYrLESA=
golang.org/x/image v0.31.0/go.mod h1:R9ec5Lcp96v9FTF+ajwaH3uGxPH4fKfHHAVbUILxghA=
golang.org/x/image v0.32.0 h1:6lZQWq75h7L5IWNk0r+SCpUJ6tUVd3v4ZHnbRKLkUDQ=
golang.org/x/image v0.32.0/go.mod h1:/R37rrQmKXtO6tYXAjtDLwQgFLHmhW+V6ayXlxzP2Pc=
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190606173856-1492cefac77f/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=

View File

@@ -123,7 +123,6 @@ func (t *loggingTransport) RoundTrip(req *http.Request) (*http.Response, error)
// actual retry
resp, err = http.DefaultTransport.RoundTrip(req)
if err == nil {
slog.Debug("RESPONSE", "id", requestid, "status", resp.StatusCode,
"contentlength", resp.ContentLength, "retry", retries)

View File

@@ -19,6 +19,7 @@ package main
import (
"bytes"
"errors"
"fmt"
"image"
_ "image/gif"
@@ -27,6 +28,7 @@ import (
"log/slog"
"os"
"path/filepath"
"slices"
_ "golang.org/x/image/webp"
@@ -43,18 +45,6 @@ type Image struct {
Mime string
}
// used for logging to avoid printing Data
func (img *Image) LogValue() slog.Value {
return slog.GroupValue(
slog.String("filename", img.Filename),
slog.String("uri", img.URI),
slog.String("hash", img.Hash.ToString()),
)
}
// holds all images of an ad
type Cache []*goimagehash.ImageHash
// filename comes from the scraper, it contains directory/base w/o suffix
func NewImage(buf *bytes.Reader, filename, uri string) (*Image, error) {
_, imgconfig, err := image.DecodeConfig(buf)
@@ -75,7 +65,7 @@ func NewImage(buf *bytes.Reader, filename, uri string) (*Image, error) {
}
if imgconfig == "" {
return nil, fmt.Errorf("failed to process image: unknown or unsupported image format (supported: jpg,png,gif,webp)")
return nil, errors.New("failed to process image: unknown or unsupported image format (supported: jpg,png,gif,webp)")
}
filename += "." + imgconfig
@@ -92,6 +82,18 @@ func NewImage(buf *bytes.Reader, filename, uri string) (*Image, error) {
return img, nil
}
// used for logging to avoid printing Data
func (img *Image) LogValue() slog.Value {
return slog.GroupValue(
slog.String("filename", img.Filename),
slog.String("uri", img.URI),
slog.String("hash", img.Hash.ToString()),
)
}
// holds all images of an ad
type Cache []*goimagehash.ImageHash
// Calculate diff hash of the image
func (img *Image) CalcHash() error {
jpgdata, format, err := image.Decode(img.Data)
@@ -100,7 +102,7 @@ func (img *Image) CalcHash() error {
}
if format == "" {
return fmt.Errorf("failed to decode image: unknown or unsupported image format (supported: jpg,png,gif,webp)")
return errors.New("failed to decode image: unknown or unsupported image format (supported: jpg,png,gif,webp)")
}
hash1, err := goimagehash.DifferenceHash(jpgdata)
@@ -134,13 +136,9 @@ func (img *Image) Similar(hash *goimagehash.ImageHash) bool {
// check current image against all known hashes.
func (img *Image) SimilarExists(cache Cache) bool {
for _, otherimg := range cache {
if img.Similar(otherimg) {
return true
}
}
return false
return slices.ContainsFunc(cache, func(otherimg *goimagehash.ImageHash) bool {
return img.Similar(otherimg)
})
}
// read all JPG images in a ad directory, compute diff hashes and

View File

@@ -353,7 +353,7 @@ commandline flags take precedence over env!
.IX Header "BUGS"
In order to report a bug, unexpected behavior, feature requests
or to submit a patch, please open an issue on github:
<https://github.com/TLINDEN/kleingebaeck/issues>.
<https://codeberg.org/scip/kleingebaeck/issues>.
.PP
Please repeat the failing command with debugging enabled \f(CW\*(C`\-d\*(C'\fR and
include the output in the issue.

View File

@@ -170,7 +170,7 @@ ENVIRONMENT VARIABLES
BUGS
In order to report a bug, unexpected behavior, feature requests or to
submit a patch, please open an issue on github:
<https://github.com/TLINDEN/kleingebaeck/issues>.
<https://codeberg.org/scip/kleingebaeck/issues>.
Please repeat the failing command with debugging enabled "-d" and
include the output in the issue.

View File

@@ -207,7 +207,7 @@ commandline flags take precedence over env!
In order to report a bug, unexpected behavior, feature requests
or to submit a patch, please open an issue on github:
L<https://github.com/TLINDEN/kleingebaeck/issues>.
L<https://codeberg.org/scip/kleingebaeck/issues>.
Please repeat the failing command with debugging enabled C<-d> and
include the output in the issue.

12
main.go
View File

@@ -49,6 +49,7 @@ func init() {
fmt.Println("Please open a command shell and run it from there.")
fmt.Println()
fmt.Print("Press any key to quit: ")
_, err := bufio.NewReader(os.Stdin).ReadString('\n')
if err != nil {
panic(err)
@@ -144,13 +145,8 @@ func Main(output io.Writer) int {
if err != nil {
return Die(err)
}
conf.Outdir = outdir
// used for all HTTP requests
fetch, err := NewFetcher(conf)
if err != nil {
return Die(err)
}
conf.Outdir = outdir
// setup ad dir registry, needed to check for duplicates
DirsVisited = make(map[string]int)
@@ -159,14 +155,14 @@ func Main(output io.Writer) int {
case len(conf.Adlinks) >= 1:
// directly backup ad listing[s]
for _, uri := range conf.Adlinks {
err := ScrapeAd(fetch, uri)
err = ScrapeAd(conf, uri)
if err != nil {
return Die(err)
}
}
case conf.User > 0:
// backup all ads of the given user (via config or cmdline)
err := ScrapeUser(fetch)
err := ScrapeUser(conf)
if err != nil {
return Die(err)
}

View File

@@ -405,6 +405,7 @@ func InitValidSources() []Adsource {
empty := AdConfig{}
// prepare urls for the listing pages
// nolint:prealloc
ads := []Adsource{
{
uri: fmt.Sprintf("%s%s?userId=1", Baseuri, Listuri),
@@ -436,7 +437,7 @@ func InitInvalidSources() []Adsource {
ads := []Adsource{
{
// valid ad page but without content
uri: fmt.Sprintf("%s/s-anzeige/empty/1", Baseuri),
uri: Baseuri + "/s-anzeige/empty/1",
content: GetTemplate(nil, &empty, EMPTYPAGE),
},
{
@@ -446,18 +447,18 @@ func InitInvalidSources() []Adsource {
},
{
// some invalid page path
uri: fmt.Sprintf("%s/anzeige/name/1", Baseuri),
uri: Baseuri + "/anzeige/name/1",
content: GetTemplate(nil, &empty, "<html></html>"),
},
{
// some none-ad page
uri: fmt.Sprintf("%s/anzeige/name/1/foo/bar", Baseuri),
uri: Baseuri + "/anzeige/name/1/foo/bar",
content: GetTemplate(nil, &empty, "<html>HTTP 404: /eine-anzeige/ does not exist!</html>"),
status: 404,
},
{
// valid ad page but 503
uri: fmt.Sprintf("%s/s-anzeige/503/1", Baseuri),
uri: Baseuri + "/s-anzeige/503/1",
content: GetTemplate(nil, &empty, "<html>HTTP 503: service unavailable</html>"),
status: 503,
},
@@ -531,6 +532,7 @@ func TestMain(t *testing.T) {
defer func() { os.Args = oldargs }()
httpmock.Activate()
defer httpmock.DeactivateAndReset()
// prepare httpmock responders
@@ -568,6 +570,7 @@ func TestMainInvalids(t *testing.T) {
defer func() { os.Args = oldargs }()
httpmock.Activate()
defer httpmock.DeactivateAndReset()
// prepare httpmock responders

View File

@@ -19,7 +19,9 @@ package main
import (
"bytes"
"errors"
"fmt"
"io"
"log/slog"
"path/filepath"
"strconv"
@@ -32,30 +34,44 @@ import (
// extract links from all ad listing pages (that is: use pagination)
// and scrape every page
func ScrapeUser(fetch *Fetcher) error {
func ScrapeUser(conf *Config) error {
adlinks := []string{}
baseuri := fmt.Sprintf("%s%s?userId=%d", Baseuri, Listuri, fetch.Config.User)
baseuri := fmt.Sprintf("%s%s?userId=%d", Baseuri, Listuri, conf.User)
page := 1
uri := baseuri
slog.Info("fetching ad pages", "user", fetch.Config.User)
slog.Info("fetching ad pages", "user", conf.User)
for {
var index Index
slog.Debug("fetching page", "uri", uri)
body, err := fetch.Get(uri)
fetch, err := NewFetcher(conf)
if err != nil {
return err
}
body, err := fetch.Get(uri)
defer func() {
if body == nil {
return
}
// make sure body is completely flushed
if _, err := io.Copy(io.Discard, body); err != nil {
panic(err)
}
if err := body.Close(); err != nil {
panic(err)
}
}()
if err != nil {
return err
}
err = goq.NewDecoder(body).Decode(&index)
if err != nil {
return fmt.Errorf("failed to goquery decode HTML index body: %w", err)
@@ -77,12 +93,12 @@ func ScrapeUser(fetch *Fetcher) error {
}
for index, adlink := range adlinks {
err := ScrapeAd(fetch, Baseuri+adlink)
err := ScrapeAd(conf, Baseuri+adlink)
if err != nil {
return err
}
if fetch.Config.Limit > 0 && index == fetch.Config.Limit-1 {
if conf.Limit > 0 && index == conf.Limit-1 {
break
}
}
@@ -91,7 +107,7 @@ func ScrapeUser(fetch *Fetcher) error {
}
// scrape an ad. uri is the full uri of the ad, dir is the basedir
func ScrapeAd(fetch *Fetcher, uri string) error {
func ScrapeAd(conf *Config, uri string) error {
now := time.Now()
advertisement := &Ad{
@@ -112,16 +128,31 @@ func ScrapeAd(fetch *Fetcher, uri string) error {
// get the ad
slog.Debug("fetching ad page", "uri", uri)
body, err := fetch.Get(uri)
fetch, err := NewFetcher(conf)
if err != nil {
return err
}
body, err := fetch.Get(uri)
defer func() {
if body == nil {
return
}
// make sure body is completely flushed
if _, err := io.Copy(io.Discard, body); err != nil {
panic(err)
}
if err := body.Close(); err != nil {
panic(err)
}
}()
if err != nil {
return err
}
// extract ad contents with goquery/goq
err = goq.NewDecoder(body).Decode(&advertisement)
if err != nil {
@@ -135,7 +166,7 @@ func ScrapeAd(fetch *Fetcher, uri string) error {
if advertisement.Incomplete() {
slog.Debug("got ad", "ad", advertisement)
return fmt.Errorf("could not extract ad data from page, got empty struct")
return errors.New("could not extract ad data from page, got empty struct")
}
advertisement.DecodeAttributes()
@@ -184,14 +215,11 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error {
egroup := new(errgroup.Group)
for _, imguri := range advertisement.Images {
imguri := imguri
// we append the suffix later in NewImage() based on image format
basefilename := filepath.Join(adpath, fmt.Sprintf("%d", img))
basefilename := filepath.Join(adpath, strconv.Itoa(img))
egroup.Go(func() error {
// wait a little
throttle := GetThrottleTime()
time.Sleep(throttle)
@@ -241,6 +269,7 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error {
return nil
})
img++
}

View File

@@ -127,7 +127,6 @@ func WriteImage(filename string, reader *bytes.Reader) error {
}()
_, err = reader.WriteTo(file)
if err != nil {
return fmt.Errorf("failed to write to image file: %w", err)
}
@@ -157,7 +156,6 @@ func ReadImage(filename string) (*bytes.Buffer, error) {
func fileExists(filename string) bool {
info, err := os.Stat(filename)
if err != nil {
// return false on any error
return false
@@ -172,11 +170,13 @@ func CheckAdVisited(conf *Config, adname string) bool {
if Exists(DirsVisited, adname) {
if conf.ForceDownload {
slog.Warn("an ad with the same name has already been downloaded, overwriting", "addir", adname)
return true
}
// don't overwrite
slog.Warn("an ad with the same name has already been downloaded, skipping (use -f to overwrite)", "addir", adname)
return false
}

View File

@@ -53,7 +53,6 @@ func man() error {
man.Stderr = os.Stderr
err := man.Run()
if err != nil {
return fmt.Errorf("failed to execute 'less': %w", err)
}