mirror of
https://codeberg.org/scip/kleingebaeck.git
synced 2026-11-22 22:48:59 +01:00
Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d9f2f0a712 | ||
| 0d5ee0b485 | |||
|
|
b44c1ba1dc | ||
| b6350344be | |||
| e5ff2bbc5c | |||
|
|
230bbf3d53 |
47
.github/workflows/ci.yaml
vendored
47
.github/workflows/ci.yaml
vendored
@@ -1,47 +0,0 @@
|
|||||||
name: build-and-test
|
|
||||||
on: [push]
|
|
||||||
jobs:
|
|
||||||
build:
|
|
||||||
strategy:
|
|
||||||
matrix:
|
|
||||||
version: [1.24]
|
|
||||||
os: [ubuntu-latest, windows-latest, macos-latest]
|
|
||||||
name: Build
|
|
||||||
runs-on: ${{ matrix.os }}
|
|
||||||
steps:
|
|
||||||
- name: Set up Go
|
|
||||||
uses: actions/setup-go@v6
|
|
||||||
with:
|
|
||||||
go-version: ${{ matrix.version }}
|
|
||||||
id: go
|
|
||||||
|
|
||||||
- name: checkout
|
|
||||||
uses: actions/checkout@v5
|
|
||||||
|
|
||||||
- name: build
|
|
||||||
run: go build
|
|
||||||
|
|
||||||
- name: test
|
|
||||||
run: make test
|
|
||||||
|
|
||||||
- name: Update coverage report
|
|
||||||
uses: ncruces/go-coverage-report@main
|
|
||||||
with:
|
|
||||||
report: true
|
|
||||||
chart: true
|
|
||||||
amend: true
|
|
||||||
if: |
|
|
||||||
matrix.os == 'ubuntu-latest' &&
|
|
||||||
github.event_name == 'push'
|
|
||||||
continue-on-error: true
|
|
||||||
|
|
||||||
golangci:
|
|
||||||
name: lint
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- uses: actions/setup-go@v6
|
|
||||||
with:
|
|
||||||
go-version: 1.24
|
|
||||||
- uses: actions/checkout@v5
|
|
||||||
- name: golangci-lint
|
|
||||||
uses: golangci/golangci-lint-action@v8
|
|
||||||
34
.github/workflows/pushimage.yaml
vendored
34
.github/workflows/pushimage.yaml
vendored
@@ -1,34 +0,0 @@
|
|||||||
name: build-push-image
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
tags:
|
|
||||||
- 'v*'
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
build-and-push-image:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
permissions:
|
|
||||||
packages: write
|
|
||||||
steps:
|
|
||||||
- name: Checkout repository
|
|
||||||
uses: actions/checkout@v5
|
|
||||||
|
|
||||||
- name: Log in to the Container registry
|
|
||||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef
|
|
||||||
with:
|
|
||||||
registry: https://ghcr.io
|
|
||||||
username: ${{ github.actor }}
|
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
|
|
||||||
- name: Build and push Docker image
|
|
||||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83
|
|
||||||
with:
|
|
||||||
push: true
|
|
||||||
tags: ghcr.io/tlinden/kleingebaeck:${{ github.ref_name}}
|
|
||||||
|
|
||||||
- name: Build and push latest Docker image
|
|
||||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83
|
|
||||||
with:
|
|
||||||
push: true
|
|
||||||
tags: ghcr.io/tlinden/kleingebaeck:latest
|
|
||||||
87
.github/workflows/release.yaml
vendored
87
.github/workflows/release.yaml
vendored
@@ -1,87 +0,0 @@
|
|||||||
name: build-release
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
tags:
|
|
||||||
- "v*.*.*"
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
release:
|
|
||||||
name: Build Release Assets
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: Checkout code
|
|
||||||
uses: actions/checkout@v5
|
|
||||||
|
|
||||||
- name: Set up Go
|
|
||||||
uses: actions/setup-go@v6
|
|
||||||
with:
|
|
||||||
go-version: 1.24.5
|
|
||||||
|
|
||||||
- name: Build the executables
|
|
||||||
run: ./mkrel.sh kleingebaeck ${{ github.ref_name}}
|
|
||||||
|
|
||||||
- name: List the executables
|
|
||||||
run: ls -l ./releases
|
|
||||||
|
|
||||||
- name: Upload the binaries
|
|
||||||
uses: svenstaro/upload-release-action@v2
|
|
||||||
with:
|
|
||||||
repo_token: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
tag: ${{ github.ref_name }}
|
|
||||||
file: ./releases/*
|
|
||||||
file_glob: true
|
|
||||||
|
|
||||||
- name: Build Changelog
|
|
||||||
id: github_release
|
|
||||||
uses: mikepenz/release-changelog-builder-action@v5
|
|
||||||
env:
|
|
||||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
with:
|
|
||||||
mode: "PR"
|
|
||||||
configurationJson: |
|
|
||||||
{
|
|
||||||
"template": "#{{CHANGELOG}}\n\n**Full Changelog**: #{{RELEASE_DIFF}}",
|
|
||||||
"pr_template": "- #{{TITLE}} (##{{NUMBER}}) by #{{AUTHOR}}\n#{{BODY}}",
|
|
||||||
"empty_template": "- no changes",
|
|
||||||
"categories": [
|
|
||||||
{
|
|
||||||
"title": "## New Features",
|
|
||||||
"labels": ["add", "feature"]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"title": "## Bug Fixes",
|
|
||||||
"labels": ["fix", "bug", "revert"]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"title": "## Documentation Enhancements",
|
|
||||||
"labels": ["doc"]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"title": "## Refactoring Efforts",
|
|
||||||
"labels": ["refactor"]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"title": "## Miscellaneus Changes",
|
|
||||||
"labels": []
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"ignore_labels": [
|
|
||||||
"duplicate", "good first issue", "help wanted", "invalid", "question", "wontfix"
|
|
||||||
],
|
|
||||||
"label_extractor": [
|
|
||||||
{
|
|
||||||
"pattern": "(.) (.+)",
|
|
||||||
"target": "$1"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"pattern": "(.) (.+)",
|
|
||||||
"target": "$1",
|
|
||||||
"on_property": "title"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
||||||
- name: Create Release
|
|
||||||
uses: softprops/action-gh-release@v2
|
|
||||||
with:
|
|
||||||
body: ${{steps.github_release.outputs.changelog}}
|
|
||||||
106
.golangci.yml
Normal file
106
.golangci.yml
Normal file
@@ -0,0 +1,106 @@
|
|||||||
|
version: "2"
|
||||||
|
|
||||||
|
linters:
|
||||||
|
enable:
|
||||||
|
- errcheck
|
||||||
|
- govet
|
||||||
|
- ineffassign
|
||||||
|
- staticcheck
|
||||||
|
- unused
|
||||||
|
- arangolint
|
||||||
|
- asasalint
|
||||||
|
- asciicheck
|
||||||
|
- bidichk
|
||||||
|
- bodyclose
|
||||||
|
- canonicalheader
|
||||||
|
- clickhouselint
|
||||||
|
- containedctx
|
||||||
|
#- contextcheck
|
||||||
|
- copyloopvar
|
||||||
|
#- cyclop
|
||||||
|
- decorder
|
||||||
|
- dogsled
|
||||||
|
- dupword
|
||||||
|
- durationcheck
|
||||||
|
- embeddedstructfieldcheck
|
||||||
|
- errchkjson
|
||||||
|
- errname
|
||||||
|
#- errorlint
|
||||||
|
- exhaustive
|
||||||
|
- exptostd
|
||||||
|
- fatcontext
|
||||||
|
- funcorder
|
||||||
|
#- funlen
|
||||||
|
- ginkgolinter
|
||||||
|
- gocheckcompilerdirectives
|
||||||
|
#- gochecknoinits
|
||||||
|
- gochecksumtype
|
||||||
|
#- gocognit
|
||||||
|
- gocritic
|
||||||
|
#- gocyclo
|
||||||
|
- goheader
|
||||||
|
- gomoddirectives
|
||||||
|
- gomodguard_v2
|
||||||
|
- goprintffuncname
|
||||||
|
#- gosec
|
||||||
|
- gosmopolitan
|
||||||
|
- grouper
|
||||||
|
- iface
|
||||||
|
- importas
|
||||||
|
- inamedparam
|
||||||
|
- interfacebloat
|
||||||
|
- intrange
|
||||||
|
- iotamixing
|
||||||
|
- lll
|
||||||
|
- loggercheck
|
||||||
|
- makezero
|
||||||
|
- misspell
|
||||||
|
- modernize
|
||||||
|
- nakedret
|
||||||
|
#- nestif
|
||||||
|
- nilerr
|
||||||
|
- nilnesserr
|
||||||
|
- nlreturn
|
||||||
|
- nonamedreturns
|
||||||
|
- nosprintfhostport
|
||||||
|
- perfsprint
|
||||||
|
- prealloc
|
||||||
|
- promlinter
|
||||||
|
- protogetter
|
||||||
|
- reassign
|
||||||
|
- recvcheck
|
||||||
|
- rowserrcheck
|
||||||
|
- sloglint
|
||||||
|
- spancheck
|
||||||
|
- sqlclosecheck
|
||||||
|
- tagalign
|
||||||
|
- testableexamples
|
||||||
|
- testifylint
|
||||||
|
- testpackage
|
||||||
|
- thelper
|
||||||
|
- tparallel
|
||||||
|
- unconvert
|
||||||
|
- unparam
|
||||||
|
- unqueryvet
|
||||||
|
- usestdlibvars
|
||||||
|
- usetesting
|
||||||
|
- varnamelen
|
||||||
|
- wastedassign
|
||||||
|
- whitespace
|
||||||
|
- wsl_v5
|
||||||
|
- zerologlint
|
||||||
|
|
||||||
|
settings:
|
||||||
|
varnamelen:
|
||||||
|
ignore-names:
|
||||||
|
- err
|
||||||
|
- wg
|
||||||
|
- mu
|
||||||
|
- ts
|
||||||
|
- to
|
||||||
|
- es
|
||||||
|
- op
|
||||||
|
- fd
|
||||||
|
- id
|
||||||
|
- fn
|
||||||
|
|
||||||
69
.goreleaser.yaml
Normal file
69
.goreleaser.yaml
Normal file
@@ -0,0 +1,69 @@
|
|||||||
|
# vim: set ts=2 sw=2 tw=0 fo=cnqoj
|
||||||
|
|
||||||
|
version: 2
|
||||||
|
|
||||||
|
before:
|
||||||
|
hooks:
|
||||||
|
- go mod tidy
|
||||||
|
|
||||||
|
gitea_urls:
|
||||||
|
api: https://codeberg.org/api/v1
|
||||||
|
download: https://codeberg.org
|
||||||
|
|
||||||
|
builds:
|
||||||
|
- env:
|
||||||
|
- CGO_ENABLED=0
|
||||||
|
goos:
|
||||||
|
- linux
|
||||||
|
- windows
|
||||||
|
- darwin
|
||||||
|
- freebsd
|
||||||
|
|
||||||
|
archives:
|
||||||
|
- formats: [tar.gz]
|
||||||
|
# this name template makes the OS and Arch compatible with the results of `uname`.
|
||||||
|
name_template: >-
|
||||||
|
{{ .ProjectName }}_
|
||||||
|
{{- title .Os }}_
|
||||||
|
{{- if eq .Arch "amd64" }}x86_64
|
||||||
|
{{- else if eq .Arch "386" }}i386
|
||||||
|
{{- else }}{{ .Arch }}{{ end }}
|
||||||
|
{{- if .Arm }}v{{ .Arm }}{{ end }}_{{ .Tag }}
|
||||||
|
# use zip for windows archives
|
||||||
|
format_overrides:
|
||||||
|
- goos: windows
|
||||||
|
formats: [zip]
|
||||||
|
- goos: linux
|
||||||
|
formats: [tar.gz,binary]
|
||||||
|
files:
|
||||||
|
- src: "*.md"
|
||||||
|
strip_parent: true
|
||||||
|
- src: "docs/*"
|
||||||
|
strip_parent: true
|
||||||
|
- src: Makefile.dist
|
||||||
|
dst: Makefile
|
||||||
|
wrap_in_directory: true
|
||||||
|
|
||||||
|
changelog:
|
||||||
|
sort: asc
|
||||||
|
filters:
|
||||||
|
exclude:
|
||||||
|
- "^docs:"
|
||||||
|
- "^test:"
|
||||||
|
groups:
|
||||||
|
- title: Improved
|
||||||
|
regexp: '^.*?(feat|add|new)(\([[:word:]]+\))??!?:.+$'
|
||||||
|
order: 0
|
||||||
|
- title: Fixed
|
||||||
|
regexp: '^.*?(bug|fix)(\([[:word:]]+\))??!?:.+$'
|
||||||
|
order: 1
|
||||||
|
- title: Changed
|
||||||
|
order: 999
|
||||||
|
|
||||||
|
release:
|
||||||
|
header: "# Release Notes"
|
||||||
|
footer: >-
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
Full Changelog: [{{ .PreviousTag }}...{{ .Tag }}](https://codeberg.org/scip/kleingebaeck/compare/{{ .PreviousTag }}...{{ .Tag }})
|
||||||
38
.woodpecker/build.yaml
Normal file
38
.woodpecker/build.yaml
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
matrix:
|
||||||
|
platform:
|
||||||
|
- linux/amd64
|
||||||
|
goversion:
|
||||||
|
- 1.26
|
||||||
|
lintversion:
|
||||||
|
- v2.12.2
|
||||||
|
|
||||||
|
labels:
|
||||||
|
platform: ${platform}
|
||||||
|
|
||||||
|
steps:
|
||||||
|
build:
|
||||||
|
when:
|
||||||
|
event: [push]
|
||||||
|
image: golang:${goversion}
|
||||||
|
commands:
|
||||||
|
- go get
|
||||||
|
- go build
|
||||||
|
|
||||||
|
linter:
|
||||||
|
when:
|
||||||
|
event: [push]
|
||||||
|
image: golang:${goversion}
|
||||||
|
commands:
|
||||||
|
- curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin ${lintversion}
|
||||||
|
- golangci-lint --version
|
||||||
|
- golangci-lint run ./...
|
||||||
|
depends_on: [build]
|
||||||
|
|
||||||
|
test:
|
||||||
|
when:
|
||||||
|
event: [push]
|
||||||
|
image: golang:${goversion}
|
||||||
|
commands:
|
||||||
|
- go get
|
||||||
|
- go test -v -cover
|
||||||
|
depends_on: [build,linter]
|
||||||
32
.woodpecker/image.yaml
Normal file
32
.woodpecker/image.yaml
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
# https://woodpecker-ci.org/plugins/docker-buildx
|
||||||
|
# enable Package unit and go to /scip/-/packages after building to link to proj
|
||||||
|
|
||||||
|
variables:
|
||||||
|
- &repo codeberg.org/${CI_REPO_OWNER}/kleingebaeck
|
||||||
|
|
||||||
|
steps:
|
||||||
|
dryrun:
|
||||||
|
image: docker.io/woodpeckerci/plugin-docker-buildx:6.0.3
|
||||||
|
settings:
|
||||||
|
dockerfile: Dockerfile
|
||||||
|
platforms: linux/amd64
|
||||||
|
dry_run: true
|
||||||
|
repo: *repo
|
||||||
|
tags: latest
|
||||||
|
when:
|
||||||
|
event: [pull_request]
|
||||||
|
|
||||||
|
publish:
|
||||||
|
image: docker.io/woodpeckerci/plugin-docker-buildx:6.0.3
|
||||||
|
settings:
|
||||||
|
dockerfile: Dockerfile
|
||||||
|
platforms: linux/amd64
|
||||||
|
repo: *repo
|
||||||
|
registry: codeberg.org
|
||||||
|
tags: latest,${CI_COMMIT_SHA:0:8},${CI_COMMIT_TAG}
|
||||||
|
username: ${CI_REPO_OWNER}
|
||||||
|
password:
|
||||||
|
from_secret: REGISTRY_TOKEN
|
||||||
|
when:
|
||||||
|
event: [tag]
|
||||||
|
branch: main
|
||||||
15
.woodpecker/release.yaml
Normal file
15
.woodpecker/release.yaml
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
# build release
|
||||||
|
|
||||||
|
labels:
|
||||||
|
platform: linux/amd64
|
||||||
|
|
||||||
|
steps:
|
||||||
|
goreleaser:
|
||||||
|
image: goreleaser/goreleaser
|
||||||
|
when:
|
||||||
|
event: [tag]
|
||||||
|
environment:
|
||||||
|
GITEA_TOKEN:
|
||||||
|
from_secret: DEPLOY_TOKEN
|
||||||
|
commands:
|
||||||
|
- goreleaser release --clean --verbose
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
FROM golang:1.24-alpine as builder
|
FROM golang:1.26-alpine as builder
|
||||||
|
|
||||||
RUN apk update
|
RUN apk update
|
||||||
RUN apk upgrade
|
RUN apk upgrade
|
||||||
|
|||||||
20
Makefile.dist
Normal file
20
Makefile.dist
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
# -*-make-*-
|
||||||
|
|
||||||
|
.PHONY: install all
|
||||||
|
|
||||||
|
tool = rpn
|
||||||
|
PREFIX = /usr/local
|
||||||
|
UID = root
|
||||||
|
GID = 0
|
||||||
|
|
||||||
|
all:
|
||||||
|
@echo "Type 'sudo make install' to install the tool."
|
||||||
|
@echo "To change prefix, type 'sudo make install PREFIX=/opt'"
|
||||||
|
|
||||||
|
install:
|
||||||
|
install -d -o $(UID) -g $(GID) $(PREFIX)/bin
|
||||||
|
install -d -o $(UID) -g $(GID) $(PREFIX)/man/man1
|
||||||
|
install -d -o $(UID) -g $(GID) $(PREFIX)/share/doc
|
||||||
|
install -o $(UID) -g $(GID) -m 555 $(tool) $(PREFIX)/sbin/
|
||||||
|
install -o $(UID) -g $(GID) -m 444 $(tool).1 $(PREFIX)/man/man1/
|
||||||
|
install -o $(UID) -g $(GID) -m 444 *.md $(PREFIX)/share/doc/
|
||||||
37
README-de.md
37
README-de.md
@@ -1,13 +1,12 @@
|
|||||||
## Kleingebäck - kleinanzeigen.de Backup
|
## Kleingebäck - kleinanzeigen.de Backup
|
||||||
|
|
||||||

|

|
||||||
|
|
||||||
[](https://goreportcard.com/report/github.com/tlinden/kleingebaeck)
|
[](https://goreportcard.com/report/codeberg.org/scip/kleingebaeck)
|
||||||
[](https://github.com/tlinden/kleingebaeck/actions)
|
[](https://ci.codeberg.org/repos/15530)
|
||||||
[](https://raw.githack.com/wiki/tlinden/kleingebaeck/coverage.html)
|
|
||||||

|

|
||||||
[](https://github.com/TLINDEN/kleingebaeck/releases/latest)
|
[](https://codeberg.org/scip/kleingebaeck/releases)
|
||||||
[](https://github.com/tlinden/kleingebaeck/blob/main/README.md)
|
[](https://codeberg.org/scip/kleingebaeck/raw/branch/main/README.md)
|
||||||
|
|
||||||
Mit diesem Tool kann man seine Anzeigen bei https://kleinanzeigen.de sichern.
|
Mit diesem Tool kann man seine Anzeigen bei https://kleinanzeigen.de sichern.
|
||||||
|
|
||||||
@@ -23,40 +22,40 @@ Fertige vorcompilierte Programme älter als Version `v0.3.12` sind von
|
|||||||
Schwachstellen in der Behandlung von HTTP und Zertifikaten
|
Schwachstellen in der Behandlung von HTTP und Zertifikaten
|
||||||
betroffen. Falls Du eine ältere Kleingebäck-Version im Einsatz hast,
|
betroffen. Falls Du eine ältere Kleingebäck-Version im Einsatz hast,
|
||||||
bitte update auf Version `v0.3.12` oder höher. Bitte lies auch die [Release Notes für
|
bitte update auf Version `v0.3.12` oder höher. Bitte lies auch die [Release Notes für
|
||||||
v0.3.12](https://github.com/TLINDEN/kleingebaeck/releases/tag/v0.3.12)
|
v0.3.12](https://codeberg.org/scip/kleingebaeck/releases/tag/v0.3.12)
|
||||||
für mehr Details.
|
für mehr Details.
|
||||||
|
|
||||||
## Screenshots
|
## Screenshots
|
||||||
|
|
||||||
Das ist die Hauptseite meines kleinanzeigen.de Accounts:
|
Das ist die Hauptseite meines kleinanzeigen.de Accounts:
|
||||||
|
|
||||||

|

|
||||||
|
|
||||||
Sichern ich meine Anzeigen:
|
Sichern ich meine Anzeigen:
|
||||||
|
|
||||||

|

|
||||||
|
|
||||||
Backupverzeichnis nach dem Download:
|
Backupverzeichnis nach dem Download:
|
||||||
|
|
||||||

|

|
||||||
|
|
||||||
Verzeichnis einer Anzeige:
|
Verzeichnis einer Anzeige:
|
||||||
|
|
||||||

|

|
||||||
|
|
||||||
**Das gleiche unter Windows:**
|
**Das gleiche unter Windows:**
|
||||||
|
|
||||||
Anzeigen Sichern:
|
Anzeigen Sichern:
|
||||||
|
|
||||||

|

|
||||||
|
|
||||||
Backupverzeichnis nach dem Download
|
Backupverzeichnis nach dem Download
|
||||||
|
|
||||||

|

|
||||||
|
|
||||||
Und eine Anzeige:
|
Und eine Anzeige:
|
||||||
|
|
||||||

|

|
||||||
|
|
||||||
## Installation
|
## Installation
|
||||||
|
|
||||||
@@ -66,7 +65,7 @@ für seine Plattform herunter und kann direkt loslegen.
|
|||||||
|
|
||||||
### Installation des vorcompilierten Programms
|
### Installation des vorcompilierten Programms
|
||||||
|
|
||||||
Auf der Seite [des letzten Releases](https://github.com/TLINDEN/kleingebaeck/releases/latest) findet man das Program für sein Betriebssystem und die Plattform (z.b. Windows + Intel)
|
Auf der Seite [des letzten Releases](https://codeberg.org/scip/kleingebaeck/releases) findet man das Program für sein Betriebssystem und die Plattform (z.b. Windows + Intel)
|
||||||
|
|
||||||
Es gibt 2 Varianten:
|
Es gibt 2 Varianten:
|
||||||
|
|
||||||
@@ -105,7 +104,7 @@ hilfreich, aber nicht unbedingt erforderlich.
|
|||||||
Um das Programm zu compilieren, muss man folgende Schritte ausführen:
|
Um das Programm zu compilieren, muss man folgende Schritte ausführen:
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
git clone https://github.com/TLINDEN/kleingebaeck.git
|
git clone https://codeberg.org/scip/kleingebaeck.git
|
||||||
cd kleingebaeck
|
cd kleingebaeck
|
||||||
go mod tidy
|
go mod tidy
|
||||||
make # (oder make)
|
make # (oder make)
|
||||||
@@ -146,7 +145,7 @@ gemountet.
|
|||||||
Die Optionen `-u XXX -v` sind kleingebäck Optionen. Ersetze `XXX`
|
Die Optionen `-u XXX -v` sind kleingebäck Optionen. Ersetze `XXX`
|
||||||
durch Deine tatsächliche kleinanzeigen.de Userid.
|
durch Deine tatsächliche kleinanzeigen.de Userid.
|
||||||
|
|
||||||
Eine Liste verfügbarer Images findet man [hier](https://github.com/tlinden/kleingebaeck/pkgs/container/kleingebaeck/versions?filters%5Bversion_type%5D=tagged)
|
Eine Liste verfügbarer Images findet man [hier](https://codeberg.org/scip/kleingebaeck/pkgs/container/kleingebaeck/versions?filters%5Bversion_type%5D=tagged)
|
||||||
|
|
||||||
## Kommandozeilen Optionen:
|
## Kommandozeilen Optionen:
|
||||||
|
|
||||||
@@ -277,7 +276,7 @@ ignorieren, die beim Bilderdownload auftreten könnten.
|
|||||||
## Documentation
|
## Documentation
|
||||||
|
|
||||||
Die Dokumentation kann man
|
Die Dokumentation kann man
|
||||||
[online](https://github.com/TLINDEN/kleingebaeck/blob/main/kleingebaeck.pod)
|
[online](https://codeberg.org/scip/kleingebaeck/raw/branch/main/kleingebaeck.pod)
|
||||||
oder lokal lesen mit: `kleingebaeck --manual`. Hat man das Tool mit
|
oder lokal lesen mit: `kleingebaeck --manual`. Hat man das Tool mit
|
||||||
dem Tarball installiert, funktioniert auch `man kleingebaeck`.
|
dem Tarball installiert, funktioniert auch `man kleingebaeck`.
|
||||||
|
|
||||||
@@ -293,7 +292,7 @@ das doch der beste Weg, die Anfrage zu übersehen und zu vergessen.
|
|||||||
|
|
||||||
Um einen Fehler, ein unerwartetes Verhalten, eine Feature Request oder
|
Um einen Fehler, ein unerwartetes Verhalten, eine Feature Request oder
|
||||||
einen Patch zu übermitteln, eröffne daher bitte einen Issue unter:
|
einen Patch zu übermitteln, eröffne daher bitte einen Issue unter:
|
||||||
https://github.com/TLINDEN/kleingebaeck/issues. Danke!
|
https://codeberg.org/scip/kleingebaeck/issues. Danke!
|
||||||
|
|
||||||
Bitte gebe den fehlgeschlagenen Befehl an, rufe es auch mit Debugging
|
Bitte gebe den fehlgeschlagenen Befehl an, rufe es auch mit Debugging
|
||||||
`-d` auf.
|
`-d` auf.
|
||||||
|
|||||||
35
README.md
35
README.md
@@ -1,13 +1,12 @@
|
|||||||
## Kleingebäck - kleinanzeigen.de Backup
|
## Kleingebäck - kleinanzeigen.de Backup
|
||||||
|
|
||||||

|

|
||||||
|
|
||||||
[](https://goreportcard.com/report/github.com/tlinden/kleingebaeck)
|
[](https://goreportcard.com/report/codeberg.org/scip/kleingebaeck)
|
||||||
[](https://github.com/tlinden/kleingebaeck/actions)
|
[](https://ci.codeberg.org/repos/15530)
|
||||||
[](https://raw.githack.com/wiki/tlinden/kleingebaeck/coverage.html)
|
|
||||||

|

|
||||||
[](https://github.com/TLINDEN/kleingebaeck/releases/latest)
|
[](https://codeberg.org/scip/kleingebaeck/releases)
|
||||||
[](https://github.com/tlinden/kleingebaeck/blob/main/README-de.md)
|
[](https://codeberg.org/scip/kleingebaeck/raw/branch/main/README-de.md)
|
||||||
|
|
||||||
[Die deutsche Version des READMEs findet Ihr hier](README-de.md).
|
[Die deutsche Version des READMEs findet Ihr hier](README-de.md).
|
||||||
|
|
||||||
@@ -24,40 +23,40 @@ Binary releases prior to version `v0.3.11` are affected by
|
|||||||
vulnerabilities in HTTP and certificate handling. If you are using
|
vulnerabilities in HTTP and certificate handling. If you are using
|
||||||
such a binary, please update to `v0.3.12` or higher. Please also refer
|
such a binary, please update to `v0.3.12` or higher. Please also refer
|
||||||
to the [Release Notes of
|
to the [Release Notes of
|
||||||
v0.3.12](https://github.com/TLINDEN/kleingebaeck/releases/tag/v0.3.12)
|
v0.3.12](https://codeberg.org/scip/kleingebaeck/releases/tag/v0.3.12)
|
||||||
for more details.
|
for more details.
|
||||||
|
|
||||||
## Screenshots
|
## Screenshots
|
||||||
|
|
||||||
This is the index of my kleinanzeigen.de Account:
|
This is the index of my kleinanzeigen.de Account:
|
||||||
|
|
||||||

|

|
||||||
|
|
||||||
Here I download my ads on the commandline:
|
Here I download my ads on the commandline:
|
||||||
|
|
||||||

|

|
||||||
|
|
||||||
And this is the backup directory after download:
|
And this is the backup directory after download:
|
||||||
|
|
||||||

|

|
||||||
|
|
||||||
Here's a directory for one ad:
|
Here's a directory for one ad:
|
||||||
|
|
||||||

|

|
||||||
|
|
||||||
**The same thing under windows:**
|
**The same thing under windows:**
|
||||||
|
|
||||||
Downloading ads:
|
Downloading ads:
|
||||||
|
|
||||||

|

|
||||||
|
|
||||||
Backup directory after download:
|
Backup directory after download:
|
||||||
|
|
||||||

|

|
||||||
|
|
||||||
And one ad listing directory:
|
And one ad listing directory:
|
||||||
|
|
||||||

|

|
||||||
|
|
||||||
## Installation
|
## Installation
|
||||||
|
|
||||||
@@ -68,7 +67,7 @@ releases page and you're good to go.
|
|||||||
### Installation using a pre-compiled binary
|
### Installation using a pre-compiled binary
|
||||||
|
|
||||||
Go to the [latest release
|
Go to the [latest release
|
||||||
page](https://github.com/TLINDEN/kleingebaeck/releases/latest) and
|
page](https://codeberg.org/scip/kleingebaeck/releases) and
|
||||||
look for your OS and platform. There are two options to install the binary:
|
look for your OS and platform. There are two options to install the binary:
|
||||||
|
|
||||||
1. Directly download the binary for your platform,
|
1. Directly download the binary for your platform,
|
||||||
@@ -138,7 +137,7 @@ docker run. And the local directory `myads` will be mapped to
|
|||||||
The options `-u XXX -v` are kleingebaeck options, replace `XXX` with
|
The options `-u XXX -v` are kleingebaeck options, replace `XXX` with
|
||||||
your actual kleinanzeigen.de user id.
|
your actual kleinanzeigen.de user id.
|
||||||
|
|
||||||
A list of available images is [here](https://github.com/tlinden/kleingebaeck/pkgs/container/kleingebaeck/versions?filters%5Bversion_type%5D=tagged)
|
A list of available images is [here](https://codeberg.org/scip/kleingebaeck/pkgs/container/kleingebaeck/versions?filters%5Bversion_type%5D=tagged)
|
||||||
|
|
||||||
## Commandline options:
|
## Commandline options:
|
||||||
|
|
||||||
@@ -262,7 +261,7 @@ errors.
|
|||||||
|
|
||||||
## Documentation
|
## Documentation
|
||||||
|
|
||||||
You can read the documentation [online](https://github.com/TLINDEN/kleingebaeck/blob/main/kleingebaeck.pod) or locally once you have installed kleingebaeck with: `kleingebaeck --manual`.
|
You can read the documentation [online](https://codeberg.org/scip/kleingebaeck/raw/branch/main/kleingebaeck.pod) or locally once you have installed kleingebaeck with: `kleingebaeck --manual`.
|
||||||
|
|
||||||
## Kleingebäck?
|
## Kleingebäck?
|
||||||
|
|
||||||
@@ -278,7 +277,7 @@ that's the best way for me to forget to do something.
|
|||||||
|
|
||||||
In order to report a bug, unexpected behavior, feature requests or to
|
In order to report a bug, unexpected behavior, feature requests or to
|
||||||
submit a patch, please open an issue on github:
|
submit a patch, please open an issue on github:
|
||||||
https://github.com/TLINDEN/kleingebaeck/issues.
|
https://codeberg.org/scip/kleingebaeck/issues.
|
||||||
|
|
||||||
Please repeat the failing command with debugging enabled `-d` and
|
Please repeat the failing command with debugging enabled `-d` and
|
||||||
include the output in the issue.
|
include the output in the issue.
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ I don't agree with the "responsible disclosure" process most projects
|
|||||||
(and companies) work these days.
|
(and companies) work these days.
|
||||||
|
|
||||||
So, if you find a vulnerability of any kind, please just open an
|
So, if you find a vulnerability of any kind, please just open an
|
||||||
[issue](https://github.com/TLINDEN/kleingebaeck/issues). Please add
|
[issue](https://codeberg.org/scip/kleingebaeck/issues). Please add
|
||||||
all details required to reproduce the vulnerability. You won't be chased.
|
all details required to reproduce the vulnerability. You won't be chased.
|
||||||
|
|
||||||
That's just all about it.
|
That's just all about it.
|
||||||
|
|||||||
22
ad.go
22
ad.go
@@ -29,16 +29,8 @@ type Index struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type Ad struct {
|
type Ad struct {
|
||||||
Title string `goquery:"h1"`
|
Title string `goquery:"h1"`
|
||||||
Slug string
|
Details string `goquery:".addetailslist--detail,text"`
|
||||||
ID string
|
|
||||||
Details string `goquery:".addetailslist--detail,text"`
|
|
||||||
Attributes map[string]string // processed afterwards
|
|
||||||
Condition string // post processed from details for backward compatibility
|
|
||||||
Type string // post processed from details for backward compatibility
|
|
||||||
Color string // post processed from details for backward compatibility
|
|
||||||
Material string // post processed from details for backward compatibility
|
|
||||||
Category string
|
|
||||||
CategoryTree []string `goquery:".breadcrump-link,text"`
|
CategoryTree []string `goquery:".breadcrump-link,text"`
|
||||||
Price string `goquery:"h2#viewad-price"`
|
Price string `goquery:"h2#viewad-price"`
|
||||||
Created string `goquery:"#viewad-extra-info,text"`
|
Created string `goquery:"#viewad-extra-info,text"`
|
||||||
@@ -46,6 +38,14 @@ type Ad struct {
|
|||||||
Images []string `goquery:".galleryimage-element img,[src]"`
|
Images []string `goquery:".galleryimage-element img,[src]"`
|
||||||
Shipping string `goquery:".boxedarticle--details--shipping,text"` // not always filled
|
Shipping string `goquery:".boxedarticle--details--shipping,text"` // not always filled
|
||||||
Expire string
|
Expire string
|
||||||
|
Slug string
|
||||||
|
ID string
|
||||||
|
Attributes map[string]string // processed afterwards
|
||||||
|
Condition string // post processed from details for backward compatibility
|
||||||
|
Type string // post processed from details for backward compatibility
|
||||||
|
Color string // post processed from details for backward compatibility
|
||||||
|
Material string // post processed from details for backward compatibility
|
||||||
|
Category string
|
||||||
|
|
||||||
// runtime computed
|
// runtime computed
|
||||||
Year, Day, Month string
|
Year, Day, Month string
|
||||||
@@ -94,7 +94,7 @@ func (ad *Ad) CalculateExpire() {
|
|||||||
|
|
||||||
/*
|
/*
|
||||||
Decode attributes like color or condition. See
|
Decode attributes like color or condition. See
|
||||||
https://github.com/TLINDEN/kleingebaeck/issues/117
|
https://codeberg.org/scip/kleingebaeck/issues/117
|
||||||
for more details. In short: the HTML delivered by
|
for more details. In short: the HTML delivered by
|
||||||
kleinanzeigen.de has no css attribute for the keys
|
kleinanzeigen.de has no css attribute for the keys
|
||||||
so we cannot extract key=>value mappings of the
|
so we cannot extract key=>value mappings of the
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
VERSION string = "0.3.23"
|
VERSION string = "0.3.25"
|
||||||
Baseuri string = "https://www.kleinanzeigen.de"
|
Baseuri string = "https://www.kleinanzeigen.de"
|
||||||
Listuri string = "/s-bestandsliste.html"
|
Listuri string = "/s-bestandsliste.html"
|
||||||
Defaultdir string = "."
|
Defaultdir string = "."
|
||||||
@@ -140,7 +140,7 @@ func InitConfig(output io.Writer) (*Config, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Load default values using the confmap provider.
|
// Load default values using the confmap provider.
|
||||||
if err := kloader.Load(confmap.Provider(map[string]interface{}{
|
if err := kloader.Load(confmap.Provider(map[string]any{
|
||||||
"template": template,
|
"template": template,
|
||||||
"outdir": DefaultOutdirTemplate,
|
"outdir": DefaultOutdirTemplate,
|
||||||
"loglevel": "notice",
|
"loglevel": "notice",
|
||||||
@@ -200,7 +200,6 @@ func InitConfig(output io.Writer) (*Config, error) {
|
|||||||
// Load the config file[s]
|
// Load the config file[s]
|
||||||
for _, cfgfile := range configfiles {
|
for _, cfgfile := range configfiles {
|
||||||
path, err := os.Stat(cfgfile)
|
path, err := os.Stat(cfgfile)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// ignore non-existent files, but bail out on any other errors
|
// ignore non-existent files, but bail out on any other errors
|
||||||
if !os.IsNotExist(err) {
|
if !os.IsNotExist(err) {
|
||||||
|
|||||||
51
fetch.go
51
fetch.go
@@ -24,7 +24,11 @@ import (
|
|||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/cookiejar"
|
"net/http/cookiejar"
|
||||||
|
"net/http/httputil"
|
||||||
"net/url"
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
)
|
)
|
||||||
|
|
||||||
// convenient wrapper to fetch some web content
|
// convenient wrapper to fetch some web content
|
||||||
@@ -34,12 +38,25 @@ type Fetcher struct {
|
|||||||
Cookies []*http.Cookie
|
Cookies []*http.Cookie
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Pos is a counter for debug output file names
|
||||||
|
type Pos struct {
|
||||||
|
sync.Mutex
|
||||||
|
|
||||||
|
pos int
|
||||||
|
}
|
||||||
|
|
||||||
|
var pos *Pos
|
||||||
|
|
||||||
func NewFetcher(conf *Config) (*Fetcher, error) {
|
func NewFetcher(conf *Config) (*Fetcher, error) {
|
||||||
jar, err := cookiejar.New(nil)
|
jar, err := cookiejar.New(nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to create a cookie jar obj: %w", err)
|
return nil, fmt.Errorf("failed to create a cookie jar obj: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if pos == nil {
|
||||||
|
pos = &Pos{}
|
||||||
|
}
|
||||||
|
|
||||||
return &Fetcher{
|
return &Fetcher{
|
||||||
Client: &http.Client{
|
Client: &http.Client{
|
||||||
Transport: &loggingTransport{}, // implemented in http.go
|
Transport: &loggingTransport{}, // implemented in http.go
|
||||||
@@ -82,9 +99,43 @@ func (f *Fetcher) Get(uri string) (io.ReadCloser, error) {
|
|||||||
slog.Debug("got cookies?", "cookies", res.Cookies())
|
slog.Debug("got cookies?", "cookies", res.Cookies())
|
||||||
f.Cookies = res.Cookies()
|
f.Cookies = res.Cookies()
|
||||||
|
|
||||||
|
if err := f.Dump(res); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
return res.Body, nil
|
return res.Body, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (f *Fetcher) Dump(res *http.Response) error {
|
||||||
|
if f.Config.Debug && strings.Contains(res.Header.Get("Content-Type"), "text/html") {
|
||||||
|
// => dump content
|
||||||
|
pos.Lock()
|
||||||
|
defer pos.Unlock()
|
||||||
|
|
||||||
|
pos.pos++
|
||||||
|
|
||||||
|
dump, err := httputil.DumpResponse(res, true)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
file := fmt.Sprintf("out%d.html", pos.pos)
|
||||||
|
|
||||||
|
fd, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE, 0600)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := fd.Write(dump); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
slog.Debug("dump content", "file", file)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// fetch an image
|
// fetch an image
|
||||||
func (f *Fetcher) Getimage(uri string) (io.ReadCloser, error) {
|
func (f *Fetcher) Getimage(uri string) (io.ReadCloser, error) {
|
||||||
slog.Debug("fetching ad image", "uri", uri)
|
slog.Debug("fetching ad image", "uri", uri)
|
||||||
|
|||||||
6
go.mod
6
go.mod
@@ -1,8 +1,6 @@
|
|||||||
module kleingebaeck
|
module kleingebaeck
|
||||||
|
|
||||||
go 1.24.0
|
go 1.26.0
|
||||||
|
|
||||||
toolchain go1.24.5
|
|
||||||
|
|
||||||
require (
|
require (
|
||||||
astuart.co/goq v1.0.0
|
astuart.co/goq v1.0.0
|
||||||
@@ -19,7 +17,7 @@ require (
|
|||||||
github.com/mattn/go-isatty v0.0.20
|
github.com/mattn/go-isatty v0.0.20
|
||||||
github.com/spf13/pflag v1.0.10
|
github.com/spf13/pflag v1.0.10
|
||||||
github.com/tlinden/yadu v0.1.3
|
github.com/tlinden/yadu v0.1.3
|
||||||
golang.org/x/image v0.31.0
|
golang.org/x/image v0.32.0
|
||||||
golang.org/x/sync v0.17.0
|
golang.org/x/sync v0.17.0
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
2
go.sum
2
go.sum
@@ -68,6 +68,8 @@ github.com/tlinden/yadu v0.1.3/go.mod h1:l3bRmHKL9zGAR6pnBHY2HRPxBecf7L74BoBgOOp
|
|||||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
golang.org/x/image v0.31.0 h1:mLChjE2MV6g1S7oqbXC0/UcKijjm5fnJLUYKIYrLESA=
|
golang.org/x/image v0.31.0 h1:mLChjE2MV6g1S7oqbXC0/UcKijjm5fnJLUYKIYrLESA=
|
||||||
golang.org/x/image v0.31.0/go.mod h1:R9ec5Lcp96v9FTF+ajwaH3uGxPH4fKfHHAVbUILxghA=
|
golang.org/x/image v0.31.0/go.mod h1:R9ec5Lcp96v9FTF+ajwaH3uGxPH4fKfHHAVbUILxghA=
|
||||||
|
golang.org/x/image v0.32.0 h1:6lZQWq75h7L5IWNk0r+SCpUJ6tUVd3v4ZHnbRKLkUDQ=
|
||||||
|
golang.org/x/image v0.32.0/go.mod h1:/R37rrQmKXtO6tYXAjtDLwQgFLHmhW+V6ayXlxzP2Pc=
|
||||||
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
golang.org/x/net v0.0.0-20190606173856-1492cefac77f/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
golang.org/x/net v0.0.0-20190606173856-1492cefac77f/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||||
|
|||||||
1
http.go
1
http.go
@@ -123,7 +123,6 @@ func (t *loggingTransport) RoundTrip(req *http.Request) (*http.Response, error)
|
|||||||
|
|
||||||
// actual retry
|
// actual retry
|
||||||
resp, err = http.DefaultTransport.RoundTrip(req)
|
resp, err = http.DefaultTransport.RoundTrip(req)
|
||||||
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
slog.Debug("RESPONSE", "id", requestid, "status", resp.StatusCode,
|
slog.Debug("RESPONSE", "id", requestid, "status", resp.StatusCode,
|
||||||
"contentlength", resp.ContentLength, "retry", retries)
|
"contentlength", resp.ContentLength, "retry", retries)
|
||||||
|
|||||||
40
image.go
40
image.go
@@ -19,6 +19,7 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"image"
|
"image"
|
||||||
_ "image/gif"
|
_ "image/gif"
|
||||||
@@ -27,6 +28,7 @@ import (
|
|||||||
"log/slog"
|
"log/slog"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
|
||||||
_ "golang.org/x/image/webp"
|
_ "golang.org/x/image/webp"
|
||||||
|
|
||||||
@@ -43,18 +45,6 @@ type Image struct {
|
|||||||
Mime string
|
Mime string
|
||||||
}
|
}
|
||||||
|
|
||||||
// used for logging to avoid printing Data
|
|
||||||
func (img *Image) LogValue() slog.Value {
|
|
||||||
return slog.GroupValue(
|
|
||||||
slog.String("filename", img.Filename),
|
|
||||||
slog.String("uri", img.URI),
|
|
||||||
slog.String("hash", img.Hash.ToString()),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// holds all images of an ad
|
|
||||||
type Cache []*goimagehash.ImageHash
|
|
||||||
|
|
||||||
// filename comes from the scraper, it contains directory/base w/o suffix
|
// filename comes from the scraper, it contains directory/base w/o suffix
|
||||||
func NewImage(buf *bytes.Reader, filename, uri string) (*Image, error) {
|
func NewImage(buf *bytes.Reader, filename, uri string) (*Image, error) {
|
||||||
_, imgconfig, err := image.DecodeConfig(buf)
|
_, imgconfig, err := image.DecodeConfig(buf)
|
||||||
@@ -75,7 +65,7 @@ func NewImage(buf *bytes.Reader, filename, uri string) (*Image, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if imgconfig == "" {
|
if imgconfig == "" {
|
||||||
return nil, fmt.Errorf("failed to process image: unknown or unsupported image format (supported: jpg,png,gif,webp)")
|
return nil, errors.New("failed to process image: unknown or unsupported image format (supported: jpg,png,gif,webp)")
|
||||||
}
|
}
|
||||||
|
|
||||||
filename += "." + imgconfig
|
filename += "." + imgconfig
|
||||||
@@ -92,6 +82,18 @@ func NewImage(buf *bytes.Reader, filename, uri string) (*Image, error) {
|
|||||||
return img, nil
|
return img, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// used for logging to avoid printing Data
|
||||||
|
func (img *Image) LogValue() slog.Value {
|
||||||
|
return slog.GroupValue(
|
||||||
|
slog.String("filename", img.Filename),
|
||||||
|
slog.String("uri", img.URI),
|
||||||
|
slog.String("hash", img.Hash.ToString()),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// holds all images of an ad
|
||||||
|
type Cache []*goimagehash.ImageHash
|
||||||
|
|
||||||
// Calculate diff hash of the image
|
// Calculate diff hash of the image
|
||||||
func (img *Image) CalcHash() error {
|
func (img *Image) CalcHash() error {
|
||||||
jpgdata, format, err := image.Decode(img.Data)
|
jpgdata, format, err := image.Decode(img.Data)
|
||||||
@@ -100,7 +102,7 @@ func (img *Image) CalcHash() error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if format == "" {
|
if format == "" {
|
||||||
return fmt.Errorf("failed to decode image: unknown or unsupported image format (supported: jpg,png,gif,webp)")
|
return errors.New("failed to decode image: unknown or unsupported image format (supported: jpg,png,gif,webp)")
|
||||||
}
|
}
|
||||||
|
|
||||||
hash1, err := goimagehash.DifferenceHash(jpgdata)
|
hash1, err := goimagehash.DifferenceHash(jpgdata)
|
||||||
@@ -134,13 +136,9 @@ func (img *Image) Similar(hash *goimagehash.ImageHash) bool {
|
|||||||
|
|
||||||
// check current image against all known hashes.
|
// check current image against all known hashes.
|
||||||
func (img *Image) SimilarExists(cache Cache) bool {
|
func (img *Image) SimilarExists(cache Cache) bool {
|
||||||
for _, otherimg := range cache {
|
return slices.ContainsFunc(cache, func(otherimg *goimagehash.ImageHash) bool {
|
||||||
if img.Similar(otherimg) {
|
return img.Similar(otherimg)
|
||||||
return true
|
})
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return false
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// read all JPG images in a ad directory, compute diff hashes and
|
// read all JPG images in a ad directory, compute diff hashes and
|
||||||
|
|||||||
@@ -353,7 +353,7 @@ commandline flags take precedence over env!
|
|||||||
.IX Header "BUGS"
|
.IX Header "BUGS"
|
||||||
In order to report a bug, unexpected behavior, feature requests
|
In order to report a bug, unexpected behavior, feature requests
|
||||||
or to submit a patch, please open an issue on github:
|
or to submit a patch, please open an issue on github:
|
||||||
<https://github.com/TLINDEN/kleingebaeck/issues>.
|
<https://codeberg.org/scip/kleingebaeck/issues>.
|
||||||
.PP
|
.PP
|
||||||
Please repeat the failing command with debugging enabled \f(CW\*(C`\-d\*(C'\fR and
|
Please repeat the failing command with debugging enabled \f(CW\*(C`\-d\*(C'\fR and
|
||||||
include the output in the issue.
|
include the output in the issue.
|
||||||
|
|||||||
@@ -170,7 +170,7 @@ ENVIRONMENT VARIABLES
|
|||||||
BUGS
|
BUGS
|
||||||
In order to report a bug, unexpected behavior, feature requests or to
|
In order to report a bug, unexpected behavior, feature requests or to
|
||||||
submit a patch, please open an issue on github:
|
submit a patch, please open an issue on github:
|
||||||
<https://github.com/TLINDEN/kleingebaeck/issues>.
|
<https://codeberg.org/scip/kleingebaeck/issues>.
|
||||||
|
|
||||||
Please repeat the failing command with debugging enabled "-d" and
|
Please repeat the failing command with debugging enabled "-d" and
|
||||||
include the output in the issue.
|
include the output in the issue.
|
||||||
|
|||||||
@@ -207,7 +207,7 @@ commandline flags take precedence over env!
|
|||||||
|
|
||||||
In order to report a bug, unexpected behavior, feature requests
|
In order to report a bug, unexpected behavior, feature requests
|
||||||
or to submit a patch, please open an issue on github:
|
or to submit a patch, please open an issue on github:
|
||||||
L<https://github.com/TLINDEN/kleingebaeck/issues>.
|
L<https://codeberg.org/scip/kleingebaeck/issues>.
|
||||||
|
|
||||||
Please repeat the failing command with debugging enabled C<-d> and
|
Please repeat the failing command with debugging enabled C<-d> and
|
||||||
include the output in the issue.
|
include the output in the issue.
|
||||||
|
|||||||
12
main.go
12
main.go
@@ -49,6 +49,7 @@ func init() {
|
|||||||
fmt.Println("Please open a command shell and run it from there.")
|
fmt.Println("Please open a command shell and run it from there.")
|
||||||
fmt.Println()
|
fmt.Println()
|
||||||
fmt.Print("Press any key to quit: ")
|
fmt.Print("Press any key to quit: ")
|
||||||
|
|
||||||
_, err := bufio.NewReader(os.Stdin).ReadString('\n')
|
_, err := bufio.NewReader(os.Stdin).ReadString('\n')
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
@@ -144,13 +145,8 @@ func Main(output io.Writer) int {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return Die(err)
|
return Die(err)
|
||||||
}
|
}
|
||||||
conf.Outdir = outdir
|
|
||||||
|
|
||||||
// used for all HTTP requests
|
conf.Outdir = outdir
|
||||||
fetch, err := NewFetcher(conf)
|
|
||||||
if err != nil {
|
|
||||||
return Die(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// setup ad dir registry, needed to check for duplicates
|
// setup ad dir registry, needed to check for duplicates
|
||||||
DirsVisited = make(map[string]int)
|
DirsVisited = make(map[string]int)
|
||||||
@@ -159,14 +155,14 @@ func Main(output io.Writer) int {
|
|||||||
case len(conf.Adlinks) >= 1:
|
case len(conf.Adlinks) >= 1:
|
||||||
// directly backup ad listing[s]
|
// directly backup ad listing[s]
|
||||||
for _, uri := range conf.Adlinks {
|
for _, uri := range conf.Adlinks {
|
||||||
err := ScrapeAd(fetch, uri)
|
err = ScrapeAd(conf, uri)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Die(err)
|
return Die(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
case conf.User > 0:
|
case conf.User > 0:
|
||||||
// backup all ads of the given user (via config or cmdline)
|
// backup all ads of the given user (via config or cmdline)
|
||||||
err := ScrapeUser(fetch)
|
err := ScrapeUser(conf)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Die(err)
|
return Die(err)
|
||||||
}
|
}
|
||||||
|
|||||||
11
main_test.go
11
main_test.go
@@ -405,6 +405,7 @@ func InitValidSources() []Adsource {
|
|||||||
empty := AdConfig{}
|
empty := AdConfig{}
|
||||||
|
|
||||||
// prepare urls for the listing pages
|
// prepare urls for the listing pages
|
||||||
|
// nolint:prealloc
|
||||||
ads := []Adsource{
|
ads := []Adsource{
|
||||||
{
|
{
|
||||||
uri: fmt.Sprintf("%s%s?userId=1", Baseuri, Listuri),
|
uri: fmt.Sprintf("%s%s?userId=1", Baseuri, Listuri),
|
||||||
@@ -436,7 +437,7 @@ func InitInvalidSources() []Adsource {
|
|||||||
ads := []Adsource{
|
ads := []Adsource{
|
||||||
{
|
{
|
||||||
// valid ad page but without content
|
// valid ad page but without content
|
||||||
uri: fmt.Sprintf("%s/s-anzeige/empty/1", Baseuri),
|
uri: Baseuri + "/s-anzeige/empty/1",
|
||||||
content: GetTemplate(nil, &empty, EMPTYPAGE),
|
content: GetTemplate(nil, &empty, EMPTYPAGE),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -446,18 +447,18 @@ func InitInvalidSources() []Adsource {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
// some invalid page path
|
// some invalid page path
|
||||||
uri: fmt.Sprintf("%s/anzeige/name/1", Baseuri),
|
uri: Baseuri + "/anzeige/name/1",
|
||||||
content: GetTemplate(nil, &empty, "<html></html>"),
|
content: GetTemplate(nil, &empty, "<html></html>"),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
// some none-ad page
|
// some none-ad page
|
||||||
uri: fmt.Sprintf("%s/anzeige/name/1/foo/bar", Baseuri),
|
uri: Baseuri + "/anzeige/name/1/foo/bar",
|
||||||
content: GetTemplate(nil, &empty, "<html>HTTP 404: /eine-anzeige/ does not exist!</html>"),
|
content: GetTemplate(nil, &empty, "<html>HTTP 404: /eine-anzeige/ does not exist!</html>"),
|
||||||
status: 404,
|
status: 404,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
// valid ad page but 503
|
// valid ad page but 503
|
||||||
uri: fmt.Sprintf("%s/s-anzeige/503/1", Baseuri),
|
uri: Baseuri + "/s-anzeige/503/1",
|
||||||
content: GetTemplate(nil, &empty, "<html>HTTP 503: service unavailable</html>"),
|
content: GetTemplate(nil, &empty, "<html>HTTP 503: service unavailable</html>"),
|
||||||
status: 503,
|
status: 503,
|
||||||
},
|
},
|
||||||
@@ -531,6 +532,7 @@ func TestMain(t *testing.T) {
|
|||||||
defer func() { os.Args = oldargs }()
|
defer func() { os.Args = oldargs }()
|
||||||
|
|
||||||
httpmock.Activate()
|
httpmock.Activate()
|
||||||
|
|
||||||
defer httpmock.DeactivateAndReset()
|
defer httpmock.DeactivateAndReset()
|
||||||
|
|
||||||
// prepare httpmock responders
|
// prepare httpmock responders
|
||||||
@@ -568,6 +570,7 @@ func TestMainInvalids(t *testing.T) {
|
|||||||
defer func() { os.Args = oldargs }()
|
defer func() { os.Args = oldargs }()
|
||||||
|
|
||||||
httpmock.Activate()
|
httpmock.Activate()
|
||||||
|
|
||||||
defer httpmock.DeactivateAndReset()
|
defer httpmock.DeactivateAndReset()
|
||||||
|
|
||||||
// prepare httpmock responders
|
// prepare httpmock responders
|
||||||
|
|||||||
55
scrape.go
55
scrape.go
@@ -19,7 +19,9 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strconv"
|
"strconv"
|
||||||
@@ -32,30 +34,44 @@ import (
|
|||||||
|
|
||||||
// extract links from all ad listing pages (that is: use pagination)
|
// extract links from all ad listing pages (that is: use pagination)
|
||||||
// and scrape every page
|
// and scrape every page
|
||||||
func ScrapeUser(fetch *Fetcher) error {
|
func ScrapeUser(conf *Config) error {
|
||||||
adlinks := []string{}
|
adlinks := []string{}
|
||||||
|
|
||||||
baseuri := fmt.Sprintf("%s%s?userId=%d", Baseuri, Listuri, fetch.Config.User)
|
baseuri := fmt.Sprintf("%s%s?userId=%d", Baseuri, Listuri, conf.User)
|
||||||
page := 1
|
page := 1
|
||||||
uri := baseuri
|
uri := baseuri
|
||||||
|
|
||||||
slog.Info("fetching ad pages", "user", fetch.Config.User)
|
slog.Info("fetching ad pages", "user", conf.User)
|
||||||
|
|
||||||
for {
|
for {
|
||||||
var index Index
|
var index Index
|
||||||
|
|
||||||
slog.Debug("fetching page", "uri", uri)
|
slog.Debug("fetching page", "uri", uri)
|
||||||
|
|
||||||
body, err := fetch.Get(uri)
|
fetch, err := NewFetcher(conf)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
body, err := fetch.Get(uri)
|
||||||
defer func() {
|
defer func() {
|
||||||
|
if body == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// make sure body is completely flushed
|
||||||
|
if _, err := io.Copy(io.Discard, body); err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
if err := body.Close(); err != nil {
|
if err := body.Close(); err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
err = goq.NewDecoder(body).Decode(&index)
|
err = goq.NewDecoder(body).Decode(&index)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to goquery decode HTML index body: %w", err)
|
return fmt.Errorf("failed to goquery decode HTML index body: %w", err)
|
||||||
@@ -77,12 +93,12 @@ func ScrapeUser(fetch *Fetcher) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for index, adlink := range adlinks {
|
for index, adlink := range adlinks {
|
||||||
err := ScrapeAd(fetch, Baseuri+adlink)
|
err := ScrapeAd(conf, Baseuri+adlink)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if fetch.Config.Limit > 0 && index == fetch.Config.Limit-1 {
|
if conf.Limit > 0 && index == conf.Limit-1 {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -91,7 +107,7 @@ func ScrapeUser(fetch *Fetcher) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// scrape an ad. uri is the full uri of the ad, dir is the basedir
|
// scrape an ad. uri is the full uri of the ad, dir is the basedir
|
||||||
func ScrapeAd(fetch *Fetcher, uri string) error {
|
func ScrapeAd(conf *Config, uri string) error {
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
|
|
||||||
advertisement := &Ad{
|
advertisement := &Ad{
|
||||||
@@ -112,16 +128,31 @@ func ScrapeAd(fetch *Fetcher, uri string) error {
|
|||||||
// get the ad
|
// get the ad
|
||||||
slog.Debug("fetching ad page", "uri", uri)
|
slog.Debug("fetching ad page", "uri", uri)
|
||||||
|
|
||||||
body, err := fetch.Get(uri)
|
fetch, err := NewFetcher(conf)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
body, err := fetch.Get(uri)
|
||||||
defer func() {
|
defer func() {
|
||||||
|
if body == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// make sure body is completely flushed
|
||||||
|
if _, err := io.Copy(io.Discard, body); err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
if err := body.Close(); err != nil {
|
if err := body.Close(); err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
// extract ad contents with goquery/goq
|
// extract ad contents with goquery/goq
|
||||||
err = goq.NewDecoder(body).Decode(&advertisement)
|
err = goq.NewDecoder(body).Decode(&advertisement)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -135,7 +166,7 @@ func ScrapeAd(fetch *Fetcher, uri string) error {
|
|||||||
if advertisement.Incomplete() {
|
if advertisement.Incomplete() {
|
||||||
slog.Debug("got ad", "ad", advertisement)
|
slog.Debug("got ad", "ad", advertisement)
|
||||||
|
|
||||||
return fmt.Errorf("could not extract ad data from page, got empty struct")
|
return errors.New("could not extract ad data from page, got empty struct")
|
||||||
}
|
}
|
||||||
|
|
||||||
advertisement.DecodeAttributes()
|
advertisement.DecodeAttributes()
|
||||||
@@ -184,14 +215,11 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error {
|
|||||||
egroup := new(errgroup.Group)
|
egroup := new(errgroup.Group)
|
||||||
|
|
||||||
for _, imguri := range advertisement.Images {
|
for _, imguri := range advertisement.Images {
|
||||||
imguri := imguri
|
|
||||||
|
|
||||||
// we append the suffix later in NewImage() based on image format
|
// we append the suffix later in NewImage() based on image format
|
||||||
basefilename := filepath.Join(adpath, fmt.Sprintf("%d", img))
|
basefilename := filepath.Join(adpath, strconv.Itoa(img))
|
||||||
|
|
||||||
egroup.Go(func() error {
|
egroup.Go(func() error {
|
||||||
// wait a little
|
// wait a little
|
||||||
|
|
||||||
throttle := GetThrottleTime()
|
throttle := GetThrottleTime()
|
||||||
time.Sleep(throttle)
|
time.Sleep(throttle)
|
||||||
|
|
||||||
@@ -241,6 +269,7 @@ func ScrapeImages(fetch *Fetcher, advertisement *Ad, addir string) error {
|
|||||||
|
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
img++
|
img++
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
4
store.go
4
store.go
@@ -127,7 +127,6 @@ func WriteImage(filename string, reader *bytes.Reader) error {
|
|||||||
}()
|
}()
|
||||||
|
|
||||||
_, err = reader.WriteTo(file)
|
_, err = reader.WriteTo(file)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to write to image file: %w", err)
|
return fmt.Errorf("failed to write to image file: %w", err)
|
||||||
}
|
}
|
||||||
@@ -157,7 +156,6 @@ func ReadImage(filename string) (*bytes.Buffer, error) {
|
|||||||
|
|
||||||
func fileExists(filename string) bool {
|
func fileExists(filename string) bool {
|
||||||
info, err := os.Stat(filename)
|
info, err := os.Stat(filename)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// return false on any error
|
// return false on any error
|
||||||
return false
|
return false
|
||||||
@@ -172,11 +170,13 @@ func CheckAdVisited(conf *Config, adname string) bool {
|
|||||||
if Exists(DirsVisited, adname) {
|
if Exists(DirsVisited, adname) {
|
||||||
if conf.ForceDownload {
|
if conf.ForceDownload {
|
||||||
slog.Warn("an ad with the same name has already been downloaded, overwriting", "addir", adname)
|
slog.Warn("an ad with the same name has already been downloaded, overwriting", "addir", adname)
|
||||||
|
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
// don't overwrite
|
// don't overwrite
|
||||||
slog.Warn("an ad with the same name has already been downloaded, skipping (use -f to overwrite)", "addir", adname)
|
slog.Warn("an ad with the same name has already been downloaded, skipping (use -f to overwrite)", "addir", adname)
|
||||||
|
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user