Dev (#8)

* fixed conf parsing: variables can now be omitted from the config
* fix newlines: use CRLF on windows
* bump version

---------

Co-authored-by: Thomas von Dein <tom@vondein.org>

Makefile

@@ -17,7 +17,7 @@
 #
 # no need to modify anything below
 tool      = kleingebaeck
-VERSION   = $(shell grep VERSION main.go | head -1 | cut -d '"' -f2)
+VERSION   = $(shell grep VERSION config.go | head -1 | cut -d '"' -f2)
 archs     = darwin freebsd linux windows
 PREFIX    = /usr/local
 UID       = root

README.md

@@ -15,13 +15,51 @@ directory, each ad into its own subdirectory. The backup will contain
 a textfile `Adlisting.txt` which contains the ad contents as the
 title, body, price etc. All images will be downloaded as well.
 
+## Installation
+
 The   tool  doesn't   need   authentication  and   doesn't  have   any
 dependencies.  Just  download the  binary for  your platform  from the
 releases page and you're good to go.
 
-The releases also include a handy tarball which you can use to install
+### Installation using a pre-compiled binary
-the tool  system-wide including the  manual page. Just extract  it and
+
-type: `make install`.
+Go            to             the            [latest            release
+page](https://github.com/TLINDEN/kleingebaeck/releases/latest)     and
+look for your OS and platform. There are two options to install the binary:
+
+1.    Directly   download    the    binary    for   your    platoform,
+   e.g. `kleingebaeck-linux-amd64-0.0.5`, rename  it to `kleingebaeck`
+   (or  whatever  you  like  more!)  and put  it  into  your  bin  dir
+   (e.g. `$HOME/bin` or as root to `/usr/local/bin`).
+
+Be sure to verify the signature of the binary file. For this also download the matching `kleingebaeck-linux-amd64-0.0.5.sha256` file and:
+
+```shell
+cat kleingebaeck-linux-amd64-0.0.5.sha25 && sha256sum kleingebaeck-linux-amd64-0.0.5
+```
+You should see the same SHA256 hash.
+
+2.  You  may  also  download  a  binary  tarball  for  your  platform,
+   e.g.  `kleingebaeck-linux-amd64-0.0.5.tar.gz`,  unpack and  install
+   it. GNU Make is required for this:
+   
+```shell
+tar xvfz kleingebaeck-linux-amd64-0.0.5.tar.gz
+cd kleingebaeck-linux-amd64-0.0.5
+sudo make install
+```
+
+### Installation from source
+
+You will need the Golang toolchain  in order to build from source. GNU
+Make will also help but is not strictly neccessary.
+
+If you want to compile the tool yourself, use `git clone` to clone the
+repository.   Then   execute   `go    mod   tidy`   to   install   all
+dependencies. Then  just enter `go  build` or -  if you have  GNU Make
+installed - `make`.
+
+To install after building either copy the binary or execute `sudo make install`.
 
 ## Commandline options:
 
@@ -67,6 +105,27 @@ The `XXXXX` part is your userid.
 Put it into the configfile as outlined above. Also specify an output
 directory. Then just execute `kleingebaeck`.
 
+Inside the output directory you'll find a new subdirectory for each
+ad. Every directory contains a file `Adlisting.txt`, which will look
+somewhat like this:
+
+```default
+Title: A book I sell
+Price: 99 € VB
+Id: 1919191919
+Category: Sachbücher
+Condition: Sehr Gut
+Created: 10.12.2023
+
+This is the description text.
+
+Pay with paypal.
+```
+
+You can change the formatting using the `template` config
+variable. The supplied sample config contains the default template.
+
+All images will be stored in the same directory.
 
 ## Kleingebäck?
 

config.go

@@ -22,11 +22,24 @@ import (
 	"github.com/hashicorp/hcl/v2/hclsimple"
 )
 
+const (
+	VERSION         string = "0.0.6"
+	Baseuri         string = "https://www.kleinanzeigen.de"
+	Listuri         string = "/s-bestandsliste.html"
+	Defaultdir      string = "."
+	DefaultTemplate string = "Title: {{.Title}}\nPrice: {{.Price}}\nId: {{.Id}}\n" +
+		"Category: {{.Category}}\nCondition: {{.Condition}}\nCreated: {{.Created}}\n\n{{.Text}}\n"
+	DefaultTemplateWin string = "Title: {{.Title}}\r\nPrice: {{.Price}}\r\nId: {{.Id}}\r\n" +
+		"Category: {{.Category}}\r\nCondition: {{.Condition}}\r\nCreated: {{.Created}}\r\n\r\n{{.Text}}\r\n"
+	Useragent string = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " +
+		"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+)
+
 type Config struct {
-	Verbose  bool   `hcl:"verbose"`
+	Verbose  *bool   `hcl:"verbose"`
-	User     int    `hcl:"user"`
+	User     *int    `hcl:"user"`
-	Outdir   string `hcl:"outdir"`
+	Outdir   *string `hcl:"outdir"`
-	Template string `hcl:"template"`
+	Template *string `hcl:"template"`
 }
 
 func ParseConfigfile(file string) (*Config, error) {

example.hcl

@@ -15,6 +15,5 @@ verbose = true
 outdir = "test"
 
 # template. leave empty to use the default one, which is:
-# Title: %s\nPrice: %s\nId: %s\nCategory: %s\nCondition: %s\nCreated: %s\nBody:\n\n%s\n
+# "Title: {{.Title}}\nPrice: {{.Price}}\nId: {{.Id}}\nCategory: {{.Category}}\nCondition: {{.Condition}}\nCreated: {{.Created}}\n\n{{.Text}}\n"
-# take care to include exactly 7 times '%s'!
 template = ""

kleingebaeck.1

@@ -133,7 +133,7 @@
 .\" ========================================================================
 .\"
 .IX Title "KLEINGEBAECK 1"
-.TH KLEINGEBAECK 1 "2023-12-16" "1" "User Commands"
+.TH KLEINGEBAECK 1 "2023-12-17" "1" "User Commands"
 .\" For nroff, turn off justification.  Always turn off hyphenation; it makes
 .\" way too many mistakes in technical documents.
 .if n .ad l
@@ -179,10 +179,11 @@ Format is simple:
 Be carefull if you want to change the template. The default one looks like this:
 .PP
 .Vb 1
-\&   Title: %s\enPrice: %s\enId: %s\enCategory: %s\enCondition: %s\enCreated: %s\enBody:\en\en%s\en
+\&   Title: {{.Title}}\enPrice: {{.Price}}\enId: {{.Id}}\enCategory: {{.Category}}\enCondition: {{.Condition}}\enCreated: {{.Created}}\en\en{{.Text}}\en
 .Ve
 .PP
-If you change it, include 7 times the '%s' format tag.
+You can left out certain fields and use any formatting you like. Refer
+to <https://pkg.go.dev/text/template> for details how to write a template.
 .SH "SETUP"
 .IX Header "SETUP"
 To setup the tool, you need to lookup your userid on

kleingebaeck.go

@@ -39,9 +39,11 @@ CONFIGURATION
     Be carefull if you want to change the template. The default one looks
     like this:
 
-       Title: %s\nPrice: %s\nId: %s\nCategory: %s\nCondition: %s\nCreated: %s\nBody:\n\n%s\n
+       Title: {{.Title}}\nPrice: {{.Price}}\nId: {{.Id}}\nCategory: {{.Category}}\nCondition: {{.Condition}}\nCreated: {{.Created}}\n\n{{.Text}}\n
 
-    If you change it, include 7 times the '%s' format tag.
+    You can left out certain fields and use any formatting you like. Refer
+    to <https://pkg.go.dev/text/template> for details how to write a
+    template.
 
 SETUP
     To setup the tool, you need to lookup your userid on kleinanzeigen.de.

kleingebaeck.pod

@@ -38,9 +38,10 @@ Format is simple:
 
 Be carefull if you want to change the template. The default one looks like this:
 
-   Title: %s\nPrice: %s\nId: %s\nCategory: %s\nCondition: %s\nCreated: %s\nBody:\n\n%s\n
+   Title: {{.Title}}\nPrice: {{.Price}}\nId: {{.Id}}\nCategory: {{.Category}}\nCondition: {{.Condition}}\nCreated: {{.Created}}\n\n{{.Text}}\n
 
-If you change it, include 7 times the '%s' format tag.
+You can left out certain fields and use any formatting you like. Refer
+to L<https://pkg.go.dev/text/template> for details how to write a template.
 
 =head1 SETUP
 

main.go

@@ -22,20 +22,13 @@ import (
 	"fmt"
 	"log/slog"
 	"os"
+	"runtime"
 	"runtime/debug"
 
 	"github.com/lmittmann/tint"
 	flag "github.com/spf13/pflag"
 )
 
-const VERSION string = "0.0.3"
-const Useragent string = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " +
-	"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
-const Baseuri string = "https://www.kleinanzeigen.de"
-const Listuri string = "/s-bestandsliste.html"
-const Defaultdir string = "."
-const DefaultTemplate string = "Title: %s\nPrice: %s\nId: %s\nCategory: %s\nCondition: %s\nCreated: %s\nBody:\n\n%s\n"
-
 const Usage string = `This is kleingebaeck, the kleinanzeigen.de backup tool.
 Usage: kleingebaeck [-dvVhmoc] [<ad-listing-url>,...]
 Options:
@@ -117,7 +110,7 @@ func Main() int {
 		return Die(err)
 	}
 
-	if enableverbose || conf.Verbose {
+	if enableverbose || *conf.Verbose {
 		logLevel.Set(slog.LevelInfo)
 	}
 
@@ -143,8 +136,8 @@ func Main() int {
 	slog.Debug("config", "conf", conf)
 
 	if len(dir) == 0 {
-		if len(conf.Outdir) > 0 {
+		if len(*conf.Outdir) > 0 {
-			dir = conf.Outdir
+			dir = *conf.Outdir
 		} else {
 			dir = Defaultdir
 		}
@@ -158,8 +151,11 @@ func Main() int {
 
 	// which template to use
 	template := DefaultTemplate
-	if len(conf.Template) > 0 {
+	if runtime.GOOS == "windows" {
-		template = conf.Template
+		template = DefaultTemplateWin
+	}
+	if len(*conf.Template) > 0 {
+		template = *conf.Template
 	}
 
 	// directly backup ad listing[s]
@@ -175,8 +171,8 @@ func Main() int {
 	}
 
 	// backup all ads of the given user (via config or cmdline)
-	if uid == 0 && conf.User > 0 {
+	if uid == 0 && *conf.User > 0 {
-		uid = conf.User
+		uid = *conf.User
 	}
 
 	if uid > 0 {

mkrel.sh

@@ -40,6 +40,11 @@ for D in $DIST; do
     os=${D/\/*/}
     arch=${D/*\//}
     binfile="releases/${tool}-${os}-${arch}-${version}"
+
+    if test "$os" = "windows"; then
+        binfile="${binfile}.exe"
+    fi
+    
     tardir="${tool}-${os}-${arch}-${version}"
     tarfile="releases/${tool}-${os}-${arch}-${version}.tar.gz"
     set -x

scrape.go

@@ -23,7 +23,7 @@ import (
 	"io"
 	"log/slog"
 	"net/http"
-	"os"
+	"path/filepath"
 	"strings"
 	"sync"
 
@@ -160,28 +160,12 @@ func Scrape(uri string, dir string, template string) error {
 	}
 	slog.Debug("extracted ad listing", "ad", ad)
 
-	// prepare output dir
+	// write listing
-	dir = dir + "/" + ad.Slug
+	err = WriteAd(dir, ad, template)
-	err = Mkdir(dir)
 	if err != nil {
 		return err
 	}
 
-	// write ad file
-	listingfile := strings.Join([]string{dir, "Adlisting.txt"}, "/")
-	f, err := os.Create(listingfile)
-	if err != nil {
-		return err
-	}
-
-	ad.Text = strings.ReplaceAll(ad.Text, "<br/>", "\n")
-	_, err = fmt.Fprintf(f, template,
-		ad.Title, ad.Price, ad.Id, ad.Category, ad.Condition, ad.Created, ad.Text)
-	if err != nil {
-		return err
-	}
-	slog.Info("wrote ad listing", "listingfile", listingfile)
-
 	return ScrapeImages(dir, ad)
 }
 
@@ -193,7 +177,7 @@ func ScrapeImages(dir string, ad *Ad) error {
 	failure := make(chan string)
 
 	for _, imguri := range ad.Images {
-		file := fmt.Sprintf("%s/%d.jpg", dir, img)
+		file := filepath.Join(dir, ad.Slug, fmt.Sprintf("%d.jpg", img))
 		go func() {
 			defer wg.Done()
 			err := Getimage(imguri, file)
@@ -230,13 +214,7 @@ func Getimage(uri, fileName string) error {
 		return errors.New("received non 200 response code")
 	}
 
-	file, err := os.Create(fileName)
+	err = WriteImage(fileName, response.Body)
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-
-	_, err = io.Copy(file, response.Body)
 	if err != nil {
 		return err
 	}

store.go

@@ -0,0 +1,78 @@
+/*
+Copyright © 2023 Thomas von Dein
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package main
+
+import (
+	"io"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	tpl "text/template"
+)
+
+func WriteAd(dir string, ad *Ad, template string) error {
+	// prepare output dir
+	dir = filepath.Join(dir, ad.Slug)
+	err := Mkdir(dir)
+	if err != nil {
+		return err
+	}
+
+	// write ad file
+	listingfile := filepath.Join(dir, "Adlisting.txt")
+	f, err := os.Create(listingfile)
+	if err != nil {
+		return err
+	}
+
+	if runtime.GOOS == "windows" {
+		ad.Text = strings.ReplaceAll(ad.Text, "<br/>", "\r\n")
+	} else {
+		ad.Text = strings.ReplaceAll(ad.Text, "<br/>", "\n")
+	}
+
+	tmpl, err := tpl.New("adlisting").Parse(template)
+	if err != nil {
+		return err
+	}
+	err = tmpl.Execute(f, ad)
+	if err != nil {
+		return err
+	}
+
+	slog.Info("wrote ad listing", "listingfile", listingfile)
+
+	return nil
+}
+
+func WriteImage(filename string, reader io.ReadCloser) error {
+	file, err := os.Create(filename)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	_, err = io.Copy(file, reader)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}