diff --git a/README.md b/README.md index 857766f..139e4de 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,136 @@ # apsystems-ez1-loki-grafana + Scraper and Grafana Dashboard for ApSystems EZ1 Solar Inverter + +# Description + +The APsystems EZ1 Microinverter has a nice Rest API, which you can use +to scrape power output data. There's a [neat python library +available](https://github.com/SonnenladenGmbH/APsystems-EZ1-API), but +I couldn't use it because my old Raspberry Pi doesn't have support for +the latest python version, which is +[required](https://github.com/SonnenladenGmbH/APsystems-EZ1-API/issues/22) +by this library. + +So, I wrote a simple shell script to scrape the API, which is simple +enough. All you need is [curl](https://github.com/curl/curl) and +[jq](https://github.com/jqlang/jq). + +I then upload the collected data to a cloud node where I happen to run +a [Grafana](https://github.com/grafana/grafana) instance. I feed the +generated logs into [Loki](https://github.com/grafana/loki) and +visualze it then in a nice dashboard: + +![Screenshot](https://github.com/TLINDEN/apsystems-ez1-loki-grafana/blob/main/dashboard-screenshot.png) + +# Installation + +## Setup the Microinverter + +The [python +library README](https://github.com/SonnenladenGmbH/APsystems-EZ1-API) +contains good advice how to enable the API on your inverter. Just +follow it precisely. + +## Setup a static ip address for your inverter + +This step is important. Connect to your router and configure the +current DHCP ip address to be static. Refer to the router +documentation on how to do this. + +## Get some Raspberry Pi or similar up as collector + +This is not part of this documetation. You'll need some system, which +runs 24/7 on your home network. I use a Raspberry Pi, but any unix +like system will do. Just make sure you have SSH access and it has +`curl` and `jq` installed and `crond` is enabled. No root permissions +are required. + +## Copy the scraper script to the collector device + +Copy the file +[scrape-inverter.sh](https://github.com/TLINDEN/apsystems-ez1-loki-grafana/blob/main/scrape-inverter.sh) +to the collector device. Ensure it is executable: + +`chmod 755 scrape-inverter.sh` + +## Customize the script + +Now edit the script and set the ip address accordingly. + +## Test the script + +If all is prepared, just execute the script. The output should look +something like this: + +```shell +you@pi: % ./scrape-inverter.sh +ts=2024-04-18T17:55:14+01:00 p1=97 p2=97 ip=192.168.128.117 version=1.7.0 deviceid=E07000000807 ssid=FOO minpower=30 maxpower=800 +``` + +## Setup the cronjob + +Now it's up to you how you want to collect the logs. You may append +the output to a log file and transfer it to somewhere else or you may +just run Loki and Grafana on the same system. + +I modified the script like this (at the end): + +```sh +if test -n "$p1" -a -n "$ip"; then + echo "ts=$ts p1=$p1 p2=$p2 ip=$ip version=$version deviceid=$deviceid ssid=$ssid minpower=$minpower maxpower=$maxpower" | ssh uber "cat >> /var/log/solar.metric" +else + echo "$ts got invalid data!" +fi +``` + +This sends the log to the cloud machine and appends it over there. + +## Loki config + +I use promtail (which is part of Loki) to feed the logs into +Loki. Here's the relevant part of the promtail config: + +```yaml +scrape_configs: +- job_name: system + static_configs: + - targets: + - localhost + labels: + job: varlogs + __path__: /var/log/*metric +``` + +To check if the logs actually show up, try the following query in the +explorer, make sure to set the source to Loki: + +```default +{filename="/var/log/solar.metric"} |= `p1` | logfmt +``` + +It should look like this: + +![Screenshot](https://github.com/TLINDEN/apsystems-ez1-loki-grafana/blob/main/lokiexplore.png) + +## Install the Grafana dashboard + +If the logs show up in Loki, create a new dashboard. Import the file +[apsystems-ez1-loki-dashboard.json](https://github.com/TLINDEN/apsystems-ez1-loki-grafana/blob/main/apsystems-ez1-loki-dashboard.json) +as JSON. + +You may modify the queries to match the filename you are using for +your logfile or course. + + +# Report bugs + +[Please open an issue](https://github.com/TLINDEN/apsystems-ez1-loki-grafana/issues). Thanks! + +# License + +This work is licensed under the terms of the BSD license. + +# Author + +Copyleft (c) 2024 Thomas von Dein diff --git a/apsystems-ez1-loki-dashboard.json b/apsystems-ez1-loki-dashboard.json new file mode 100644 index 0000000..eb93926 --- /dev/null +++ b/apsystems-ez1-loki-dashboard.json @@ -0,0 +1,562 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 31, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "HgMcR8Z4k" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 38, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "watt" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "sum", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "HgMcR8Z4k" + }, + "editorMode": "code", + "expr": "max_over_time({filename=\"/var/log/solar.metric\"} |= `p1` | logfmt | unwrap p1 | __error__=\"\" [5m]) by (instance)", + "legendFormat": "Panel 1", + "queryType": "range", + "refId": "A" + }, + { + "datasource": { + "type": "loki", + "uid": "HgMcR8Z4k" + }, + "editorMode": "code", + "expr": "max_over_time({filename=\"/var/log/solar.metric\"} |= `p2` | logfmt | unwrap p2 | __error__=\"\" [5m]) by (instance)", + "hide": false, + "legendFormat": "Panel 2", + "queryType": "range", + "refId": "B" + } + ], + "title": "Total Power Output", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "Total", + "binary": { + "left": "Panel 1", + "reducer": "sum", + "right": "Panel 2" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "HgMcR8Z4k" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 38, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "watt" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "sum", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "HgMcR8Z4k" + }, + "editorMode": "code", + "expr": "max_over_time({filename=\"/var/log/solar.metric\"} |= `p1` | logfmt | unwrap p1 | __error__=\"\" [5m]) by (instance)", + "legendFormat": "Panel 1", + "queryType": "range", + "refId": "A" + }, + { + "datasource": { + "type": "loki", + "uid": "HgMcR8Z4k" + }, + "editorMode": "code", + "expr": "max_over_time({filename=\"/var/log/solar.metric\"} |= `p2` | logfmt | unwrap p2 | __error__=\"\" [5m]) by (instance)", + "hide": false, + "legendFormat": "Panel 2", + "queryType": "range", + "refId": "B" + } + ], + "title": "Power Output by Panel", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "HgMcR8Z4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 0, + "y": 9 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "firstNotNull" + ], + "fields": "/.*/", + "values": false + }, + "text": { + "titleSize": 14, + "valueSize": 14 + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "HgMcR8Z4k" + }, + "editorMode": "code", + "expr": "{filename=\"/var/log/solar.metric\"} | logfmt", + "queryType": "range", + "refId": "A" + } + ], + "title": "PV Info", + "transformations": [ + { + "id": "extractFields", + "options": { + "format": "json", + "replace": false, + "source": "labels" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Line": true, + "Time": true, + "filename": true, + "id": true, + "job": true, + "labels": true, + "p1": false, + "p2": false, + "ts": false, + "tsNs": true, + "version": false + }, + "indexByName": { + "Line": 2, + "Time": 1, + "deviceid": 6, + "filename": 7, + "id": 4, + "ip": 8, + "job": 9, + "labels": 0, + "maxpower": 10, + "minpower": 11, + "p1": 12, + "p2": 13, + "ssid": 14, + "ts": 5, + "tsNs": 3, + "version": 15 + }, + "renameByName": { + "deviceid": "Device ID", + "ip": "IP", + "job": "", + "maxpower": "Max Power", + "minpower": "Min Power", + "p1": "Current Power Output Panel 1", + "p2": "Current Power Output Panel 2", + "ssid": "WIFI SSID", + "ts": "Responsed", + "version": "Version" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Responsed" + } + ] + } + }, + { + "disabled": true, + "id": "reduce", + "options": { + "labelsToFields": false, + "reducers": [] + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "loki", + "uid": "HgMcR8Z4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-RdYlGr" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "watth" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 18, + "x": 6, + "y": 9 + }, + "id": 7, + "maxDataPoints": 14, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "max", + "lastNotNull", + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "HgMcR8Z4k" + }, + "editorMode": "code", + "expr": "sum by (instance) (sum_over_time({filename=\"/var/log/solar.metric\"} |= \"p1\" | logfmt | __error__ = \"\" | unwrap p1 [24h])) / 24", + "key": "Q-ea8d689d-0562-4064-a378-2e92ce48de63-0", + "legendFormat": "Panel1", + "queryType": "range", + "refId": "A" + }, + { + "datasource": { + "type": "loki", + "uid": "HgMcR8Z4k" + }, + "editorMode": "code", + "expr": "sum by (instance) (sum_over_time({filename=\"/var/log/solar.metric\"} |= \"p2\" | logfmt | __error__ = \"\" | unwrap p2 [24h])) / 24", + "hide": false, + "legendFormat": "Panel2", + "queryType": "range", + "refId": "B" + } + ], + "timeFrom": "now-14d", + "title": "Total Power Output per Day", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "Total Kw/h", + "binary": { + "left": "Panel1", + "reducer": "sum", + "right": "Panel2" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "barchart" + } + ], + "refresh": "1m", + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Solar", + "uid": "c99fbea1-6d83-4976-bfd4-3e576ded13b2", + "version": 29, + "weekStart": "" +} diff --git a/dashboard-screenshot.png b/dashboard-screenshot.png new file mode 100644 index 0000000..671a8c3 Binary files /dev/null and b/dashboard-screenshot.png differ diff --git a/lokiexplore.png b/lokiexplore.png new file mode 100644 index 0000000..29884c0 Binary files /dev/null and b/lokiexplore.png differ diff --git a/scrape-inverter.sh b/scrape-inverter.sh new file mode 100644 index 0000000..9aa840e --- /dev/null +++ b/scrape-inverter.sh @@ -0,0 +1,42 @@ +#!/bin/bash +host="192.168.128.117:8050" +metrics="getOutputData" +info="getDeviceInfo" +ts=$(date +%Y-%m-%dT%H:%M:%S+01:00) + +data=$(curl -s http://$host/$metrics | jq -r '.data | join (" ")') + +if test -n "$data"; then + read -r -a params <<< "${data}" + p1="${params[0]}" # power generating on channel 1 + e1="${params[1]}" # energy generated since startup on channel 1 + te1="${params[2]}" # lifetime energy generated on channel 1 + p2="${params[3]}" + e2="${params[4]}" + te2="${params[5]}" +else + echo "$ts ot no data!" + return +fi + +infodata=$(curl -s http://$host/$info | jq -r '.data | join (" ")') + +if test -n "$infodata"; then + read -r -a params <<< "${infodata}" + deviceid="${params[0]}" + device="${params[1]}" + version="${params[2]}" + ssid="${params[3]}" + ip="${params[4]}" + minpower="${params[5]}" + maxpower="${params[6]}" +else + echo "$ts ot no info!" + return +fi + +if test -n "$p1" -a -n "$ip"; then + echo "ts=$ts p1=$p1 p2=$p2 ip=$ip version=$version deviceid=$deviceid ssid=$ssid minpower=$minpower maxpower=$maxpower" +else + echo "$ts got invalid data!" +fi