From 14b023ae20108a9a24b262c851996f2229619234 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Wed, 1 Mar 2023 17:33:04 +0100 Subject: [PATCH] DAQ: Add decoder for Fine Offset (FOSHK) weather station equipment Through the excellent `ecowitt2mqtt` machinery [1], this supports any weather station/gateway that is produced by Shenzhen Fine Offset Electronics Co., Ltd. [2] aka. Fine Offset aka. OFFSET. This includes brands that white-label Fine Offset equipment, such as: - Ambient Weather (U.S.) - Ecowitt (China, Hong Kong) - Froggit (Germany) By default, `ecowitt2mqtt` is configured to output data in the "metric" unit system. [1] https://github.com/bachya/ecowitt2mqtt [2] https://www.foshk.com/ --- CHANGES.rst | 4 +- Makefile | 2 +- doc/source/setup/python-package.rst | 2 +- kotori/daq/decoder/fineoffset.py | 211 ++++++++++++++++++ kotori/io/protocol/http.py | 11 +- packaging/wheels/build.sh | 4 +- setup.py | 3 + tasks/packaging/model.py | 4 +- ...t_ecowitt.py => test_device_fineoffset.py} | 23 +- 9 files changed, 248 insertions(+), 16 deletions(-) create mode 100644 kotori/daq/decoder/fineoffset.py rename test/{test_ecowitt.py => test_device_fineoffset.py} (72%) diff --git a/CHANGES.rst b/CHANGES.rst index 19f5792e..14dc8cd5 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -7,7 +7,9 @@ in progress =========== - CI: Update to Grafana 9.3.0 -- DAQ: Mask ``PASSKEY`` variable coming from HTTP, emitted by Ecowitt +- DAQ: Add adapter/decoder for Fine Offset weather station equipment, + with white-label products by Ambient Weather, Ecowitt, and Froggit. + Configure it to output data in "metric" unit system by default. .. _kotori-0.27.0: diff --git a/Makefile b/Makefile index 66b5f777..81595525 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,7 @@ virtualenv-docs: setup-virtualenv # Install requirements for development. virtualenv-dev: setup-virtualenv @$(pip) install --upgrade --prefer-binary --requirement=requirements-test.txt - @$(pip) install --upgrade --prefer-binary --editable=.[daq,daq_geospatial,export,scientific,firmware] + @$(pip) install --upgrade --prefer-binary --editable=.[daq,daq_geospatial,daq_fineoffset,export,scientific,firmware] # Install requirements for releasing. install-releasetools: setup-virtualenv diff --git a/doc/source/setup/python-package.rst b/doc/source/setup/python-package.rst index 5044c67f..f19f4821 100644 --- a/doc/source/setup/python-package.rst +++ b/doc/source/setup/python-package.rst @@ -54,7 +54,7 @@ Kotori releases are published to https://pypi.org/project/kotori/ :: pip install --user kotori[daq,export] # Install more extra features - pip install --user kotori[daq,daq_geospatial,export,plotting,scientific,firmware] + pip install --user kotori[daq,daq_geospatial,daq_fineoffset,export,plotting,scientific,firmware] # Install particular version pip install --user kotori[daq,export]==0.26.6 diff --git a/kotori/daq/decoder/fineoffset.py b/kotori/daq/decoder/fineoffset.py new file mode 100644 index 00000000..e09763cd --- /dev/null +++ b/kotori/daq/decoder/fineoffset.py @@ -0,0 +1,211 @@ +# -*- coding: utf-8 -*- +# (c) 2023 Andreas Motl +import json +import typing as t + + +class FineOffsetDecoder: + """ + Decode data format submitted by Fine Offset (FOSHK) weather stations, using the + excellent `ecowitt2mqtt` machinery [1]. + + By wrapping it into a Kotori decoder, it will simplify operation and maintenance. + Effectively, there are fewer moving parts involved, yet all features can be leveraged: + + - anonymization of data + - convenience of unit conversion + - additional calculated values + - integrated test coverage + - no installation overhead + + Despite the name of the library, `ecowitt2mqtt` [2] supports any weather + station/gateway that is produced by Shenzhen Fine Offset Electronics Co., Ltd. [3] + aka. Fine Offset aka. OFFSET. This includes brands that white-label Fine Offset + equipment, such as: + + - Ambient Weather (U.S.) + - Ecowitt (China, Hong Kong) + - Froggit (Germany) + + ...and many others. For more information on how these brands relate to one another, + see the forum post at [4]. + + Although there are some small differences between how these various branded devices + are configured, `ecowitt2mqtt` endeavors to incorporate them all with minimal effort + on the user's part [5]. + + `ecowitt2mqtt` currently supports the following input data formats [6]: + + - `ambient_weather` + - `ecowitt` + + [1] https://community.hiveeyes.org/t/more-data-acquisition-payload-formats-for-kotori/1421/17 + [2] https://github.com/bachya/ecowitt2mqtt + [3] https://www.foshk.com/ + [4] https://www.wxforum.net/index.php?topic=40730.0 + [5] https://github.com/bachya/ecowitt2mqtt/tree/dev#supported-brands + [6] https://github.com/bachya/ecowitt2mqtt/tree/dev#input-data-formats + + Example data + ============ + + This is an input data sample provided by an Ecowitt weather station, then + converted to the "metric" unit system and with additional computed values + by `ecowitt2mqtt`, displayed in the "Output" section. + + Input + ----- + :: + + { + "PASSKEY": "B950C...[obliterated]", + "stationtype": "EasyWeatherPro_V5.0.6", + "runtime": "456128", + "dateutc": "2023-02-20 16:02:19", + "tempinf": "69.8", + "humidityin": "47", + "baromrelin": "29.713", + "baromabsin": "29.713", + "tempf": "48.4", + "humidity": "80", + "winddir": "108", + "windspeedmph": "1.12", + "windgustmph": "4.92", + "maxdailygust": "12.97", + "solarradiation": "1.89", + "uv": "0", + "rainratein": "0.000", + "eventrainin": "0.000", + "hourlyrainin": "0.000", + "dailyrainin": "0.028", + "weeklyrainin": "0.098", + "monthlyrainin": "0.909", + "yearlyrainin": "0.909", + "temp1f": "45.0", + "humidity1": "90", + "soilmoisture1": "46", + "soilmoisture2": "53", + "tf_ch1": "41.9", + "rrain_piezo": "0.000", + "erain_piezo": "0.000", + "hrain_piezo": "0.000", + "drain_piezo": "0.028", + "wrain_piezo": "0.043", + "mrain_piezo": "0.492", + "yrain_piezo": "0.492", + "wh65batt": "0", + "wh25batt": "0", + "batt1": "0", + "soilbatt1": "1.6", + "soilbatt2": "1.6", + "tf_batt1": "1.60", + "wh90batt": "3.04", + "freq": "868M", + "model": "HP1000SE-PRO_Pro_V1.8.5", + } + + Output + ------ + :: + + { + "runtime": 456128.0, + "tempin": 20.999999999999996, + "humidityin": 47.0, + "baromrel": 1006.1976567045213, + "baromabs": 1006.1976567045213, + "temp": 9.11111111111111, + "humidity": 80.0, + "winddir": 108.0, + "windspeed": 1.8024652800000003, + "windgust": 7.91797248, + "maxdailygust": 20.873191679999998, + "solarradiation": 1.89, + "uv": 0.0, + "rainrate": 0.0, + "eventrain": 0.0, + "hourlyrain": 0.0, + "dailyrain": 0.7112, + "weeklyrain": 2.4892000000000003, + "monthlyrain": 23.0886, + "yearlyrain": 23.0886, + "temp1": 7.222222222222222, + "humidity1": 90.0, + "soilmoisture1": 46.0, + "soilmoisture2": 53.0, + "tf_ch1": 5.499999999999999, + "rrain_piezo": 0.0, + "erain_piezo": 0.0, + "hrain_piezo": 0.0, + "drain_piezo": 0.7112, + "wrain_piezo": 1.0921999999999998, + "mrain_piezo": 12.4968, + "yrain_piezo": 12.4968, + "wh65batt": "OFF", + "wh25batt": "OFF", + "batt1": "OFF", + "soilbatt1": 1.6, + "soilbatt2": 1.6, + "tf_batt1": 1.6, + "wh90batt": 3.04, + "beaufortscale": 1, + "dewpoint": 5.846942096976985, + "feelslike": 9.11111111111111, + "frostpoint": 4.706401162443284, + "frostrisk": "No risk", + "heatindex": 8.166666666666668, + "humidex": 9, + "humidex_perception": "Comfortable", + "humidityabs": 7.101409765339333, + "humidityabsin": 7.101409765339333, + "relative_strain_index": null, + "relative_strain_index_perception": null, + "safe_exposure_time_skin_type_1": null, + "safe_exposure_time_skin_type_2": null, + "safe_exposure_time_skin_type_3": null, + "safe_exposure_time_skin_type_4": null, + "safe_exposure_time_skin_type_5": null, + "safe_exposure_time_skin_type_6": null, + "simmerindex": null, + "simmerzone": null, + "solarradiation_perceived": 47.57669425765605, + "thermalperception": "Dry", + "windchill": null + } + + """ + + @staticmethod + def detect(data: t.Dict[str, str]) -> bool: + """ + Determine whether the data payload is submitted by a Fine Offset device. + + TODO: Maybe leverage field names in `ecowitt2mqtt.data.DEFAULT_KEYS_TO_IGNORE`? + """ + return "PASSKEY" in data and "stationtype" in data and "model" in data + + @staticmethod + def decode(data: t.Dict[str, str]) -> t.Dict[str, t.Any]: + """ + Decode data payload submitted by a Fine Offset device, using `ecowitt2mqtt`. + """ + from ecowitt2mqtt.config import Config + from ecowitt2mqtt.data import ProcessedData + + config = Config( + { + # Both configuration variables are currently *required* by `ecowitt2mqtt`. + # Fortunately, `mqtt_broker` can be left empty. + # TODO: Can this be improved if upstream would accept a corresponding patch? + "hass_discovery": True, + "mqtt_broker": "", + # Output values in *metric* unit system by default. + # TODO: Make output unit system configurable. + "output_unit_system": "metric", + } + ) + processed_data = ProcessedData(config=config, data=data) + converted_data = { + key: value.value for key, value in processed_data.output.items() + } + return converted_data diff --git a/kotori/io/protocol/http.py b/kotori/io/protocol/http.py index b1f1eb8d..5de69c5b 100644 --- a/kotori/io/protocol/http.py +++ b/kotori/io/protocol/http.py @@ -20,6 +20,8 @@ from twisted.web.server import Site from twisted.web.error import Error from twisted.python.compat import nativeString + +from kotori.daq.decoder.fineoffset import FineOffsetDecoder from kotori.io.router.path import PathRoutingEngine from kotori.io.export.tabular import UniversalTabularExporter from kotori.io.export.plot import UniversalPlotter @@ -330,11 +332,10 @@ def read_request(self, bucket): if request.method == 'POST': data = self.data_acquisition(bucket) - # Mask `PASSKEY` ingress variable. - # https://github.com/daq-tools/kotori/discussions/122 - # https://community.hiveeyes.org/t/ecowitt-wunderground-api-fur-weather-hiveeyes-org-nutzbar/4735 - if "PASSKEY" in data: - del data["PASSKEY"] + # Decode data from specific devices. + # TODO: Handle decoding data from specific devices in a more generic way. + if FineOffsetDecoder.detect(data): + data = FineOffsetDecoder.decode(data) return data diff --git a/packaging/wheels/build.sh b/packaging/wheels/build.sh index bdde5aed..1b4ca022 100755 --- a/packaging/wheels/build.sh +++ b/packaging/wheels/build.sh @@ -17,9 +17,9 @@ function invoke_build() { flavor=$1 if [ $flavor = "full" ]; then - extras="daq,daq_geospatial,export,plotting,firmware,scientific" + extras="daq,daq_geospatial,daq_fineoffset,export,plotting,firmware,scientific" elif [ $flavor = "standard" ]; then - extras="daq,daq_geospatial,export" + extras="daq,daq_geospatial,daq_fineoffset,export" else echo "ERROR: Package flavor '${flavor}' unknown or not implemented" exit 1 diff --git a/setup.py b/setup.py index b02a4c1b..bae422b0 100644 --- a/setup.py +++ b/setup.py @@ -88,6 +88,9 @@ 'tabulate==0.7.5', # 0.8.2 'sympy==0.7.6.1', # 1.1.1 ], + 'daq_fineoffset': [ + 'ecowitt2mqtt<=2023.02.1' + ], 'storage_plus': [ 'alchimia>=0.4,<1', ], diff --git a/tasks/packaging/model.py b/tasks/packaging/model.py index 4e3fefb5..741bf44a 100644 --- a/tasks/packaging/model.py +++ b/tasks/packaging/model.py @@ -83,10 +83,10 @@ def resolve(self): self.features = "daq" elif self.flavor == "standard": self.name = "kotori-standard" - self.features = "daq,daq_geospatial,export" + self.features = "daq,daq_geospatial,daq_fineoffset,export" elif self.flavor == "full": self.name = "kotori" - self.features = "daq,daq_geospatial,export,plotting,firmware,scientific" + self.features = "daq,daq_geospatial,daq_fineoffset,export,plotting,firmware,scientific" else: raise ValueError("Unknown package flavor") diff --git a/test/test_ecowitt.py b/test/test_device_fineoffset.py similarity index 72% rename from test/test_ecowitt.py rename to test/test_device_fineoffset.py index 01bf99de..83d1bf17 100644 --- a/test/test_ecowitt.py +++ b/test/test_device_fineoffset.py @@ -9,7 +9,7 @@ @pytest_twisted.inlineCallbacks @pytest.mark.http -def test_ecowitt_post(machinery, create_influxdb, reset_influxdb): +def test_device_ecowitt_post(machinery, create_influxdb, reset_influxdb): """ Submit single reading in ``x-www-form-urlencoded`` format to HTTP API and proof it is stored in the InfluxDB database. @@ -78,11 +78,26 @@ def test_ecowitt_post(machinery, create_influxdb, reset_influxdb): # Proof that data arrived in InfluxDB. record = influx_sensors.get_first_record() - assert record["tempf"] == 48.4 + # Standard values, converted to "metric" unit system. + # Temperature converted from 48.4 degrees Fahrenheit, wind speed converted + # from 1.12 mph, humidity untouched. + assert record["temp"] == 9.11111111111111 assert record["humidity"] == 80.0 - assert record["model"] == "HP1000SE-PRO_Pro_V1.8.5" + assert record["windspeed"] == 1.8024652800000003 - # Make sure this will not be public. + # Verify the data includes additional computed fields. + assert record["dewpoint"] == 5.846942096976985 + assert record["feelslike"] == 9.11111111111111 + assert record["frostpoint"] == 4.706401162443284 + assert record["frostrisk"] == "No risk" + assert record["thermalperception"] == "Dry" + + # Make sure those fields got purged, so they don't leak into public data. assert "PASSKEY" not in record + assert "stationtype" not in record + assert "model" not in record + + # Timestamp field also gets removed, probably to avoid ambiguities. + assert "dateutc" not in record yield record