From 4a7cfa13a857960c1e7849d45712db430a17e0f4 Mon Sep 17 00:00:00 2001 From: Andrei Belov Date: Tue, 15 Oct 2024 17:12:18 +0400 Subject: [PATCH 1/9] Add example script for loading historical data --- examples/dump_data/dump_sensors.py | 1 + examples/dump_data/dump_timeseries.py | 33 +++ examples/load_data/.gitignore | 1 + examples/load_data/check_sorted.sh | 12 + examples/load_data/export_timeseries.py | 347 ++++++++++++++++++++++++ examples/load_data/get_monthly.sh | 36 +++ src/zont_api/zont_api.py | 24 ++ 7 files changed, 454 insertions(+) create mode 100755 examples/dump_data/dump_timeseries.py create mode 100644 examples/load_data/.gitignore create mode 100755 examples/load_data/check_sorted.sh create mode 100755 examples/load_data/export_timeseries.py create mode 100755 examples/load_data/get_monthly.sh diff --git a/examples/dump_data/dump_sensors.py b/examples/dump_data/dump_sensors.py index d1335da..450808f 100755 --- a/examples/dump_data/dump_sensors.py +++ b/examples/dump_data/dump_sensors.py @@ -15,6 +15,7 @@ def main(): sensors_data.extend(device.get_analog_inputs()) sensors_data.extend(device.get_analog_temperature_sensors()) sensors_data.extend(device.get_boiler_adapters()) + sensors_data.extend(device.get_heating_circuits()) print(json.dumps(sensors_data, ensure_ascii=False)) diff --git a/examples/dump_data/dump_timeseries.py b/examples/dump_data/dump_timeseries.py new file mode 100755 index 0000000..1a43a5b --- /dev/null +++ b/examples/dump_data/dump_timeseries.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 + +import sys +import json +from datetime import datetime, timedelta +from zont_api import ZontAPI + + +def main(): + zapi = ZontAPI() + devices = zapi.get_devices() + + dt_to = datetime.now() + dt_from = dt_to - timedelta(minutes=5) + + data_types = [ + "z3k_temperature", + "z3k_heating_circuit", + "z3k_boiler_adapter", + "z3k_analog_input", + ] + + data = [] + + for device in devices: + interval = (dt_from.timestamp(), dt_to.timestamp()) + data.append(zapi.load_data(device.id, data_types=data_types, interval=interval)) + + print(json.dumps(data, ensure_ascii=False)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/load_data/.gitignore b/examples/load_data/.gitignore new file mode 100644 index 0000000..8fce603 --- /dev/null +++ b/examples/load_data/.gitignore @@ -0,0 +1 @@ +data/ diff --git a/examples/load_data/check_sorted.sh b/examples/load_data/check_sorted.sh new file mode 100755 index 0000000..3dd7bee --- /dev/null +++ b/examples/load_data/check_sorted.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +if [ ! -d "data" ]; then + exit 0 +fi + +trap "rm -f tmp.$$" EXIT + +for f in $(find data/ -type f -name "*.csv"); do + cat $f | sort -k1 -n -t ',' >tmp.$$ + diff -u $f tmp.$$ +done diff --git a/examples/load_data/export_timeseries.py b/examples/load_data/export_timeseries.py new file mode 100755 index 0000000..3e0b97f --- /dev/null +++ b/examples/load_data/export_timeseries.py @@ -0,0 +1,347 @@ +#!/usr/bin/env python3 +""" +Export timeseries data from Zont servers to local csv files. +""" + +import os +import sys +import logging +import argparse +from datetime import datetime, timedelta +import csv +from http.client import HTTPConnection + +from zont_api import ZontAPI, ZontDevice + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", +) + +logger = logging.getLogger(os.path.basename(__file__)) + +zont_api_log = logging.getLogger("zont_api") +zont_api_log.setLevel(logging.INFO) + + +class MetricStats: + """ + Helper class to count unique metric series. + """ + + def __init__(self): + self.empty = True + self.oldest_ts = datetime.now().timestamp() + self.newest_ts = 0 + self.stats = {} + + def __str__(self): + if self.newest_ts > 0: + return ( + f"{self.total_metrics()} metrics, {self.total_values()} values, " + f"oldest @ {self.oldest_ts} {datetime.fromtimestamp(self.oldest_ts)}, " + f"newest @ {self.newest_ts} {datetime.fromtimestamp(self.newest_ts)}" + ) + return f"{self.total_metrics()} metrics, {self.total_values()} values" + + def update(self, metric_name, series_count, oldest_ts=None, newest_ts=None): + """ + Update stats counters + range (optional). + """ + self.empty = False + + if oldest_ts is not None: + self.oldest_ts = min(self.oldest_ts, oldest_ts) + + if newest_ts is not None: + self.newest_ts = max(self.newest_ts, newest_ts) + + if metric_name not in self.stats: + self.stats[metric_name] = series_count + return + self.stats[metric_name] += series_count + + def total_metrics(self): + """ + Get total number of unique metric names. + """ + return len(self.stats.keys()) + + def total_values(self): + """ + Get total sum of all values (metric series). + """ + values = 0 + for _, series_count in self.stats.items(): + values += series_count + return values + + +global_stats = MetricStats() + + +def save_csv(metric_name, arr, targetdir="."): + """ + Create csv file out of timeseries array for a given metric. + """ + result = [] + + for e in arr: + result.append({"timestamp": e[0], metric_name: e[1]}) + + output_file = f"{targetdir}/{metric_name}.csv" + logger.debug( + "saving %d entries for %s to %s", len(result), metric_name, output_file + ) + + os.makedirs(targetdir, exist_ok=True) + + with open(output_file, "w", encoding="utf-8") as csvout: + fieldnames = ["timestamp", metric_name] + writer = csv.DictWriter(csvout, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(result) + + +def datetime_str_to_ts(datetime_str: str): + """ + Convert datetime string to UNIX timestamp. + """ + for format_str in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d"): + try: + return datetime.strptime(datetime_str, format_str) + except ValueError: + continue + raise ValueError(f'could not get timestamp from "{datetime_str}"') + + +def export_data( + dt_from: datetime, dt_to: datetime, zapi: ZontAPI, device: ZontDevice, args +): + """ + Fetch timeseries data for a given device, do all required conversions, + and save csv file for every exported metric series. + """ + iteration_stats = MetricStats() + + targetdir = args.targetdir + if args.period == "hourly": + targetdir = dt_from.strftime(f"{args.targetdir}/%Y/%Y-%m/%Y-%m-%d/%H") + elif args.period == "daily": + targetdir = dt_from.strftime(f"{args.targetdir}/%Y/%Y-%m/%Y-%m-%d") + + data_types = [ + "z3k_analog_input", + "z3k_temperature", + "z3k_boiler_adapter", + "z3k_heating_circuit", + ] + + interval = (dt_from.timestamp(), dt_to.timestamp()) + data = zapi.load_data(device.id, data_types=data_types, interval=interval) + + # analog inputs like power controller + # for analog_input in device.get_analog_inputs(): + # analog_input_id = str(analog_input.get("id")) + for analog_input_id in data.get("z3k_analog_input").keys(): + subtree = data.get("z3k_analog_input").get(analog_input_id, {}) + for k, v in subtree.items(): + if len(v) == 0: + continue + metric = f"device_{device.id}.analog_input_{analog_input_id}.{k}" + timeseries = zapi.convert_delta_time_array(subtree.get(k)) + global_stats.update( + metric, + len(timeseries), + oldest_ts=timeseries[0][0], + newest_ts=timeseries[-1][0], + ) + iteration_stats.update(metric, len(timeseries)) + save_csv(metric, timeseries, targetdir=targetdir) + + # analog temperature sensors (wired) + # for analog_temperature_sensor in device.get_analog_temperature_sensors(): + # analog_temperature_sensor_id = str(analog_temperature_sensor.get("id")) + for analog_temperature_sensor_id in data.get("z3k_temperature").keys(): + dta = data.get("z3k_temperature").get(analog_temperature_sensor_id, []) + if len(dta) == 0: + continue + metric = f"device_{device.id}.analog_temperature_sensor_{analog_temperature_sensor_id}" + timeseries = zapi.convert_delta_time_array(dta) + global_stats.update( + metric, + len(timeseries), + oldest_ts=timeseries[0][0], + newest_ts=timeseries[-1][0], + ) + iteration_stats.update(metric, len(timeseries)) + save_csv(metric, timeseries, targetdir=targetdir) + + # boiler adapters + # for boiler_adapter in device.get_boiler_adapters(): + # boiler_adapter_id = str(boiler_adapter.get("id")) + for boiler_adapter_id in data.get("z3k_boiler_adapter").keys(): + subtree = data.get("z3k_boiler_adapter").get(boiler_adapter_id, {}) + for k, v in subtree.items(): + # if k == "s": + # continue + if len(v) == 0: + continue + metric = f"device_{device.id}.boiler_adapter_{boiler_adapter_id}.{k}" + timeseries = zapi.convert_delta_time_array(subtree.get(k)) + global_stats.update( + metric, + len(timeseries), + oldest_ts=timeseries[0][0], + newest_ts=timeseries[-1][0], + ) + iteration_stats.update(metric, len(timeseries)) + save_csv(metric, timeseries, targetdir=targetdir) + + # heating circuits + # for heating_circuit in device.get_heating_circuits(): + # heating_circuit_id = str(heating_circuit.get("id")) + for heating_circuit_id in data.get("z3k_heating_circuit").keys(): + subtree = data.get("z3k_heating_circuit").get(heating_circuit_id, {}) + for k, v in subtree.items(): + if len(v) == 0: + continue + metric = f"device_{device.id}.heating_circuit_{heating_circuit_id}.{k}" + timeseries = zapi.convert_delta_time_array(subtree.get(k)) + global_stats.update( + metric, + len(timeseries), + oldest_ts=timeseries[0][0], + newest_ts=timeseries[-1][0], + ) + iteration_stats.update(metric, len(timeseries)) + save_csv(metric, timeseries, targetdir=targetdir) + + logger.info( + "range %s to %s (period=%s): %s", + dt_from, + dt_to, + args.period, + iteration_stats, + ) + + return 0 + + +def main(): + """ + Entrypoint. + """ + parser = argparse.ArgumentParser( + description="Export timeseries data from Zont servers to local csv files", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "--from", + type=str, + dest="from_datetime", + required=True, + help="starting timestamp", + ) + parser.add_argument( + "--to", type=str, dest="to_datetime", required=True, help="snding timestamp" + ) + parser.add_argument( + "--exclusive-to", + action="store_true", + default=True, + help="exclude 1 second from ending timestamp", + ) + parser.add_argument( + "--period", + type=str, + default=None, + choices=("hourly", "daily"), + help="split output to periods (dedicated directories will be created under TARGETDIR)", + ) + parser.add_argument( + "--targetdir", + type=str, + default="data", + help="target directory for exported data", + ) + parser.add_argument( + "--verbose", + action="store_true", + default=False, + help="enable extra debugging output", + ) + args = parser.parse_args() + + if args.verbose: + logger.setLevel(logging.DEBUG) + zont_api_log.setLevel(logging.DEBUG) + urllib3_log = logging.getLogger("urllib3") + urllib3_log.setLevel(logging.DEBUG) + HTTPConnection.debuglevel = 1 + + dt_from = datetime_str_to_ts(args.from_datetime) + dt_to = datetime_str_to_ts(args.to_datetime) + dt_delta = dt_to - dt_from + if args.exclusive_to: + dt_to -= timedelta(seconds=1) + + logger.info( + "started with dt_from=%s, dt_to=%s, dt_delta=%s", + dt_from, + dt_to, + dt_delta, + ) + + zapi = ZontAPI() + devices = zapi.get_devices() + device = devices[0] + + if args.period is None: + export_data(dt_from, dt_to, zapi, device, args) + + elif args.period == "hourly": + if dt_delta < timedelta(hours=1): + raise ValueError( + f"can not split to hours as dt_delta < 1 hour ({dt_delta})" + ) + + dt_ifrom = dt_from + while True: + dt_ito = dt_ifrom + timedelta(hours=1) - timedelta(seconds=1) + export_data(dt_ifrom, dt_ito, zapi, device, args) + dt_ifrom = dt_ito + timedelta(seconds=1) + dt_ito = dt_ifrom + timedelta(hours=1) - timedelta(seconds=1) + if dt_ito > dt_to: + break + + elif args.period == "daily": + if dt_delta < timedelta(hours=24): + raise ValueError( + f"can not split to days as dt_delta < 24 hours ({dt_delta})" + ) + + dt_ifrom = dt_from + while True: + dt_ito = dt_ifrom + timedelta(hours=24) - timedelta(seconds=1) + export_data(dt_ifrom, dt_ito, zapi, device, args) + dt_ifrom = dt_ito + timedelta(seconds=1) + dt_ito = dt_ifrom + timedelta(hours=24) - timedelta(seconds=1) + if dt_ito > dt_to: + break + + if not global_stats.empty: + logger.info("========[ summary of collected metrics follows ]========") + for k, v in iter(sorted(global_stats.stats.items())): + logger.info("%s: %d", k, v) + logger.info("========[ %s ]========", global_stats) + else: + logger.info("no timeseries collected for a given period") + + logger.info("stopped") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/load_data/get_monthly.sh b/examples/load_data/get_monthly.sh new file mode 100755 index 0000000..b2b2cdc --- /dev/null +++ b/examples/load_data/get_monthly.sh @@ -0,0 +1,36 @@ +#!/bin/bash -x + +TARGET_DIR="${HOME}/home-net/baxi-connect" + +# ./export_timeseries.py --from=2023-07-01 --to=2023-08-01 --verbose --targetdir=${HOME}/home-net/baxi-connect/2023-07 >${HOME}/home-net/baxi-connect/2023-07/export.out 2>&1 + +START_YEAR=2023 +START_MONTH=11 +END_YEAR=2024 +END_MONTH=9 + +YEAR=${START_YEAR} +MONTH=${START_MONTH} + +while true; do + printf -v DATESTR_FROM "%04d-%02d-01" "${YEAR}" "${MONTH}" + printf -v TARGET_SUBDIR "%04d-%02d" "${YEAR}" "${MONTH}" + + MONTH=$((MONTH + 1)) + if ((MONTH > 12)); then + MONTH=1 + YEAR=$((YEAR + 1)) + fi + + printf -v DATESTR_TO "%04d-%02d-01" "${YEAR}" "$((MONTH))" + + mkdir -p "${TARGET_DIR}/${TARGET_SUBDIR}" + ./export_timeseries.py --from="${DATESTR_FROM}" --to="${DATESTR_TO}" --targetdir="${TARGET_DIR}/${TARGET_SUBDIR}" >"${TARGET_DIR}/${TARGET_SUBDIR}/export.out" 2>&1 + + if ((MONTH > END_MONTH)) && ((YEAR >= END_YEAR)); then + break + fi + + echo "Press any key to continue or Ctrl+C to exit..." >&2 + read -r +done diff --git a/src/zont_api/zont_api.py b/src/zont_api/zont_api.py index 3f5faf3..aa7ead1 100644 --- a/src/zont_api/zont_api.py +++ b/src/zont_api/zont_api.py @@ -484,6 +484,30 @@ def get_boiler_adapters(self): ) return elements + def get_heating_circuits(self): + """ + Get list of heating circuits available on a given device + + :return: [] of heating circuits + """ + source = self.data.get("z3k_config").get("heating_circuits") + + if not isinstance(source, list): + return None + + elements = [] + for element in source: + elements.append( + { + "device_id": self.id, + "id": element.get("id"), + "family": "heating_circuits", + "name": element.get("name"), + "type": element.get("type"), + } + ) + return elements + def get_sensor_name(self, sensor_id: int) -> str: """ Obtain sensor name by its ID From a5b38174a80ef0ea7dab78e7ed098c6145901ec6 Mon Sep 17 00:00:00 2001 From: Andrei Belov Date: Tue, 15 Oct 2024 17:18:12 +0400 Subject: [PATCH 2/9] Use semicolon as csv delimiter Some series from sensors produce non-numerical data in DTA, and this change would still allow to keep those in csv files for future analysis. --- examples/load_data/export_timeseries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/load_data/export_timeseries.py b/examples/load_data/export_timeseries.py index 3e0b97f..880fb1a 100755 --- a/examples/load_data/export_timeseries.py +++ b/examples/load_data/export_timeseries.py @@ -98,7 +98,7 @@ def save_csv(metric_name, arr, targetdir="."): with open(output_file, "w", encoding="utf-8") as csvout: fieldnames = ["timestamp", metric_name] - writer = csv.DictWriter(csvout, fieldnames=fieldnames) + writer = csv.DictWriter(csvout, fieldnames=fieldnames, delimiter=";") writer.writeheader() writer.writerows(result) From 0673a3c9173dc711e992a2623f0196153c4c015f Mon Sep 17 00:00:00 2001 From: Andrei Belov Date: Tue, 15 Oct 2024 18:11:00 +0400 Subject: [PATCH 3/9] tests: add DTA conversion with duplicate datapoints --- tests/zont_api.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/zont_api.py b/tests/zont_api.py index 8c55189..b290618 100644 --- a/tests/zont_api.py +++ b/tests/zont_api.py @@ -174,6 +174,38 @@ def test_convert_dta_ignore_zero_deltas(): ] +def test_convert_dta_with_duplicate_datapoints(): + """ + Convert data time array with duplicate datapoints (timestamp + value) + + TODO: as Zont servers evidently emit duplicate datapoints sometimes, + it would be good to have a logic to filter those. + """ + source_dta = [ + [1000, 1], + [-10, 2], + [-10, 3], + [1030, 666], + [1030, 666], + [1030, 666], + [-10, 4], + [-10, 5], + ] + + result_dta = ZontAPI.convert_delta_time_array(ZontAPI, source_dta) + + assert result_dta == [ + [1000, 1], + [1010, 2], + [1020, 3], + [1030, 666], + [1030, 666], + [1030, 666], + [1040, 4], + [1050, 5], + ] + + def test_convert_dta_combined_payload(): """ Convert data time array with multiple payload values per element From fd99d660a9e4f6e5b0ad4c679e1f10e1bd1547af Mon Sep 17 00:00:00 2001 From: Andrei Belov Date: Tue, 15 Oct 2024 18:41:22 +0400 Subject: [PATCH 4/9] feat: add a way to filter duplicate datapoints --- src/zont_api/zont_api.py | 19 +++++++++++++++-- tests/zont_api.py | 44 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 58 insertions(+), 5 deletions(-) diff --git a/src/zont_api/zont_api.py b/src/zont_api/zont_api.py index aa7ead1..c0e968f 100644 --- a/src/zont_api/zont_api.py +++ b/src/zont_api/zont_api.py @@ -325,7 +325,9 @@ def load_data( return responses[0] - def convert_delta_time_array(self, delta_time_array, sort=True, reverse=False): + def convert_delta_time_array( + self, delta_time_array, sort=True, reverse=False, filter_duplicates=False + ): """ Converts delta time array to another array with absolute timestamps instead of relative ones @@ -334,6 +336,7 @@ def convert_delta_time_array(self, delta_time_array, sort=True, reverse=False): (https://zont-online.ru/api/docs/?python#delta-time-array) :param sort: bool - sort results by absolute timestamp :param reverse: bool - reverse sort order + :param filter_duplicates: bool - filter out duplicate datapoints :return: [] with absolute timestamps """ @@ -343,7 +346,7 @@ def convert_delta_time_array(self, delta_time_array, sort=True, reverse=False): ) result = [] - latest_stamp, curr_stamp = 0, 0 + latest_stamp, curr_stamp, element_count = 0, 0, 0 for element in delta_time_array: if not isinstance(element, list): @@ -366,7 +369,19 @@ def convert_delta_time_array(self, delta_time_array, sort=True, reverse=False): # we do not expect zero value here continue + # filter out duplicate datapoint(s) if required + if filter_duplicates and element_count > 0: + if result[-1][0] == curr_stamp: + logger.debug( + 'duplicate datapoint detected at %d: "%s" vs previous "%s"', + curr_stamp, + element[1:], + result[-1][1:], + ) + continue + result.append([curr_stamp] + element[1:]) + element_count += 1 if sort: return sorted(result, key=lambda t: t[0], reverse=reverse) diff --git a/tests/zont_api.py b/tests/zont_api.py index b290618..c6b4f06 100644 --- a/tests/zont_api.py +++ b/tests/zont_api.py @@ -3,6 +3,7 @@ """ import os +import logging import pytest from zont_api import ZontAPIException, ZontAPI @@ -12,6 +13,10 @@ __copyright__ = f"Copyright (c) {__author__}" +# zont_api_log = logging.getLogger("zont_api") +# zont_api_log.setLevel(logging.INFO) + + def test_init_api_without_params(monkeypatch): """ Initialization without token or client @@ -177,9 +182,7 @@ def test_convert_dta_ignore_zero_deltas(): def test_convert_dta_with_duplicate_datapoints(): """ Convert data time array with duplicate datapoints (timestamp + value) - - TODO: as Zont servers evidently emit duplicate datapoints sometimes, - it would be good to have a logic to filter those. + without filtering """ source_dta = [ [1000, 1], @@ -206,6 +209,41 @@ def test_convert_dta_with_duplicate_datapoints(): ] +def test_convert_dta_with_duplicate_datapoints_filter(caplog): + """ + Convert data time array with duplicate datapoints (timestamp + value) + with filtering out all subsequent duplicates with the same timestamp + without checking a value. + + Zont servers evidently emit duplicate datapoints sometimes, + so having a knob to filter those out could be handy. + """ + caplog.set_level(logging.DEBUG) + source_dta = [ + [1000, 1], + [-10, 2], + [-10, 3], + [1030, 666], + [1030, 667], + [1030, 668], + [-10, 4], + [-10, 5], + ] + + result_dta = ZontAPI.convert_delta_time_array( + ZontAPI, source_dta, filter_duplicates=True + ) + + assert result_dta == [ + [1000, 1], + [1010, 2], + [1020, 3], + [1030, 666], + [1040, 4], + [1050, 5], + ] + + def test_convert_dta_combined_payload(): """ Convert data time array with multiple payload values per element From 93651070dd1f82b50cdeb09841f98d78ad12d977 Mon Sep 17 00:00:00 2001 From: Andrei Belov Date: Tue, 15 Oct 2024 19:35:06 +0400 Subject: [PATCH 5/9] export_timeseries: add option to filter duplicate datapoints --- examples/load_data/export_timeseries.py | 22 ++++++++++++++++++---- examples/load_data/get_monthly.sh | 2 +- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/examples/load_data/export_timeseries.py b/examples/load_data/export_timeseries.py index 880fb1a..4c18426 100755 --- a/examples/load_data/export_timeseries.py +++ b/examples/load_data/export_timeseries.py @@ -149,7 +149,9 @@ def export_data( if len(v) == 0: continue metric = f"device_{device.id}.analog_input_{analog_input_id}.{k}" - timeseries = zapi.convert_delta_time_array(subtree.get(k)) + timeseries = zapi.convert_delta_time_array( + subtree.get(k), filter_duplicates=args.filter_duplicates + ) global_stats.update( metric, len(timeseries), @@ -167,7 +169,9 @@ def export_data( if len(dta) == 0: continue metric = f"device_{device.id}.analog_temperature_sensor_{analog_temperature_sensor_id}" - timeseries = zapi.convert_delta_time_array(dta) + timeseries = zapi.convert_delta_time_array( + dta, filter_duplicates=args.filter_duplicates + ) global_stats.update( metric, len(timeseries), @@ -188,7 +192,9 @@ def export_data( if len(v) == 0: continue metric = f"device_{device.id}.boiler_adapter_{boiler_adapter_id}.{k}" - timeseries = zapi.convert_delta_time_array(subtree.get(k)) + timeseries = zapi.convert_delta_time_array( + subtree.get(k), filter_duplicates=args.filter_duplicates + ) global_stats.update( metric, len(timeseries), @@ -207,7 +213,9 @@ def export_data( if len(v) == 0: continue metric = f"device_{device.id}.heating_circuit_{heating_circuit_id}.{k}" - timeseries = zapi.convert_delta_time_array(subtree.get(k)) + timeseries = zapi.convert_delta_time_array( + subtree.get(k), filter_duplicates=args.filter_duplicates + ) global_stats.update( metric, len(timeseries), @@ -259,6 +267,12 @@ def main(): choices=("hourly", "daily"), help="split output to periods (dedicated directories will be created under TARGETDIR)", ) + parser.add_argument( + "--filter-duplicates", + action="store_true", + default=False, + help="filter duplicate datapoints", + ) parser.add_argument( "--targetdir", type=str, diff --git a/examples/load_data/get_monthly.sh b/examples/load_data/get_monthly.sh index b2b2cdc..a864c0f 100755 --- a/examples/load_data/get_monthly.sh +++ b/examples/load_data/get_monthly.sh @@ -25,7 +25,7 @@ while true; do printf -v DATESTR_TO "%04d-%02d-01" "${YEAR}" "$((MONTH))" mkdir -p "${TARGET_DIR}/${TARGET_SUBDIR}" - ./export_timeseries.py --from="${DATESTR_FROM}" --to="${DATESTR_TO}" --targetdir="${TARGET_DIR}/${TARGET_SUBDIR}" >"${TARGET_DIR}/${TARGET_SUBDIR}/export.out" 2>&1 + ./export_timeseries.py --from="${DATESTR_FROM}" --to="${DATESTR_TO}" --filter-duplicates --targetdir="${TARGET_DIR}/${TARGET_SUBDIR}" >"${TARGET_DIR}/${TARGET_SUBDIR}/export.out" 2>&1 if ((MONTH > END_MONTH)) && ((YEAR >= END_YEAR)); then break From 892e9bd40eac0da3bc9b727c6c9e10d2329c81a2 Mon Sep 17 00:00:00 2001 From: Andrei Belov Date: Wed, 16 Oct 2024 11:32:34 +0400 Subject: [PATCH 6/9] tests: add basic load_data tests --- tests/zont_api_load_data.py | 185 ++++++++++++++++++++++++++++++++++++ 1 file changed, 185 insertions(+) create mode 100644 tests/zont_api_load_data.py diff --git a/tests/zont_api_load_data.py b/tests/zont_api_load_data.py new file mode 100644 index 0000000..0e15b0d --- /dev/null +++ b/tests/zont_api_load_data.py @@ -0,0 +1,185 @@ +""" +Tests for Zont load_data API +""" + +import logging +from unittest.mock import patch, MagicMock +from zont_api import ZontAPI + +from tests.zont_device import MOCK_API_RESPONSE_SINGLE_DEVICE + +__author__ = "Andrei Belov" +__license__ = "MIT" +__copyright__ = f"Copyright (c) {__author__}" + + +DATA_TYPES_ALL = [ + "z3k_temperature", + "z3k_heating_circuit", + "z3k_boiler_adapter", + "z3k_analog_input", +] + +MOCK_DATA_EMPTY_RESPONSE = { + "ok": True, + "responses": [ + { + "device_id": 42, + "ok": True, + "time_truncated": False, + "z3k_temperature": {}, + "z3k_heating_circuit": {}, + "z3k_boiler_adapter": {}, + "z3k_analog_input": {}, + } + ], +} + +MOCK_DATA_RESPONSE_WITH_METRICS = { + "ok": True, + "responses": [ + { + "device_id": 42, + "ok": True, + "time_truncated": False, + "z3k_temperature": { + "4242": [ + [1000, 42.2], + [-60, 42.2], + [-60, 42.2], + ] + }, + "z3k_heating_circuit": { + "4343": { + "worktime": [ + [1000, 0], + [-60, 0], + [-60, 0], + ], + "status": [ + [1000, 0], + [-120, 0], + ], + } + }, + "z3k_boiler_adapter": { + "4444": { + "s": [ + [1000, []], + [-60, []], + [-60, []], + ], + "ot": [ + [1000, 7.0], + [-60, 7.0], + [-60, 6.0], + ], + } + }, + "z3k_analog_input": { + "4545": { + "voltage": [ + [1000, 23.8], + [-120, 23.8], + ], + "value": [ + [1000, 238], + [-120, 238], + ], + } + }, + "timings": { + "z3k_temperature": { + "wall": 0.05566287040710449, + "proc": 0.002107100000102946, + }, + "z3k_boiler_adapter": { + "wall": 0.08274435997009277, + "proc": 0.002168881000216061, + }, + "z3k_heating_circuit": { + "wall": 0.05684089660644531, + "proc": 0.002351291000195488, + }, + "z3k_analog_input": { + "wall": 0.08126425743103027, + "proc": 0.0019758700000238605, + }, + }, + } + ], +} + + +@patch("zont_api.zont_api.requests") +def test_load_data_no_series(mock_requests, caplog): + """ + Load data with valid response but no serires + """ + caplog.set_level(logging.DEBUG) + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = MOCK_API_RESPONSE_SINGLE_DEVICE + mock_requests.post.return_value = mock_response + + zapi = ZontAPI(token="testtoken", client="testclient") + assert isinstance(zapi, ZontAPI) + devices = zapi.get_devices() + assert len(devices) == 1 + assert devices[0].id == 42 + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = MOCK_DATA_EMPTY_RESPONSE + mock_requests.post.return_value = mock_response + + data = zapi.load_data( + devices[0].id, data_types=DATA_TYPES_ALL, interval=(1000, 1120) + ) + + assert data.get("device_id") == 42 + for key_name in DATA_TYPES_ALL: + assert key_name in data.keys(), f"{key_name} present in load_data response" + assert isinstance(data.get(key_name), dict), f"{key_name} is dict" + assert bool(data.get(key_name)) is False, f"{key_name} dict is empty" + + # repeat the same request without explicit data_types and interval + data = zapi.load_data(devices[0].id) + assert data.get("device_id") == 42 + + +@patch("zont_api.zont_api.requests") +def test_load_data_with_metrics(mock_requests, caplog): + """ + Load data with valid response and some metrics + """ + caplog.set_level(logging.DEBUG) + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = MOCK_API_RESPONSE_SINGLE_DEVICE + mock_requests.post.return_value = mock_response + + zapi = ZontAPI(token="testtoken", client="testclient") + assert isinstance(zapi, ZontAPI) + devices = zapi.get_devices() + assert len(devices) == 1 + assert devices[0].id == 42 + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = MOCK_DATA_RESPONSE_WITH_METRICS + mock_requests.post.return_value = mock_response + + data = zapi.load_data( + devices[0].id, data_types=DATA_TYPES_ALL, interval=(1000, 1120) + ) + + assert data.get("device_id") == 42 + for key_name in DATA_TYPES_ALL: + assert key_name in data.keys(), f"{key_name} present in load_data response" + assert isinstance(data.get(key_name), dict), f"{key_name} is dict" + assert bool(data.get(key_name)) is True, f"{key_name} dict is not empty" + + assert "timings" not in data.keys(), "timings section was removed" From b6ae5cb850b9b99fd52922e5e4c83e57a10de95e Mon Sep 17 00:00:00 2001 From: Andrei Belov Date: Wed, 16 Oct 2024 11:43:27 +0400 Subject: [PATCH 7/9] tests: removed unnecessary duplications --- tests/zont_api_load_data.py | 39 +++++++++---------------------------- 1 file changed, 9 insertions(+), 30 deletions(-) diff --git a/tests/zont_api_load_data.py b/tests/zont_api_load_data.py index 0e15b0d..f3995f4 100644 --- a/tests/zont_api_load_data.py +++ b/tests/zont_api_load_data.py @@ -4,9 +4,8 @@ import logging from unittest.mock import patch, MagicMock -from zont_api import ZontAPI +from zont_api import ZontAPI, ZontDevice -from tests.zont_device import MOCK_API_RESPONSE_SINGLE_DEVICE __author__ = "Andrei Belov" __license__ = "MIT" @@ -118,35 +117,25 @@ def test_load_data_no_series(mock_requests, caplog): """ caplog.set_level(logging.DEBUG) - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = MOCK_API_RESPONSE_SINGLE_DEVICE - mock_requests.post.return_value = mock_response - zapi = ZontAPI(token="testtoken", client="testclient") - assert isinstance(zapi, ZontAPI) - devices = zapi.get_devices() - assert len(devices) == 1 - assert devices[0].id == 42 + zdev = ZontDevice(device_data={"id": 42, "name": "testdevice"}) mock_response = MagicMock() mock_response.status_code = 200 mock_response.json.return_value = MOCK_DATA_EMPTY_RESPONSE mock_requests.post.return_value = mock_response - data = zapi.load_data( - devices[0].id, data_types=DATA_TYPES_ALL, interval=(1000, 1120) - ) + data = zapi.load_data(zdev.id, data_types=DATA_TYPES_ALL, interval=(1000, 1120)) - assert data.get("device_id") == 42 + assert data.get("device_id") == zdev.id for key_name in DATA_TYPES_ALL: assert key_name in data.keys(), f"{key_name} present in load_data response" assert isinstance(data.get(key_name), dict), f"{key_name} is dict" assert bool(data.get(key_name)) is False, f"{key_name} dict is empty" # repeat the same request without explicit data_types and interval - data = zapi.load_data(devices[0].id) - assert data.get("device_id") == 42 + data = zapi.load_data(zdev.id) + assert data.get("device_id") == zdev.id @patch("zont_api.zont_api.requests") @@ -156,27 +145,17 @@ def test_load_data_with_metrics(mock_requests, caplog): """ caplog.set_level(logging.DEBUG) - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = MOCK_API_RESPONSE_SINGLE_DEVICE - mock_requests.post.return_value = mock_response - zapi = ZontAPI(token="testtoken", client="testclient") - assert isinstance(zapi, ZontAPI) - devices = zapi.get_devices() - assert len(devices) == 1 - assert devices[0].id == 42 + zdev = ZontDevice(device_data={"id": 42, "name": "testdevice"}) mock_response = MagicMock() mock_response.status_code = 200 mock_response.json.return_value = MOCK_DATA_RESPONSE_WITH_METRICS mock_requests.post.return_value = mock_response - data = zapi.load_data( - devices[0].id, data_types=DATA_TYPES_ALL, interval=(1000, 1120) - ) + data = zapi.load_data(zdev.id, data_types=DATA_TYPES_ALL, interval=(1000, 1120)) - assert data.get("device_id") == 42 + assert data.get("device_id") == zdev.id for key_name in DATA_TYPES_ALL: assert key_name in data.keys(), f"{key_name} present in load_data response" assert isinstance(data.get(key_name), dict), f"{key_name} is dict" From 1f02587923209304f31a928528bb331b80d29f68 Mon Sep 17 00:00:00 2001 From: Andrei Belov Date: Wed, 16 Oct 2024 11:50:15 +0400 Subject: [PATCH 8/9] tests: cleanup --- tests/zont_api.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/zont_api.py b/tests/zont_api.py index c6b4f06..e96d771 100644 --- a/tests/zont_api.py +++ b/tests/zont_api.py @@ -13,10 +13,6 @@ __copyright__ = f"Copyright (c) {__author__}" -# zont_api_log = logging.getLogger("zont_api") -# zont_api_log.setLevel(logging.INFO) - - def test_init_api_without_params(monkeypatch): """ Initialization without token or client From 171529c4b7abe911639753ef907589183b178c48 Mon Sep 17 00:00:00 2001 From: Andrei Belov Date: Wed, 16 Oct 2024 12:32:37 +0400 Subject: [PATCH 9/9] Use module as a single source of truth on default Z3K data types --- examples/dump_data/dump_timeseries.py | 13 ++++--------- examples/load_data/export_timeseries.py | 11 ++--------- src/zont_api/__init__.py | 2 +- src/zont_api/zont_api.py | 17 ++++++++++------- tests/zont_api_load_data.py | 17 +++++------------ 5 files changed, 22 insertions(+), 38 deletions(-) diff --git a/examples/dump_data/dump_timeseries.py b/examples/dump_data/dump_timeseries.py index 1a43a5b..b905d51 100755 --- a/examples/dump_data/dump_timeseries.py +++ b/examples/dump_data/dump_timeseries.py @@ -3,7 +3,7 @@ import sys import json from datetime import datetime, timedelta -from zont_api import ZontAPI +from zont_api import ZontAPI, DATA_TYPES_Z3K def main(): @@ -13,18 +13,13 @@ def main(): dt_to = datetime.now() dt_from = dt_to - timedelta(minutes=5) - data_types = [ - "z3k_temperature", - "z3k_heating_circuit", - "z3k_boiler_adapter", - "z3k_analog_input", - ] - data = [] for device in devices: interval = (dt_from.timestamp(), dt_to.timestamp()) - data.append(zapi.load_data(device.id, data_types=data_types, interval=interval)) + data.append( + zapi.load_data(device.id, data_types=DATA_TYPES_Z3K, interval=interval) + ) print(json.dumps(data, ensure_ascii=False)) diff --git a/examples/load_data/export_timeseries.py b/examples/load_data/export_timeseries.py index 4c18426..ca7b482 100755 --- a/examples/load_data/export_timeseries.py +++ b/examples/load_data/export_timeseries.py @@ -11,7 +11,7 @@ import csv from http.client import HTTPConnection -from zont_api import ZontAPI, ZontDevice +from zont_api import ZontAPI, ZontDevice, DATA_TYPES_Z3K logging.basicConfig( level=logging.INFO, @@ -130,15 +130,8 @@ def export_data( elif args.period == "daily": targetdir = dt_from.strftime(f"{args.targetdir}/%Y/%Y-%m/%Y-%m-%d") - data_types = [ - "z3k_analog_input", - "z3k_temperature", - "z3k_boiler_adapter", - "z3k_heating_circuit", - ] - interval = (dt_from.timestamp(), dt_to.timestamp()) - data = zapi.load_data(device.id, data_types=data_types, interval=interval) + data = zapi.load_data(device.id, data_types=DATA_TYPES_Z3K, interval=interval) # analog inputs like power controller # for analog_input in device.get_analog_inputs(): diff --git a/src/zont_api/__init__.py b/src/zont_api/__init__.py index 9f59fef..52b6005 100644 --- a/src/zont_api/__init__.py +++ b/src/zont_api/__init__.py @@ -6,5 +6,5 @@ __license__ = "MIT" __copyright__ = f"Copyright (c) {__author__}" -from .zont_api import ZontAPIException, ZontAPI, ZontDevice +from .zont_api import ZontAPIException, ZontAPI, ZontDevice, DATA_TYPES_Z3K from .version import __version__, __release__, __build__ diff --git a/src/zont_api/zont_api.py b/src/zont_api/zont_api.py index c0e968f..044f169 100644 --- a/src/zont_api/zont_api.py +++ b/src/zont_api/zont_api.py @@ -22,6 +22,14 @@ logger = logging.getLogger(__name__) +DATA_TYPES_Z3K = [ + "z3k_analog_input", + "z3k_boiler_adapter", + "z3k_heating_circuit", + "z3k_temperature", +] + + class ZontAPIException(Exception): """ Base class for Zont API exception @@ -280,14 +288,9 @@ def load_data( now = int(time()) interval = (now - 60, now) - # if data types are not specified, use z3k subset + # if data types are not specified, use z3k subset as default if not data_types: - data_types = [ - "z3k_temperature", - "z3k_heating_circuit", - "z3k_boiler_adapter", - "z3k_analog_input", - ] + data_types = DATA_TYPES_Z3K data = { "requests": [ diff --git a/tests/zont_api_load_data.py b/tests/zont_api_load_data.py index f3995f4..2c0a754 100644 --- a/tests/zont_api_load_data.py +++ b/tests/zont_api_load_data.py @@ -4,7 +4,7 @@ import logging from unittest.mock import patch, MagicMock -from zont_api import ZontAPI, ZontDevice +from zont_api import ZontAPI, ZontDevice, DATA_TYPES_Z3K __author__ = "Andrei Belov" @@ -12,13 +12,6 @@ __copyright__ = f"Copyright (c) {__author__}" -DATA_TYPES_ALL = [ - "z3k_temperature", - "z3k_heating_circuit", - "z3k_boiler_adapter", - "z3k_analog_input", -] - MOCK_DATA_EMPTY_RESPONSE = { "ok": True, "responses": [ @@ -125,10 +118,10 @@ def test_load_data_no_series(mock_requests, caplog): mock_response.json.return_value = MOCK_DATA_EMPTY_RESPONSE mock_requests.post.return_value = mock_response - data = zapi.load_data(zdev.id, data_types=DATA_TYPES_ALL, interval=(1000, 1120)) + data = zapi.load_data(zdev.id, data_types=DATA_TYPES_Z3K, interval=(1000, 1120)) assert data.get("device_id") == zdev.id - for key_name in DATA_TYPES_ALL: + for key_name in DATA_TYPES_Z3K: assert key_name in data.keys(), f"{key_name} present in load_data response" assert isinstance(data.get(key_name), dict), f"{key_name} is dict" assert bool(data.get(key_name)) is False, f"{key_name} dict is empty" @@ -153,10 +146,10 @@ def test_load_data_with_metrics(mock_requests, caplog): mock_response.json.return_value = MOCK_DATA_RESPONSE_WITH_METRICS mock_requests.post.return_value = mock_response - data = zapi.load_data(zdev.id, data_types=DATA_TYPES_ALL, interval=(1000, 1120)) + data = zapi.load_data(zdev.id, data_types=DATA_TYPES_Z3K, interval=(1000, 1120)) assert data.get("device_id") == zdev.id - for key_name in DATA_TYPES_ALL: + for key_name in DATA_TYPES_Z3K: assert key_name in data.keys(), f"{key_name} present in load_data response" assert isinstance(data.get(key_name), dict), f"{key_name} is dict" assert bool(data.get(key_name)) is True, f"{key_name} dict is not empty"