diff --git a/packages/control/chargelog/chargelog.py b/packages/control/chargelog/chargelog.py index 6468f55c8c..674159cdc9 100644 --- a/packages/control/chargelog/chargelog.py +++ b/packages/control/chargelog/chargelog.py @@ -14,6 +14,8 @@ from helpermodules.measurement_logging.process_log_calculation import ( FILE_ERRORS, CalculationType, _analyse_energy_source, _process_entries, get_totals) +from helpermodules.measurement_logging.process_log_entry_builder import create_entry + # alte Daten: Startzeitpunkt der Ladung, Endzeitpunkt, Geladene Reichweite, Energie, Leistung, Ladedauer, LP-Nummer, # Lademodus, ID-Tag # json-Objekt: new_entry = { @@ -328,7 +330,7 @@ def write_new_entry(new_entry): def calc_energy_costs(cp, create_log_entry: bool = False): try: if cp.data.set.log.imported_since_plugged != 0 and cp.data.set.log.imported_since_mode_switch != 0: - processed_entries, reference_entries = _get_reference_entries(cp) + processed_entries, reference_entries = _get_reference_entries(cp, create_log_entry) charged_energy_by_source = calculate_charged_energy_by_source( cp, processed_entries, reference_entries, create_log_entry) _add_charged_energy_by_source(cp, charged_energy_by_source) @@ -361,16 +363,18 @@ def calculate_charged_energy_by_source(cp, processed_entries, reference_entries, reference_entries[0]["cp"][f"cp{cp.num}"]["imported"]) elif reference == ReferenceTime.END: if sum(cp.data.set.log.charged_energy_by_source.values()) == 0: + # Kurze Ladung, für die noch kein 5min-Intervall vergangen ist charged_energy = cp.data.set.log.imported_since_mode_switch else: - log.debug(f"cp.data.get.imported {cp.data.get.imported}") - charged_energy = cp.data.get.imported - \ - reference_entries[-1]["cp"][f"cp{cp.num}"]["imported"] + # Nur das noch nicht verbuchte Intervall + charged_energy = (reference_entries[-1]["cp"][f"cp{cp.num}"]["imported"] - + reference_entries[0]["cp"][f"cp{cp.num}"]["imported"]) else: raise TypeError(f"Unbekannter Referenz-Zeitpunkt {reference}") log.debug(f'power source {relative_energy_source}') log.debug(f"charged_energy {charged_energy}") - if charged_energy < 100: + # Wenn keine Energie geladen wurde, keine Anteile berechnen + if charged_energy <= 0: # wenn nur entladen wurde, keine Anteile berechnen return {source: 0 for source in ENERGY_SOURCES} return _get_charged_energy_by_source( @@ -402,14 +406,26 @@ def _get_reference_position(cp, create_log_entry: bool) -> ReferenceTime: return ReferenceTime.MIDDLE -def _get_reference_entries(cp) -> Tuple[List[Dict], List]: +def _get_reference_entries(cp, create_log_entry: bool = False) -> Tuple[List[Dict], List]: processed_entries = {} reference_entries = [] try: log_data = get_todays_daily_log() names = log_data["names"] entries = log_data["entries"] - if len(entries) >= 2: + if create_log_entry: + # beim beenden einer Ladung den aktuellen Messwert benutzen + if entries: + previous_entry = entries[-1] + else: + # Bei Tageswechsel + date_day_before = (datetime.datetime.now() + datetime.timedelta(days=-1)).strftime("%Y%m%d") + entries_day_before = get_daily_log(date_day_before)["entries"] + previous_entry = entries_day_before[-1] + + current_entry = create_entry(None, previous_entry) + reference_entries = [previous_entry, current_entry] + elif len(entries) >= 2: reference_entries = [entries[-2], entries[-1]] else: date_day_before = (datetime.datetime.now() + datetime.timedelta(days=-1)).strftime("%Y%m%d") diff --git a/packages/control/chargelog/chargelog_test.py b/packages/control/chargelog/chargelog_test.py index 6f4ce60afb..e28d11623f 100644 --- a/packages/control/chargelog/chargelog_test.py +++ b/packages/control/chargelog/chargelog_test.py @@ -9,6 +9,7 @@ from control.chargelog.chargelog import calc_energy_costs from control.chargepoint.chargepoint import Chargepoint from control.general import Prices +from helpermodules.measurement_logging import process_log_entry_builder @pytest.fixture() @@ -17,6 +18,19 @@ def mock_data() -> None: data.data.optional_data.et_module = None +@pytest.mark.parametrize("sh_values", [{}, {"sh1": {"imported": 1200, "exported": 0, "temp0": 21}}]) +def test_create_entry_optional_smarthome(mock_data, sh_values): + sh_log_data = Mock(sh_dict=sh_values) if sh_values else None + + with patch.object(process_log_entry_builder, "BrokerClient") as broker_client: + entry = process_log_entry_builder.create_entry(sh_log_data, None) + + assert entry["sh"] == sh_values + assert "cp" in entry + assert "prices" in entry + broker_client.assert_not_called() + + def mock_daily_log(monkeypatch): daily_log = {"entries": [{'bat': {'all': {'exported': 2000, 'imported': 2000, 'soc': 100}, 'bat2': {'exported': 2000, 'imported': 2000, 'soc': 100}}, @@ -92,14 +106,97 @@ def test_calc_charge_cost_reference_end(mock_data, monkeypatch): cp.data.get.imported = 4100 cp.data.set.log.charged_energy_by_source = {'grid': 1243, 'pv': 386, 'bat': 671, 'cp': 0.0} daily_log = mock_daily_log(monkeypatch) + current_entry = json.loads(json.dumps(daily_log["entries"][-1])) + current_entry["bat"]["all"]["exported"] = 4000 + current_entry["bat"]["bat2"]["exported"] = 4000 + current_entry["counter"]["counter0"]["imported"] = 4500 + current_entry["cp"]["cp4"]["imported"] = 4100 + current_entry["pv"]["all"]["exported"] = 3000 + current_entry["pv"]["pv1"]["exported"] = 3000 + create_entry_mock = Mock(return_value=current_entry) + monkeypatch.setattr(chargelog, "create_entry", create_entry_mock) - with patch("builtins.open", mock_open(read_data=json.dumps(daily_log))): + with patch("builtins.open", mock_open(read_data=json.dumps(daily_log))), \ + patch.object(process_log_entry_builder, "BrokerClient") as broker_client: calc_energy_costs(cp, True) + create_entry_mock.assert_called_once_with(None, daily_log["entries"][-1]) + broker_client.assert_not_called() assert cp.data.set.log.charged_energy_by_source == {'bat': 699.57, 'cp': 0.0, 'grid': 1300.14, 'pv': 400.29} assert round(cp.data.set.log.costs, 5) == 0.025 +def test_calc_charge_cost_short_charge_uses_current_entry(mock_data, monkeypatch): + cp = Chargepoint(4, None) + + # Kurzer Ladevorgang: insgesamt nur 80 Wh geladen. + # Es wurde vorher noch kein Energieanteil verbucht. + cp.data.set.log.imported_since_plugged = 80 + cp.data.set.log.imported_since_mode_switch = 80 + cp.data.set.log.timestamp_mode_switch = 1652683260 # 8:41 + cp.data.get.imported = 4080 + cp.data.set.log.charged_energy_by_source = { + 'bat': 0, + 'cp': 0, + 'grid': 0, + 'pv': 0 + } + + daily_log = mock_daily_log(monkeypatch) + + # Letzter gespeicherter Daily-Log ist 8:40: + # + # Grid imported = 2500 Wh + # PV exported = 2500 Wh + # CP imported = 4000 Wh + # + # Jetzt erzeugen wir den Live-Snapshot beim Ladeende um 8:42. + current_entry = json.loads(json.dumps(daily_log["entries"][-1])) + + current_entry["date"] = "8:42" + current_entry["timestamp"] = 1652683320 + + # Zwischen 8:40 und 8:42: + # + # +20 Wh Netz + # +60 Wh PV + # =80 Wh Verbrauch/Ladung + # + # Erwarteter Strommix: + # Netz = 20 / 80 = 25 % + # PV = 60 / 80 = 75 % + + current_entry["counter"]["counter0"]["imported"] = 2520 + + current_entry["pv"]["all"]["exported"] = 2560 + current_entry["pv"]["pv1"]["exported"] = 2560 + + current_entry["cp"]["all"]["imported"] = 4080 + current_entry["cp"]["cp4"]["imported"] = 4080 + + # _get_reference_entries() soll diesen aktuellen Snapshot verwenden. + monkeypatch.setattr( + chargelog, + "create_entry", + Mock(return_value=current_entry) + ) + + calc_energy_costs(cp, True) + + assert cp.data.set.log.charged_energy_by_source == pytest.approx({ + 'bat': 0, + 'cp': 0, + 'grid': 20, + 'pv': 60 + }) + + # Kosten: + # Netz: 20 Wh * 0.0003 €/Wh = 0.006 € + # PV: 60 Wh * 0.00015 €/Wh = 0.009 € + # Gesamt 0.015 € + assert cp.data.set.log.costs == pytest.approx(0.015) + + def test_calc_charge_cost_reference_end_unique_price(mock_data, monkeypatch): cp = Chargepoint(4, None) cp.data.set.log.imported_since_plugged = cp.data.set.log.imported_since_mode_switch = 3950 @@ -107,6 +204,14 @@ def test_calc_charge_cost_reference_end_unique_price(mock_data, monkeypatch): cp.data.get.imported = 4100 cp.data.set.log.charged_energy_by_source = {'grid': 1243, 'pv': 386, 'bat': 671, 'cp': 0.0} daily_log = mock_daily_log(monkeypatch) + current_entry = json.loads(json.dumps(daily_log["entries"][-1])) + current_entry["bat"]["all"]["exported"] = 4000 + current_entry["bat"]["bat2"]["exported"] = 4000 + current_entry["counter"]["counter0"]["imported"] = 4500 + current_entry["cp"]["cp4"]["imported"] = 4100 + current_entry["pv"]["all"]["exported"] = 3000 + current_entry["pv"]["pv1"]["exported"] = 3000 + monkeypatch.setattr(chargelog, "create_entry", Mock(return_value=current_entry)) data.data.general_data.data.prices = Prices(bat=0.0002, cp=0, grid=0.0002, pv=0.0002) with patch("builtins.open", mock_open(read_data=json.dumps(daily_log))): @@ -170,3 +275,33 @@ def test_calc_charge_cost_reference_middle_day_change(mock_data, monkeypatch): assert cp.data.set.log.charged_energy_by_source == { 'grid': 1242.8, 'pv': 385.8, 'bat': 671.4, 'cp': 0.0} assert round(cp.data.set.log.costs, 5) == 0.5 + + +def test_small_positive_charge_is_assigned_to_energy_sources(mock_data, monkeypatch): + cp = Chargepoint(4, None) + charged_energy = 80 + cp.data.set.log.imported_since_plugged = cp.data.set.log.imported_since_mode_switch = charged_energy + + # Beispielhafte Verteilung + processed_entries = { + "totals": { + "cp": { + "cp4": { + "energy_imported": 1000, + "bat": 250, + "cp": 0, + "grid": 500, + "pv": 250, + } + } + } + } + monkeypatch.setattr(chargelog, "_get_reference_position", Mock(return_value=chargelog.ReferenceTime.START)) + + charged_energy_by_source = chargelog.calculate_charged_energy_by_source( + cp, processed_entries, [] + ) + + assert charged_energy_by_source == {'bat': 20, 'cp': 0, 'grid': 40, 'pv': 20} + + assert sum(charged_energy_by_source.values()) == cp.data.set.log.imported_since_mode_switch diff --git a/packages/helpermodules/data_migration/data_migration.py b/packages/helpermodules/data_migration/data_migration.py index 40da4697bc..510448599d 100644 --- a/packages/helpermodules/data_migration/data_migration.py +++ b/packages/helpermodules/data_migration/data_migration.py @@ -25,7 +25,8 @@ from helpermodules.data_migration.id_mapping import MapId from helpermodules.hardware_configuration import update_hardware_configuration from helpermodules.measurement_logging.process_log import get_totals -from helpermodules.measurement_logging.write_log import LegacySmartHomeLogData, get_names +from helpermodules.measurement_logging.process_log_entry_builder import LegacySmartHomeLogData +from helpermodules.measurement_logging.write_log import get_names from helpermodules.timecheck import convert_timedelta_to_time_string, get_difference from helpermodules.utils import joined_thread_handler from helpermodules.utils.precision_math import string_to_float, string_to_int diff --git a/packages/helpermodules/measurement_logging/process_log_calculation.py b/packages/helpermodules/measurement_logging/process_log_calculation.py index b03f685697..80ea5c6a4d 100644 --- a/packages/helpermodules/measurement_logging/process_log_calculation.py +++ b/packages/helpermodules/measurement_logging/process_log_calculation.py @@ -197,10 +197,11 @@ def get_grid_counter(entry) -> Dict: return counter else: raise KeyError(f"Kein Zähler für das Netz gefunden in Eintrag '{entry['timestamp']}'.") + try: message = "" - grid_counter = get_grid_counter(entry) # Wenn neben dem "all" Eintrag kein weiterer Eintrag existiert, dann gibt es keine Komponenten. + grid_counter = get_grid_counter(entry) if ((safe_get_nested(entry, "bat", "all", "fault_state") == 2 and len(entry.get("bat", {})) > 1) or (safe_get_nested(entry, "cp", "all", "fault_state") == 2 and len(entry.get("cp", {})) > 1) or (safe_get_nested(entry, "pv", "all", "fault_state") == 2 and len(entry.get("pv", {})) > 1) or diff --git a/packages/helpermodules/measurement_logging/process_log_entry_builder.py b/packages/helpermodules/measurement_logging/process_log_entry_builder.py new file mode 100644 index 0000000000..bcb653a631 --- /dev/null +++ b/packages/helpermodules/measurement_logging/process_log_entry_builder.py @@ -0,0 +1,225 @@ + +import logging +from math import isnan +import re + +from paho.mqtt.client import Client as MqttClient, MQTTMessage +from typing import Dict, Optional, Union + +from control import data +from helpermodules.broker import BrokerClient +from helpermodules import timecheck +from helpermodules.utils.topic_parser import decode_payload, get_index +log = logging.getLogger(__name__) + + +class LegacySmartHomeLogData: + def __init__(self) -> None: + self.all_received_topics: Dict = {} + self.sh_dict: Dict = {} + self.sh_names: Dict = {} + try: + BrokerClient("smart-home-logging", self.on_connect, self.on_message).start_finite_loop() + for topic, payload in self.all_received_topics.items(): + if re.search("openWB/LegacySmartHome/config/get/Devices/[1-9]/device_configured", topic) is not None: + if decode_payload(payload) == 1: + index = get_index(topic) + self.sh_dict.update({f"sh{index}": {}}) + for topic, payload in self.all_received_topics.items(): + if f"openWB/LegacySmartHome/Devices/{index}/Wh" == topic: + self.sh_dict[f"sh{index}"].update({"imported": decode_payload(payload), "exported": 0}) + for sensor_id in range(0, 3): + if f"openWB/LegacySmartHome/Devices/{index}/TemperatureSensor{sensor_id}" == topic: + self.sh_dict[f"sh{index}"].update({f"temp{sensor_id}": decode_payload(payload)}) + for topic, payload in self.all_received_topics.items(): + if f"openWB/LegacySmartHome/config/get/Devices/{index}/device_name" == topic: + self.sh_names.update({f"sh{index}": decode_payload(payload)}) + except Exception: + log.exception("Fehler im Werte-Logging-Modul für SmartHome") + + def on_connect(self, client: MqttClient, userdata, flags: dict, rc: int): + client.subscribe("openWB/LegacySmartHome/#", 2) + + def on_message(self, client: MqttClient, userdata, msg: MQTTMessage): + self.all_received_topics.update({msg.topic: msg.payload}) + + +def create_entry(sh_log_data: Optional[LegacySmartHomeLogData], previous_entry: Optional[Dict]) -> Dict: + date = timecheck.create_timestamp_HH_MM() + current_timestamp = int(timecheck.create_timestamp()) + + try: + prices = data.data.general_data.data.prices + try: + grid_price = data.data.optional_data.ep_get_current_price() + fault_state = max(data.data.optional_data.data.electricity_pricing.flexible_tariff.get.fault_state, + data.data.optional_data.data.electricity_pricing.grid_fee.get.fault_state) + except Exception: + grid_price = prices.grid + fault_state = 0 + prices_dict = {"grid": grid_price, + "pv": prices.pv, + "bat": prices.bat, + "cp": prices.cp, + "fault_state": fault_state} + except Exception: + log.exception("Fehler im Werte-Logging-Modul für Preise") + prices_dict = {"fault_state": 0} + + try: + cp_dict = {"all": {"imported": data.data.cp_all_data.data.get.imported, + "exported": data.data.cp_all_data.data.get.exported, + "fault_state": data.data.cp_all_data.data.get.fault_state}} + except Exception: + log.exception("Fehler im Werte-Logging-Modul") + cp_dict = {} + for cp in data.data.cp_data: + try: + if "cp" in cp: + cp_dict.update({cp: {"imported": data.data.cp_data[cp].data.get.imported, + "exported": data.data.cp_data[cp].data.get.exported, + "fault_state": data.data.cp_data[cp].data.get.fault_state}}) + except Exception: + log.exception("Fehler im Werte-Logging-Modul für Ladepunkt "+str(cp)) + + ev_dict = {} + for ev in data.data.ev_data: + try: + if "ev" in ev: + ev_dict.update( + {ev: {"soc": data.data.ev_data[ev].data.get.soc, + "fault_state": data.data.ev_data[ev].data.get.fault_state}}) + except Exception: + log.exception("Fehler im Werte-Logging-Modul für EV "+str(ev)) + + counter_dict = {} + counter_all_data = data.data.counter_all_data + # Zählt alle effektiven Hausverbrauchs-Zähler, auch bei Auto-Vererbung über den Parent. + is_home_consumption_by_counter = {} + for current_counter in data.data.counter_data.values(): + try: + is_home_consumption_by_counter[current_counter.num] = counter_all_data.is_home_consumption_counter( + current_counter.num) + except Exception: + log.exception("Fehler beim Ermitteln der Hausverbrauchszähler.") + is_home_consumption_by_counter[current_counter.num] = False + + home_consumption_counter_count = sum(1 for is_hc in is_home_consumption_by_counter.values() if is_hc) + + for counter in data.data.counter_data.values(): + try: + # Der EVU-Zähler muss immer geloggt werden, unabhängig von Hausverbrauchs- oder + # extra_meter-Zuordnung - sonst findet process_log.get_grid_counter() keinen Netzzähler mehr. + is_grid_counter = counter_all_data.get_id_evu_counter() == counter.num + is_home_consumption_counter = is_home_consumption_by_counter.get(counter.num, False) + if not is_grid_counter and is_home_consumption_counter and home_consumption_counter_count == 1: + continue + skip_counter = False + if not is_grid_counter: + for consumer in data.data.consumer_data: + if counter.num == data.data.consumer_data[consumer].data.extra_meter: + skip_counter = True + break + if skip_counter is False: + counter_dict.update( + {f"counter{counter.num}": { + "imported": counter.data.get.imported, + "exported": counter.data.get.exported, + "grid": is_grid_counter, + "fault_state": counter.data.get.fault_state}}) + except Exception: + log.exception("Fehler im Werte-Logging-Modul für Zähler "+str(counter)) + + try: + pv_dict = {"all": {"exported": data.data.pv_all_data.data.get.exported, + "fault_state": data.data.pv_all_data.data.get.fault_state}} + except Exception: + log.exception("Fehler im Werte-Logging-Modul für PV-Daten") + pv_dict = {} + if data.data.pv_all_data.data.config.configured: + for pv in data.data.pv_data: + try: + pv_dict.update( + {pv: {"exported": data.data.pv_data[pv].data.get.exported, + "fault_state": data.data.pv_data[pv].data.get.fault_state}}) + except Exception: + log.exception("Fehler im Werte-Logging-Modul für Wechselrichter "+str(pv)) + + try: + bat_dict = {"all": {"imported": data.data.bat_all_data.data.get.imported, + "exported": data.data.bat_all_data.data.get.exported, + "soc": data.data.bat_all_data.data.get.soc, + "fault_state": data.data.bat_all_data.data.get.fault_state}} + except Exception: + log.exception("Fehler im Werte-Logging-Modul für Batteriespeicher-Daten") + bat_dict = {} + if data.data.bat_all_data.data.config.configured: + for bat in data.data.bat_data: + try: + bat_dict.update({bat: {"imported": data.data.bat_data[bat].data.get.imported, + "exported": data.data.bat_data[bat].data.get.exported, + "soc": data.data.bat_data[bat].data.get.soc, + "fault_state": data.data.bat_data[bat].data.get.fault_state}}) + except Exception: + log.exception("Fehler im Werte-Logging-Modul für Speicher "+str(bat)) + + try: + consumer_dict: Dict[str, Dict[str, Union[float, int]]] = {"all": { + "imported": data.data.consumer_all_data.data.get.imported, + "exported": data.data.consumer_all_data.data.get.exported, + "fault_state": data.data.consumer_all_data.data.get.fault_state}} + except Exception: + log.exception("Fehler im Werte-Logging-Modul für Verbraucher-Daten") + consumer_dict = {} + for consumer in data.data.consumer_data: + try: + consumer_dict.update({consumer: {"imported": data.data.consumer_data[consumer].data.get.imported, + "exported": data.data.consumer_data[consumer].data.get.exported, + "fault_state": data.data.consumer_data[consumer].data.get.fault_state}}) + except Exception: + log.exception("Fehler im Werte-Logging-Modul für Verbraucher "+str(consumer)) + + try: + hc_dict = {"all": { + "imported": data.data.counter_all_data.data.set.imported_home_consumption, + "fault_state": 2 if data.data.counter_all_data.data.set.invalid_home_consumption >= 3 else 0}} + except Exception: + log.exception("Fehler im Werte-Logging-Modul für Hausverbrauch") + hc_dict = {} + new_entry = { + "timestamp": current_timestamp, + "date": date, + "prices": prices_dict, + "cp": cp_dict, + "ev": ev_dict, + "counter": counter_dict, + "pv": pv_dict, + "bat": bat_dict, + "consumer": consumer_dict, + "sh": sh_log_data.sh_dict if sh_log_data is not None else {}, + "hc": hc_dict + } + + return fix_values(new_entry, previous_entry) + + +def fix_values(new_entry: Dict, previous_entry: Optional[Dict]) -> Dict: + def find_and_fix_value(value_name): + if value.get(value_name) is not None: + if value[value_name] == 0: + try: + if (previous_entry[group][component][value_name] is not None and + isnan(previous_entry[group][component][value_name]) is False): + value[value_name] = previous_entry[group][component][value_name] + except KeyError: + log.exception("Es konnte kein vorheriger Wert gefunden werden.") + if previous_entry is not None: + for group, value in new_entry.items(): + if group not in ("bat", "consumer", "counter", "cp", "pv", "hc"): + continue + for component, value in value.items(): + find_and_fix_value("exported") + find_and_fix_value("imported") + else: + log.warning("Keine vorherigen Werte vorhanden, um aktuelle Werte auf Plausibilität zu prüfen.") + return new_entry diff --git a/packages/helpermodules/measurement_logging/process_log_unit_test.py b/packages/helpermodules/measurement_logging/process_log_unit_test.py index d9940f0da5..365c0e4276 100644 --- a/packages/helpermodules/measurement_logging/process_log_unit_test.py +++ b/packages/helpermodules/measurement_logging/process_log_unit_test.py @@ -1,15 +1,19 @@ from copy import deepcopy import json import os +from typing import Dict from unittest.mock import Mock, mock_open import pytest + from helpermodules.measurement_logging.process_log import ( + analyse_percentage, get_totals, _collect_daily_log_data, + analyse_percentage_totals, _get_last_entry_for_period, _apply_source_totals, - analyse_percentage_totals) +) def test_get_totals(daily_log_sample, daily_log_totals): @@ -21,6 +25,87 @@ def test_get_totals(daily_log_sample, daily_log_totals): assert totals == daily_log_totals +@pytest.mark.parametrize("name", + ["regular", + "negative consumption", + "cp export"]) +def test_analyse_percentage(name: str, + daily_log_entry_percentage: Dict, + daily_log_entry_percentage_negative_consumption: Dict, + daily_log_entry_percentage_cp_discharge: Dict): + # setup + if name == "regular": + data = daily_log_entry_percentage + expected = deepcopy(data) + expected.update({"energy_source": {'bat': 0.2398, 'cp': 0.0, 'grid': 0.6504, 'pv': 0.1099}}) + elif name == "negative consumption": + data = daily_log_entry_percentage_negative_consumption + expected = deepcopy(data) + expected.update({"energy_source": {'bat': 0.0, 'cp': 0.0, 'grid': 0.0, 'pv': 0.0}}) + elif name == "cp export": + data = daily_log_entry_percentage_cp_discharge + expected = deepcopy(data) + expected.update({"energy_source": {'bat': 0.2006, 'cp': 0.1459, 'grid': 0.5441, 'pv': 0.1094}}) + + # execution + entry, message = analyse_percentage(data) + + # evaluation + assert entry == expected + assert message == "" + + +@pytest.mark.parametrize("test_case, entry_data, expected_energy_source, should_be_unchanged", [ + ( + "zero_consumption", + { + "timestamp": 1234567890, + "date": "00:31", + "bat": {"all": {"energy_imported": 5.0, "energy_exported": 5.0, "fault_state": 0}}, + "cp": {"all": {"energy_exported": 0.0, "fault_state": 0}}, + "pv": {"all": {"energy_exported": 0.0, "fault_state": 0}}, + "counter": {"counter0": {"grid": True, "energy_imported": 5.0, "energy_exported": 5.0, "fault_state": 0}} + }, + {"grid": 0, "pv": 0, "bat": 0, "cp": 0}, + False + ), + ( + "missing_sections", + { + "timestamp": 1234567890, + "date": "00:31", + "counter": {"counter0": {"grid": True, "energy_imported": 10.0, "energy_exported": 2.0, "fault_state": 0}} + }, + {"grid": 1.0, "pv": 0.0, "bat": 0.0, "cp": 0.0}, + False + ), + ( + "no_grid_counter", + { + "timestamp": 1234567890, + "date": "00:31", + "bat": {"all": {"energy_imported": 0.0, "energy_exported": 5.0, "fault_state": 0}}, + "counter": {"counter0": {"grid": False, "energy_imported": 10.0, "energy_exported": 2.0, "fault_state": 0}} + }, + None, + True + ) +]) +def test_analyse_percentage_edge_cases(test_case, entry_data, expected_energy_source, should_be_unchanged): + # execution + result, message = analyse_percentage(entry_data) + + # evaluation + if should_be_unchanged: + # Entry should be unchanged, no energy_source added due to error + assert result["timestamp"] == entry_data["timestamp"] + assert "energy_source" not in result or result.get("energy_source") is None + else: + # Energy source should be calculated correctly + assert result["energy_source"] == expected_energy_source + assert message == "" + + def test_analyse_percentage_totals(): # setup current_dir = os.path.dirname(os.path.abspath(__file__)) diff --git a/packages/helpermodules/measurement_logging/write_log.py b/packages/helpermodules/measurement_logging/write_log.py index fea8806629..e4397da48c 100644 --- a/packages/helpermodules/measurement_logging/write_log.py +++ b/packages/helpermodules/measurement_logging/write_log.py @@ -1,21 +1,18 @@ import os import json import logging -from math import isnan from pathlib import Path -import re import string -from paho.mqtt.client import Client as MqttClient, MQTTMessage -from typing import Dict, Optional, Union +from typing import Dict, Optional from control import data -from helpermodules.broker import BrokerClient from helpermodules import timecheck from helpermodules.constants import DEFAULT_COLORS from helpermodules.utils.json_file_handler import write_and_check -from helpermodules.utils.topic_parser import decode_payload, get_index from modules.common.utils.component_parser import get_component_name_by_id, get_component_color_by_id +from helpermodules.measurement_logging.process_log_entry_builder import LegacySmartHomeLogData, create_entry + log = logging.getLogger(__name__) # erstellt für jeden Tag eine Datei, die die Daten für den Langzeitgraph enthält. @@ -107,37 +104,6 @@ # } -class LegacySmartHomeLogData: - def __init__(self) -> None: - self.all_received_topics: Dict = {} - self.sh_dict: Dict = {} - self.sh_names: Dict = {} - try: - BrokerClient("smart-home-logging", self.on_connect, self.on_message).start_finite_loop() - for topic, payload in self.all_received_topics.items(): - if re.search("openWB/LegacySmartHome/config/get/Devices/[1-9]/device_configured", topic) is not None: - if decode_payload(payload) == 1: - index = get_index(topic) - self.sh_dict.update({f"sh{index}": {}}) - for topic, payload in self.all_received_topics.items(): - if f"openWB/LegacySmartHome/Devices/{index}/Wh" == topic: - self.sh_dict[f"sh{index}"].update({"imported": decode_payload(payload), "exported": 0}) - for sensor_id in range(0, 3): - if f"openWB/LegacySmartHome/Devices/{index}/TemperatureSensor{sensor_id}" == topic: - self.sh_dict[f"sh{index}"].update({f"temp{sensor_id}": decode_payload(payload)}) - for topic, payload in self.all_received_topics.items(): - if f"openWB/LegacySmartHome/config/get/Devices/{index}/device_name" == topic: - self.sh_names.update({f"sh{index}": decode_payload(payload)}) - except Exception: - log.exception("Fehler im Werte-Logging-Modul für SmartHome") - - def on_connect(self, client: MqttClient, userdata, flags: dict, rc: int): - client.subscribe("openWB/LegacySmartHome/#", 2) - - def on_message(self, client: MqttClient, userdata, msg: MQTTMessage): - self.all_received_topics.update({msg.topic: msg.payload}) - - def save_log(): try: parent_file = Path(__file__).resolve().parents[3] / "data" / "daily_log" @@ -200,187 +166,6 @@ def get_previous_entry(parent_file: Path, content: Dict) -> Optional[Dict]: return previous_entry -def create_entry(sh_log_data: LegacySmartHomeLogData, previous_entry: Optional[Dict]) -> Dict: - date = timecheck.create_timestamp_HH_MM() - current_timestamp = int(timecheck.create_timestamp()) - - try: - prices = data.data.general_data.data.prices - try: - grid_price = data.data.optional_data.ep_get_current_price() - fault_state = max(data.data.optional_data.data.electricity_pricing.flexible_tariff.get.fault_state, - data.data.optional_data.data.electricity_pricing.grid_fee.get.fault_state) - except Exception: - grid_price = prices.grid - fault_state = 0 - prices_dict = {"grid": grid_price, - "pv": prices.pv, - "bat": prices.bat, - "cp": prices.cp, - "fault_state": fault_state} - except Exception: - log.exception("Fehler im Werte-Logging-Modul für Preise") - prices_dict = {"fault_state": 0} - - try: - cp_dict = {"all": {"imported": data.data.cp_all_data.data.get.imported, - "exported": data.data.cp_all_data.data.get.exported, - "fault_state": data.data.cp_all_data.data.get.fault_state}} - except Exception: - log.exception("Fehler im Werte-Logging-Modul") - cp_dict = {} - for cp in data.data.cp_data: - try: - if "cp" in cp: - cp_dict.update({cp: {"imported": data.data.cp_data[cp].data.get.imported, - "exported": data.data.cp_data[cp].data.get.exported, - "fault_state": data.data.cp_data[cp].data.get.fault_state}}) - except Exception: - log.exception("Fehler im Werte-Logging-Modul für Ladepunkt "+str(cp)) - - ev_dict = {} - for ev in data.data.ev_data: - try: - if "ev" in ev: - ev_dict.update( - {ev: {"soc": data.data.ev_data[ev].data.get.soc, - "fault_state": data.data.ev_data[ev].data.get.fault_state}}) - except Exception: - log.exception("Fehler im Werte-Logging-Modul für EV "+str(ev)) - - counter_dict = {} - counter_all_data = data.data.counter_all_data - # Zählt alle effektiven Hausverbrauchs-Zähler, auch bei Auto-Vererbung über den Parent. - is_home_consumption_by_counter = {} - for current_counter in data.data.counter_data.values(): - try: - is_home_consumption_by_counter[current_counter.num] = counter_all_data.is_home_consumption_counter( - current_counter.num) - except Exception: - log.exception("Fehler beim Ermitteln der Hausverbrauchszähler.") - is_home_consumption_by_counter[current_counter.num] = False - - home_consumption_counter_count = sum(1 for is_hc in is_home_consumption_by_counter.values() if is_hc) - - for counter in data.data.counter_data.values(): - try: - # Der EVU-Zähler muss immer geloggt werden, unabhängig von Hausverbrauchs- oder - # extra_meter-Zuordnung - sonst findet process_log.get_grid_counter() keinen Netzzähler mehr. - is_grid_counter = counter_all_data.get_id_evu_counter() == counter.num - is_home_consumption_counter = is_home_consumption_by_counter.get(counter.num, False) - if not is_grid_counter and is_home_consumption_counter and home_consumption_counter_count == 1: - continue - skip_counter = False - if not is_grid_counter: - for consumer in data.data.consumer_data: - if counter.num == data.data.consumer_data[consumer].data.extra_meter: - skip_counter = True - break - if skip_counter is False: - counter_dict.update( - {f"counter{counter.num}": { - "imported": counter.data.get.imported, - "exported": counter.data.get.exported, - "grid": is_grid_counter, - "fault_state": counter.data.get.fault_state}}) - except Exception: - log.exception("Fehler im Werte-Logging-Modul für Zähler "+str(counter)) - - try: - pv_dict = {"all": {"exported": data.data.pv_all_data.data.get.exported, - "fault_state": data.data.pv_all_data.data.get.fault_state}} - except Exception: - log.exception("Fehler im Werte-Logging-Modul für PV-Daten") - pv_dict = {} - if data.data.pv_all_data.data.config.configured: - for pv in data.data.pv_data: - try: - pv_dict.update( - {pv: {"exported": data.data.pv_data[pv].data.get.exported, - "fault_state": data.data.pv_data[pv].data.get.fault_state}}) - except Exception: - log.exception("Fehler im Werte-Logging-Modul für Wechselrichter "+str(pv)) - - try: - bat_dict = {"all": {"imported": data.data.bat_all_data.data.get.imported, - "exported": data.data.bat_all_data.data.get.exported, - "soc": data.data.bat_all_data.data.get.soc, - "fault_state": data.data.bat_all_data.data.get.fault_state}} - except Exception: - log.exception("Fehler im Werte-Logging-Modul für Batteriespeicher-Daten") - bat_dict = {} - if data.data.bat_all_data.data.config.configured: - for bat in data.data.bat_data: - try: - bat_dict.update({bat: {"imported": data.data.bat_data[bat].data.get.imported, - "exported": data.data.bat_data[bat].data.get.exported, - "soc": data.data.bat_data[bat].data.get.soc, - "fault_state": data.data.bat_data[bat].data.get.fault_state}}) - except Exception: - log.exception("Fehler im Werte-Logging-Modul für Speicher "+str(bat)) - - try: - consumer_dict: Dict[str, Dict[str, Union[float, int]]] = {"all": { - "imported": data.data.consumer_all_data.data.get.imported, - "exported": data.data.consumer_all_data.data.get.exported, - "fault_state": data.data.consumer_all_data.data.get.fault_state}} - except Exception: - log.exception("Fehler im Werte-Logging-Modul für Verbraucher-Daten") - consumer_dict = {} - for consumer in data.data.consumer_data: - try: - consumer_dict.update({consumer: {"imported": data.data.consumer_data[consumer].data.get.imported, - "exported": data.data.consumer_data[consumer].data.get.exported, - "fault_state": data.data.consumer_data[consumer].data.get.fault_state}}) - except Exception: - log.exception("Fehler im Werte-Logging-Modul für Verbraucher "+str(consumer)) - - try: - hc_dict = {"all": { - "imported": data.data.counter_all_data.data.set.imported_home_consumption, - "fault_state": 2 if data.data.counter_all_data.data.set.invalid_home_consumption >= 3 else 0}} - except Exception: - log.exception("Fehler im Werte-Logging-Modul für Hausverbrauch") - hc_dict = {} - new_entry = { - "timestamp": current_timestamp, - "date": date, - "prices": prices_dict, - "cp": cp_dict, - "ev": ev_dict, - "counter": counter_dict, - "pv": pv_dict, - "bat": bat_dict, - "consumer": consumer_dict, - "sh": sh_log_data.sh_dict, - "hc": hc_dict - } - - return fix_values(new_entry, previous_entry) - - -def fix_values(new_entry: Dict, previous_entry: Optional[Dict]) -> Dict: - def find_and_fix_value(value_name): - if value.get(value_name) is not None: - if value[value_name] == 0: - try: - if (previous_entry[group][component][value_name] is not None and - isnan(previous_entry[group][component][value_name]) is False): - value[value_name] = previous_entry[group][component][value_name] - except KeyError: - log.exception("Es konnte kein vorheriger Wert gefunden werden.") - if previous_entry is not None: - for group, value in new_entry.items(): - if group not in ("bat", "consumer", "counter", "cp", "pv", "hc"): - continue - for component, value in value.items(): - find_and_fix_value("exported") - find_and_fix_value("imported") - else: - log.warning("Keine vorherigen Werte vorhanden, um aktuelle Werte auf Plausibilität zu prüfen.") - return new_entry - - def get_names(elements: Dict, sh_names: Dict, valid_names: Optional[Dict] = None) -> Dict: """ Ermittelt die Namen der Komponenten, Fahrzeuge, Ladepunkte und SmartHome-Geräte, welche in elements vorhanden sind und gibt diese als Dictionary zurück. diff --git a/packages/helpermodules/measurement_logging/write_log_test.py b/packages/helpermodules/measurement_logging/write_log_test.py index 1ff6302b4f..2616d62108 100644 --- a/packages/helpermodules/measurement_logging/write_log_test.py +++ b/packages/helpermodules/measurement_logging/write_log_test.py @@ -5,6 +5,9 @@ from helpermodules.measurement_logging.write_log import get_names +from helpermodules.measurement_logging.process_log_entry_builder import fix_values + + def test_get_names(daily_log_totals, monkeypatch): # setup component_names_mock = Mock(side_effect=["Speicher", "Zähler", "Wechselrichter"]) @@ -56,7 +59,7 @@ def test_fix_values(): 'timestamp': 1709109001} # execution - fixed_values = write_log.fix_values(new_entry, previous_entry) + fixed_values = fix_values(new_entry, previous_entry) # evaluation assert fixed_values == {'bat': {'all': {'exported': 0, 'imported': 2369.658, 'soc': 97}, @@ -106,7 +109,7 @@ def test_fix_values_missing_components(): 'timestamp': 1709109001} # execution - fixed_values = write_log.fix_values(new_entry, previous_entry) + fixed_values = fix_values(new_entry, previous_entry) # evaluation assert fixed_values == {'bat': {'all': {'exported': 0, 'imported': 2369.658, 'soc': 97}, @@ -152,6 +155,9 @@ def test_save_log_keeps_existing_names_and_colors_when_writing_new_entry(monkeyp write_log.save_log() # evaluation + write_log.LegacySmartHomeLogData.assert_called_once_with() + write_log.create_entry.assert_called_once_with( + write_log.LegacySmartHomeLogData.return_value, existing_content["entries"][-1]) written_content = write_and_check_mock.call_args.args[1] assert written_content["names"] == { "cp1": "Bestehender Ladepunkt",