From c7d1ab739fdad8541c0ee053da9965b5ce23061f Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 14 Jun 2026 12:27:41 +0100 Subject: [PATCH 01/74] feat: add clipping peak cost penalty to optimizer metric --- apps/predbat/config.py | 43 ++++++++++++++++++ apps/predbat/fetch.py | 6 +++ apps/predbat/plan.py | 33 +++++++++++++- apps/predbat/prediction.py | 28 +++++++++++- apps/predbat/tests/test_infra.py | 42 ++++++++++++++++- apps/predbat/tests/test_model.py | 78 ++++++++++++++++++++++++++++++++ 6 files changed, 227 insertions(+), 3 deletions(-) diff --git a/apps/predbat/config.py b/apps/predbat/config.py index 0a1bbdb2f..31ad5f4f5 100644 --- a/apps/predbat/config.py +++ b/apps/predbat/config.py @@ -1428,6 +1428,49 @@ "default": 20, "enable": "predheat_enable", }, + { + "name": "clipping_peak_enable", + "friendly_name": "Clipping Peak Enable", + "type": "switch", + "default": False, + "icon": "mdi:chart-bell-curve-cumulative", + }, + { + "name": "clipping_cost_weight", + "friendly_name": "Clipping Cost Weight", + "type": "input_number", + "min": 0, + "max": 10.0, + "step": 0.1, + "unit": "x", + "icon": "mdi:multiplication", + "enable": "clipping_peak_enable", + "default": 1.0, + }, + { + "name": "clipping_peak_amplification", + "friendly_name": "Clipping Peak Amplification", + "type": "input_number", + "min": 0.5, + "max": 3.0, + "step": 0.1, + "unit": "x", + "icon": "mdi:arrow-expand-vertical", + "enable": "clipping_peak_enable", + "default": 1.0, + }, + { + "name": "clipping_limit_override", + "friendly_name": "Clipping Limit Override", + "type": "input_number", + "min": 0, + "max": 50000, + "step": 100, + "unit": "W", + "icon": "mdi:flash-alert", + "enable": "clipping_peak_enable", + "default": 0, + }, { "name": "inverter_limit_override", "friendly_name": "Inverter Limit Override", diff --git a/apps/predbat/fetch.py b/apps/predbat/fetch.py index 3a5444260..450495b55 100644 --- a/apps/predbat/fetch.py +++ b/apps/predbat/fetch.py @@ -2310,6 +2310,12 @@ def fetch_config_options(self): self.carbon_enable = self.get_arg("carbon_enable") self.carbon_metric = self.get_arg("carbon_metric") + # Clipping peak cost penalty model + self.clipping_peak_enable = self.get_arg("clipping_peak_enable") + self.clipping_cost_weight = self.get_arg("clipping_cost_weight") + self.clipping_peak_amplification = self.get_arg("clipping_peak_amplification") + self.clipping_limit_override = self.get_arg("clipping_limit_override") / MINUTE_WATT if self.get_arg("clipping_limit_override") else 0 + # iBoost solar diverter model self.iboost_enable = self.get_arg("iboost_enable") self.iboost_gas = self.get_arg("iboost_gas") diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index 2ef436436..96090b9e7 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -959,6 +959,32 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): pv_forecast_minute_step = self.step_data_history(self.pv_forecast_minute, self.minutes_now, forward=True, cloud_factor=self.metric_cloud_coverage) pv_forecast_minute10_step = self.step_data_history(self.pv_forecast_minute10, self.minutes_now, forward=True, cloud_factor=min(self.metric_cloud_coverage + 0.2, 1.0) if self.metric_cloud_coverage else None, flip=True) + # Clipping peak: create a peak PV stream for clipping cost detection + # Uses raw forecast WITHOUT cloud model (no energy-conserving oscillation that smooths peaks) + # Scaled by amplification factor for safety margin + pv_forecast_peak_step = None + clipping_limit_effective = 0 + if self.clipping_peak_enable: + pv_forecast_peak_step = self.step_data_history( + self.pv_forecast_minute, self.minutes_now, forward=True, cloud_factor=None + ) + # Apply amplification factor + if self.clipping_peak_amplification != 1.0: + pv_forecast_peak_step = {k: v * self.clipping_peak_amplification for k, v in pv_forecast_peak_step.items()} + + # Calculate effective clipping limit: most restrictive hardware constraint + if self.clipping_limit_override > 0: + clipping_limit_effective = self.clipping_limit_override + else: + limits = [] + if self.inverter_limit > 0: + limits.append(self.inverter_limit) + if self.export_limit > 0: + limits.append(self.export_limit) + if self.pv_ac_limit > 0: + limits.append(self.pv_ac_limit) + clipping_limit_effective = min(limits) if limits else 0 + # Save step data for debug self.load_minutes_step = load_minutes_step self.load_minutes_step10 = load_minutes_step10 @@ -970,7 +996,12 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): self.calculate_yesterday() # Creation prediction object - self.prediction = Prediction(self, pv_forecast_minute_step, pv_forecast_minute10_step, load_minutes_step, load_minutes_step10) + self.prediction = Prediction( + self, pv_forecast_minute_step, pv_forecast_minute10_step, load_minutes_step, load_minutes_step10, + pv_forecast_peak_step=pv_forecast_peak_step, + clipping_limit=clipping_limit_effective, + clipping_cost_weight=self.clipping_cost_weight if self.clipping_peak_enable else 0, + ) # Check if LoadML is active and disable thread pools as it causes lockup due to race conditions with NumPy load_ml_comp = self.components.get_component("load_ml") if self.components else None diff --git a/apps/predbat/prediction.py b/apps/predbat/prediction.py index 6846d56e9..8cf4ae85c 100644 --- a/apps/predbat/prediction.py +++ b/apps/predbat/prediction.py @@ -92,7 +92,7 @@ class Prediction: Class to hold prediction input and output data and the run function """ - def __init__(self, base=None, pv_forecast_minute_step=None, pv_forecast_minute10_step=None, load_minutes_step=None, load_minutes_step10=None, soc_kw=None, soc_max=None): + def __init__(self, base=None, pv_forecast_minute_step=None, pv_forecast_minute10_step=None, load_minutes_step=None, load_minutes_step10=None, soc_kw=None, soc_max=None, pv_forecast_peak_step=None, clipping_limit=0, clipping_cost_weight=0): global PRED_GLOBAL if base: self.minutes_now = base.minutes_now @@ -195,6 +195,9 @@ def __init__(self, base=None, pv_forecast_minute_step=None, pv_forecast_minute10 self.prediction_cache = {} self.plan_interval_minutes = base.plan_interval_minutes self.charge_scaling10 = base.charge_scaling10 + self.pv_forecast_peak_step = pv_forecast_peak_step + self.clipping_limit = clipping_limit + self.clipping_cost_weight = clipping_cost_weight # Store this dictionary in global so we can reconstruct it in the thread without passing the data PRED_GLOBAL["dict"] = self.__dict__.copy() @@ -481,6 +484,7 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi first_charge = end_record export_to_first_charge = 0 clipped_today = 0 + clipping_penalty_total = 0 predict_soc = {} car_charging_soc_next = self.car_charging_soc_next[:] iboost_next = self.iboost_next @@ -516,6 +520,10 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi pv_ac_limit = self.pv_ac_limit * step set_charge_low_power = self.set_charge_window and self.set_charge_low_power and (save in ["best", "best10", "test"]) carbon_enable = self.carbon_enable + pv_forecast_peak_step = self.pv_forecast_peak_step + clipping_limit = self.clipping_limit + clipping_limit_step = clipping_limit * step if clipping_limit else 0 + clipping_cost_weight = self.clipping_cost_weight reserve = self.reserve soc_max = self.soc_max reserve_percent = calc_percent_limit(reserve, soc_max) @@ -658,6 +666,24 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi pv_now = pv_forecast_minute_step_flat[minute] load_yesterday = load_minutes_step_flat[minute] + # Clipping peak cost penalty: check if worst-case PV would exceed the clipping limit + # and add a cost to the metric if the battery can't absorb the excess. + # This makes the optimizer prefer plans that leave battery headroom during peak solar. + if pv_forecast_peak_step and clipping_limit_step > 0 and clipping_cost_weight > 0: + peak_pv = pv_forecast_peak_step.get(minute, 0) + if peak_pv > clipping_limit_step: + potential_clip = peak_pv - clipping_limit_step + # How much could the battery absorb right now? + battery_headroom = max(soc_max - soc, 0) * battery_loss + # Cap by max charge rate (conservative — doesn't account for curve, but safe) + max_charge_step = battery_rate_max_charge * battery_rate_max_scaling * step + absorbable = min(battery_headroom, max_charge_step) + unmitigated_clip = max(potential_clip - absorbable, 0) + if unmitigated_clip > 0: + clipping_penalty = unmitigated_clip * export_rate * clipping_cost_weight + metric += clipping_penalty + clipping_penalty_total += clipping_penalty + # Count PV kWh pv_kwh += pv_now diff --git a/apps/predbat/tests/test_infra.py b/apps/predbat/tests/test_infra.py index 715b04ea0..fe6560543 100644 --- a/apps/predbat/tests/test_infra.py +++ b/apps/predbat/tests/test_infra.py @@ -354,6 +354,10 @@ def get_default_config(self): "load_filter_modal": True, "carbon_enable": False, "carbon_metric": 0, + "clipping_peak_enable": False, + "clipping_cost_weight": 1.0, + "clipping_peak_amplification": 1.0, + "clipping_limit_override": 0, "iboost_enable": False, "iboost_gas": 4.0, "iboost_gas_export": 4.0, @@ -517,6 +521,10 @@ def reset_inverter(my_predbat): my_predbat.set_export_window = True my_predbat.set_charge_freeze = True my_predbat.set_export_freeze = True + my_predbat.clipping_peak_enable = False + my_predbat.clipping_cost_weight = 0 + my_predbat.clipping_peak_amplification = 1.0 + my_predbat.clipping_limit_override = 0 def plot(name, prediction): @@ -609,6 +617,10 @@ def simple_scenario( calculate_export_on_pv=True, assert_clipped=0, pv_ac_limit=0, + clipping_peak_enable=False, + clipping_cost_weight=0, + clipping_peak_amplification=1.0, + clipping_limit_override=0, ): """ No PV, No Load @@ -706,6 +718,12 @@ def simple_scenario( my_predbat.inverter_can_charge_during_export = inverter_can_charge_during_export my_predbat.charge_scaling10 = charge_scaling10 + # Clipping peak penalty settings + my_predbat.clipping_peak_enable = clipping_peak_enable + my_predbat.clipping_cost_weight = clipping_cost_weight + my_predbat.clipping_peak_amplification = clipping_peak_amplification + my_predbat.clipping_limit_override = clipping_limit_override + if my_predbat.iboost_enable and (((not iboost_solar) and (not iboost_charging)) or iboost_smart): my_predbat.iboost_plan = my_predbat.plan_iboost_smart() else: @@ -741,10 +759,32 @@ def simple_scenario( my_predbat.num_cars = 0 my_predbat.car_charging_slots[0] = [] + # Build peak PV step data for clipping tests + pv_peak_step = None + clipping_limit_eff = 0 + if clipping_peak_enable: + pv_peak_step = {k: v * clipping_peak_amplification for k, v in pv_step.items()} + if clipping_limit_override > 0: + clipping_limit_eff = clipping_limit_override + else: + limits = [] + if my_predbat.inverter_limit > 0: + limits.append(my_predbat.inverter_limit) + if my_predbat.export_limit > 0: + limits.append(my_predbat.export_limit) + if my_predbat.pv_ac_limit > 0: + limits.append(my_predbat.pv_ac_limit) + clipping_limit_eff = min(limits) if limits else 0 + if prediction_handle: prediction = prediction_handle else: - prediction = Prediction(my_predbat, pv_step, pv10_step, load_step, load10_step) + prediction = Prediction( + my_predbat, pv_step, pv10_step, load_step, load10_step, + pv_forecast_peak_step=pv_peak_step, + clipping_limit=clipping_limit_eff, + clipping_cost_weight=clipping_cost_weight if clipping_peak_enable else 0, + ) compute_charge_limit = False if charge_limit_best is None: diff --git a/apps/predbat/tests/test_model.py b/apps/predbat/tests/test_model.py index a27a6df37..ed009e139 100644 --- a/apps/predbat/tests/test_model.py +++ b/apps/predbat/tests/test_model.py @@ -1892,6 +1892,84 @@ def run_model_tests(my_predbat): # pv_ac_limit must NOT apply to hybrid inverters (PV is DC-coupled, clipping handled by inverter_limit) failed |= simple_scenario("pv_ac_limit_hybrid_ignored", my_predbat, 0, 2.0, assert_final_metric=-export_rate * 24, assert_final_soc=24, with_battery=True, hybrid=True, pv_ac_limit=1.5, assert_clipped=0) + # ---- Clipping Peak Cost Penalty Tests ---- + reset_rates(my_predbat, import_rate, export_rate) + reset_inverter(my_predbat) + + # No penalty when peak PV is below the clipping limit + # 0.5kW PV, 1kW inverter limit => no clipping, metric is just export revenue + failed |= simple_scenario( + "clipping_peak_no_clip", + my_predbat, + 0, + 0.5, + assert_final_metric=-export_rate * 24 * 0.5, + assert_final_soc=0, + with_battery=False, + inverter_limit=1.0, + clipping_peak_enable=True, + clipping_cost_weight=1.0, + ) + + # Penalty when peak PV exceeds inverter limit and no battery to absorb + # 2kW PV, 1kW inverter limit, no battery => with_battery=False means battery_rate_max_scaling=0 + # AC-coupled PV isn't clipped by inverter_limit, so full 2kW is exported + # Without clipping penalty: metric = -export_rate * 24 * 2 = -240p + # With clipping penalty: extra cost added for the 1kW excess above inverter_limit + failed_no_penalty, pred_no_penalty = simple_scenario( + "clipping_peak_baseline", + my_predbat, + 0, + 2.0, + assert_final_metric=-export_rate * 24 * 2.0, + assert_final_soc=0, + with_battery=False, + inverter_limit=1.0, + clipping_peak_enable=False, + return_prediction_handle=True, + ) + failed |= failed_no_penalty + + failed_with_penalty, pred_with_penalty = simple_scenario( + "clipping_peak_with_penalty", + my_predbat, + 0, + 2.0, + assert_final_metric=-export_rate * 24 * 2.0, # will differ due to penalty; checked below + assert_final_soc=0, + with_battery=False, + inverter_limit=1.0, + clipping_peak_enable=True, + clipping_cost_weight=1.0, + return_prediction_handle=True, + ignore_failed=True, + ) + # The penalty should make the metric less negative (higher) than without + metric_no_penalty = round(pred_no_penalty.predict_metric_best[max(pred_no_penalty.predict_metric_best.keys())] / 100.0, 2) if pred_no_penalty.predict_metric_best else 0 + metric_with_penalty = round(pred_with_penalty.predict_metric_best[max(pred_with_penalty.predict_metric_best.keys())] / 100.0, 2) if pred_with_penalty.predict_metric_best else 0 + if metric_with_penalty <= metric_no_penalty: + print("ERROR: clipping_peak_with_penalty metric {} should be > {} (penalty should increase metric)".format(metric_with_penalty, metric_no_penalty)) + failed = True + else: + print("Run scenario clipping_peak_with_penalty: PASS (metric {} > baseline {})".format(metric_with_penalty, metric_no_penalty)) + + # No penalty when battery has headroom to absorb excess + # 2kW PV, 1kW inverter limit, but battery at 0% with 100kWh capacity => battery absorbs all excess + failed |= simple_scenario( + "clipping_peak_battery_absorbs", + my_predbat, + 0, + 2.0, + assert_final_metric=-export_rate * 24, + assert_final_soc=24, + with_battery=True, + battery_soc=0.0, + battery_size=100.0, + inverter_limit=1.0, + clipping_peak_enable=True, + clipping_cost_weight=1.0, + ) + if failed: print("**** ERROR: Some Model tests failed ****") return failed From 30ef1d501bd378ea5b7490c0d229389650851d98 Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 14 Jun 2026 12:36:58 +0100 Subject: [PATCH 02/74] test: add side-by-side clipping comparison script (6 scenarios) --- apps/predbat/tests/compare_clipping.py | 262 +++++++++++++++++++++++++ 1 file changed, 262 insertions(+) create mode 100644 apps/predbat/tests/compare_clipping.py diff --git a/apps/predbat/tests/compare_clipping.py b/apps/predbat/tests/compare_clipping.py new file mode 100644 index 000000000..d7e212cde --- /dev/null +++ b/apps/predbat/tests/compare_clipping.py @@ -0,0 +1,262 @@ +# ----------------------------------------------------------------------------- +# Clipping Approach Comparison: Cloud-Model Penalty vs Baseline +# +# Runs identical scenarios with and without clipping_peak_enable and compares +# the optimizer metrics, final SoC, and clipping amounts. +# +# Usage: python apps\predbat\tests\compare_clipping.py +# ----------------------------------------------------------------------------- +# fmt: off +# pylint: disable=line-too-long + +import sys +import os +import math +import time + +# Add parent dirs to path +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) + +from tests.test_infra import reset_rates, reset_inverter, simple_scenario, Prediction + + +# --------------------------------------------------------------------------- +# Scenario definitions: hand-crafted clipping challenge cases +# --------------------------------------------------------------------------- + +SCENARIOS = [ + { + "name": "Clear Sky Peak Clipping", + "description": "7kWp panels, 5kW inverter limit, sunny day. Peak PV exceeds inverter limit. Battery starts at 80%.", + "pv_kw": 7.0, # peak generation in kW (exceeds inverter limit) + "load_kw": 0.5, # constant household load + "inverter_limit_kw": 5.0, # AC inverter limit + "export_limit_kw": 10.0, # no export restriction + "battery_size_kwh": 9.5, # typical UK hybrid battery + "battery_soc_percent": 80, # already mostly charged from overnight + "battery_rate_kw": 3.0, # charge/discharge rate + "import_rate_p": 25.0, # standard import rate + "export_rate_p": 15.0, # SEG export rate + "charge_target_percent": 100, # optimizer wants to charge to 100% + "hybrid": True, # DC-coupled hybrid inverter + }, + { + "name": "Cloudy Day - Intermittent Peaks", + "description": "6kWp panels, 3.6kW inverter. Cloud model would show spikes. Battery starts at 90% from overnight charge.", + "pv_kw": 6.0, + "load_kw": 0.3, + "inverter_limit_kw": 3.6, + "export_limit_kw": 10.0, + "battery_size_kwh": 9.5, + "battery_soc_percent": 90, + "battery_rate_kw": 2.6, + "import_rate_p": 25.0, + "export_rate_p": 15.0, + "charge_target_percent": 100, + "hybrid": True, + }, + { + "name": "Export Limited System", + "description": "12kWp array, 5kW DNO export limit. Large system hitting export cap, not inverter limit.", + "pv_kw": 4.0, # after inverter, 4kW net + "load_kw": 0.5, + "inverter_limit_kw": 10.0, # large inverter + "export_limit_kw": 3.0, # tight DNO export limit + "battery_size_kwh": 13.5, + "battery_soc_percent": 75, + "battery_rate_kw": 3.6, + "import_rate_p": 25.0, + "export_rate_p": 12.0, + "charge_target_percent": 100, + "hybrid": False, # AC-coupled + }, + { + "name": "Negative Import Rates", + "description": "Peak PV + negative import rates. Should the optimizer charge despite clipping risk?", + "pv_kw": 6.0, + "load_kw": 0.5, + "inverter_limit_kw": 5.0, + "export_limit_kw": 10.0, + "battery_size_kwh": 9.5, + "battery_soc_percent": 50, + "battery_rate_kw": 3.0, + "import_rate_p": -5.0, # NEGATIVE import rate (paid to consume) + "export_rate_p": 4.0, # low export rate + "charge_target_percent": 100, + "hybrid": True, + }, + { + "name": "Small Battery, Big Array", + "description": "4.8kWh battery with 10kW array and 5kW inverter. Battery fills quickly, lots of clipping.", + "pv_kw": 10.0, + "load_kw": 0.8, + "inverter_limit_kw": 5.0, + "export_limit_kw": 10.0, + "battery_size_kwh": 4.8, + "battery_soc_percent": 60, + "battery_rate_kw": 2.4, + "import_rate_p": 30.0, + "export_rate_p": 15.0, + "charge_target_percent": 100, + "hybrid": True, + }, + { + "name": "No Clipping Risk (Control)", + "description": "3kW panels with 5kW inverter. PV never exceeds limit. Penalty should not affect plan.", + "pv_kw": 3.0, + "load_kw": 0.5, + "inverter_limit_kw": 5.0, + "export_limit_kw": 10.0, + "battery_size_kwh": 9.5, + "battery_soc_percent": 50, + "battery_rate_kw": 3.0, + "import_rate_p": 25.0, + "export_rate_p": 15.0, + "charge_target_percent": 100, + "hybrid": True, + }, +] + + +def run_comparison(my_predbat): + """Run all clipping scenarios with and without the penalty, and compare.""" + + print("\n" + "=" * 100) + print("CLIPPING APPROACH COMPARISON: Cloud-Model Penalty vs Baseline") + print("=" * 100) + + results = [] + + for scenario in SCENARIOS: + print("\n" + "-" * 80) + print("Scenario: {}".format(scenario["name"])) + print(" {}".format(scenario["description"])) + print("-" * 80) + + # Common setup + reset_inverter(my_predbat) + reset_rates(my_predbat, scenario["import_rate_p"], scenario["export_rate_p"]) + + common_args = { + "pv_amount": scenario["pv_kw"], + "load_amount": scenario["load_kw"], + "inverter_limit": scenario["inverter_limit_kw"], + "export_limit": scenario["export_limit_kw"], + "battery_size": scenario["battery_size_kwh"], + "battery_soc": scenario["battery_size_kwh"] * scenario["battery_soc_percent"] / 100.0, + "battery_rate_max_charge": scenario["battery_rate_kw"], + "with_battery": True, + "hybrid": scenario["hybrid"], + "charge": scenario["battery_size_kwh"] * scenario["charge_target_percent"] / 100.0, + "save": "best", + "return_prediction_handle": True, + "ignore_failed": True, + "quiet": True, + } + + # --- Run A: Baseline (no clipping penalty) --- + t_start = time.perf_counter() + failed_a, pred_a = simple_scenario( + scenario["name"] + " [baseline]", + my_predbat, + assert_final_metric=0, + assert_final_soc=0, + clipping_peak_enable=False, + **common_args, + ) + time_a = time.perf_counter() - t_start + + # --- Run B: With clipping penalty --- + reset_inverter(my_predbat) + reset_rates(my_predbat, scenario["import_rate_p"], scenario["export_rate_p"]) + + t_start = time.perf_counter() + failed_b, pred_b = simple_scenario( + scenario["name"] + " [penalty]", + my_predbat, + assert_final_metric=0, + assert_final_soc=0, + clipping_peak_enable=True, + clipping_cost_weight=1.0, + clipping_peak_amplification=1.0, + **common_args, + ) + time_b = time.perf_counter() - t_start + + # Extract results + metric_a = round(pred_a.predict_metric_best[max(pred_a.predict_metric_best.keys())] / 100.0, 4) if pred_a.predict_metric_best else 0 + metric_b = round(pred_b.predict_metric_best[max(pred_b.predict_metric_best.keys())] / 100.0, 4) if pred_b.predict_metric_best else 0 + soc_a = round(list(pred_a.predict_soc.values())[-1], 2) if pred_a.predict_soc else 0 + soc_b = round(list(pred_b.predict_soc.values())[-1], 2) if pred_b.predict_soc else 0 + clipped_a = round(pred_a.predict_clipped_best[max(pred_a.predict_clipped_best.keys())], 2) if pred_a.predict_clipped_best else 0 + clipped_b = round(pred_b.predict_clipped_best[max(pred_b.predict_clipped_best.keys())], 2) if pred_b.predict_clipped_best else 0 + + result = { + "name": scenario["name"], + "metric_baseline": metric_a, + "metric_penalty": metric_b, + "metric_diff": round(metric_b - metric_a, 4), + "soc_baseline": soc_a, + "soc_penalty": soc_b, + "clipped_baseline": clipped_a, + "clipped_penalty": clipped_b, + "time_baseline": round(time_a, 3), + "time_penalty": round(time_b, 3), + } + results.append(result) + + print(" {:>20}: {:>10} {:>10} {:>10}".format("", "Baseline", "Penalty", "Diff")) + print(" {:>20}: {:>10.4f} {:>10.4f} {:>+10.4f}".format("Metric (£)", metric_a, metric_b, metric_b - metric_a)) + print(" {:>20}: {:>10.2f} {:>10.2f} {:>+10.2f}".format("Final SoC (kWh)", soc_a, soc_b, soc_b - soc_a)) + print(" {:>20}: {:>10.2f} {:>10.2f} {:>+10.2f}".format("Clipped (kWh)", clipped_a, clipped_b, clipped_b - clipped_a)) + print(" {:>20}: {:>10.3f}s {:>9.3f}s {:>+9.3f}s".format("Runtime", time_a, time_b, time_b - time_a)) + + # Summary table + print("\n" + "=" * 100) + print("SUMMARY TABLE") + print("=" * 100) + header = "{:<35} {:>10} {:>10} {:>10} {:>10} {:>10} {:>10}".format( + "Scenario", "Met_Base", "Met_Pen", "Met_Diff", "Clip_Base", "Clip_Pen", "Time_Pen" + ) + print(header) + print("-" * len(header)) + for r in results: + print("{:<35} {:>10.4f} {:>10.4f} {:>+10.4f} {:>10.2f} {:>10.2f} {:>9.3f}s".format( + r["name"][:35], + r["metric_baseline"], + r["metric_penalty"], + r["metric_diff"], + r["clipped_baseline"], + r["clipped_penalty"], + r["time_penalty"], + )) + + # Analysis + print("\n" + "=" * 100) + print("ANALYSIS") + print("=" * 100) + penalty_scenarios = [r for r in results if r["metric_diff"] > 0.001] + neutral_scenarios = [r for r in results if abs(r["metric_diff"]) <= 0.001] + + print("Scenarios where penalty changes metric: {} / {}".format(len(penalty_scenarios), len(results))) + print("Scenarios where penalty is neutral: {} / {}".format(len(neutral_scenarios), len(results))) + + if penalty_scenarios: + avg_diff = sum(r["metric_diff"] for r in penalty_scenarios) / len(penalty_scenarios) + print("Average metric increase (penalty scenarios): £{:+.4f}".format(avg_diff)) + print(" -> This represents the clipping cost the optimizer now accounts for.") + print(" -> In a full optimizer run, this would cause it to reduce charge targets.") + + avg_time_diff = sum(r["time_penalty"] - r["time_baseline"] for r in results) / len(results) + print("\nAverage compute time overhead: {:+.3f}s".format(avg_time_diff)) + + return results + + +if __name__ == "__main__": + # Bootstrap a minimal predbat instance for testing + # Run from project root: python apps\predbat\tests\compare_clipping.py + from unit_test import create_predbat + my_predbat = create_predbat() + run_comparison(my_predbat) From 27aecf64472ab311b92965508948d887e50945fb Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 14 Jun 2026 14:06:43 +0100 Subject: [PATCH 03/74] feat: Add clipping observability, clearsky integration and web chart to cloud model --- apps/predbat/config.py | 13 ++++ apps/predbat/fetch.py | 51 +++++++++++++++- apps/predbat/output.py | 49 +++++++++++++++ apps/predbat/plan.py | 32 +++++++--- apps/predbat/solcast.py | 128 +++++++++++++++++++++++++++++++++++----- apps/predbat/web.py | 81 +++++++++++++++++++++++++ 6 files changed, 329 insertions(+), 25 deletions(-) diff --git a/apps/predbat/config.py b/apps/predbat/config.py index 31ad5f4f5..febc51b84 100644 --- a/apps/predbat/config.py +++ b/apps/predbat/config.py @@ -1435,6 +1435,14 @@ "default": False, "icon": "mdi:chart-bell-curve-cumulative", }, + { + "name": "clipping_use_clearsky_peaks", + "friendly_name": "Clipping Use ClearSky Peaks", + "type": "switch", + "default": True, + "icon": "mdi:weather-sunny-alert", + "enable": "clipping_peak_enable", + }, { "name": "clipping_cost_weight", "friendly_name": "Clipping Cost Weight", @@ -2271,4 +2279,9 @@ "gateway_mqtt_host": {"type": "string", "empty": False}, "gateway_mqtt_port": {"type": "integer", "zero": False}, "gateway_mqtt_token": {"type": "string", "empty": False}, + "clipping_peak_enable": {"type": "boolean"}, + "clipping_use_clearsky_peaks": {"type": "boolean"}, + "clipping_cost_weight": {"type": "float"}, + "clipping_peak_amplification": {"type": "float"}, + "clipping_limit_override": {"type": "integer"}, } diff --git a/apps/predbat/fetch.py b/apps/predbat/fetch.py index 450495b55..750cd759c 100644 --- a/apps/predbat/fetch.py +++ b/apps/predbat/fetch.py @@ -1020,7 +1020,7 @@ def fetch_sensor_data(self, save=True): self.cost_today_sofar, self.carbon_today_sofar = self.today_cost(self.import_today, self.export_today, self.car_charging_energy, self.load_minutes, save=save) # Fetch PV forecast if enabled, today must be enabled, other days are optional - self.pv_forecast_minute, self.pv_forecast_minute10 = self.fetch_pv_forecast() + self.pv_forecast_minute, self.pv_forecast_minute10, self.pv_forecast_minute90, self.pv_forecast_minuteCS, self.pv_forecast_minuteMAX = self.fetch_pv_forecast() if self.load_minutes and not self.load_forecast_only: # Apply modal filter to historical data @@ -1233,11 +1233,21 @@ def fetch_pv_forecast(self): """ pv_forecast_minute = {} pv_forecast_minute10 = {} + pv_forecast_minute90 = {} + pv_forecast_minuteCS = {} + pv_forecast_minuteHIST = {} # Get data from forecast sensor entity_id = "sensor." + self.prefix + "_pv_forecast_raw" pv_forecast_packed_ld = self.get_state_wrapper(entity_id=entity_id, attribute="forecast") pv_forecast10_packed_ld = self.get_state_wrapper(entity_id=entity_id, attribute="forecast10") + pv_forecast90_packed_ld = self.get_state_wrapper(entity_id=entity_id, attribute="forecast90") + pv_forecastCS_packed_ld = self.get_state_wrapper(entity_id=entity_id, attribute="forecast_clearsky") + pv_forecastHIST_packed_ld = self.get_state_wrapper(entity_id=entity_id, attribute="forecast_historical") + if pv_forecastCS_packed_ld is None: + pv_forecastCS_packed_ld = self.get_state_wrapper(entity_id=entity_id, attribute="forecastCS") + if pv_forecastHIST_packed_ld is None: + pv_forecastHIST_packed_ld = self.get_state_wrapper(entity_id=entity_id, attribute="forecastMAX") relative_time = self.get_state_wrapper(entity_id=entity_id, attribute="relative_time") try: relative_time = datetime.strptime(relative_time, TIME_FORMAT) @@ -1248,6 +1258,9 @@ def fetch_pv_forecast(self): # Convert keys to integers and values to floats pv_forecast_packed = {} pv_forecast10_packed = {} + pv_forecast90_packed = {} + pv_forecastCS_packed = {} + pv_forecastHIST_packed = {} if pv_forecast_packed_ld: for key, value in pv_forecast_packed_ld.items(): @@ -1265,10 +1278,37 @@ def fetch_pv_forecast(self): except (ValueError, TypeError): pass + if pv_forecast90_packed_ld: + for key, value in pv_forecast90_packed_ld.items(): + try: + minute = int(key) + pv_forecast90_packed[minute] = float(value) + except (ValueError, TypeError): + pass + + if pv_forecastCS_packed_ld: + for key, value in pv_forecastCS_packed_ld.items(): + try: + minute = int(key) + pv_forecastCS_packed[minute] = float(value) + except (ValueError, TypeError): + pass + + if pv_forecastHIST_packed_ld: + for key, value in pv_forecastHIST_packed_ld.items(): + try: + minute = int(key) + pv_forecastHIST_packed[minute] = float(value) + except (ValueError, TypeError): + pass + # Unpack the forecast data max_minute = max(pv_forecast_packed.keys()) if pv_forecast_packed else 0 last_value = 0 last_value10 = 0 + last_value90 = 0 + last_valueCS = 0 + last_valueHIST = 0 # The forecast could be for a different time to our relative time, so we need to offset the minutes to align with our midnight_utc. # relative_time is the midnight at which the forecast was saved, so stored minute keys are relative to that midnight. # We subtract the offset so that stored minute X (= relative_time + X) maps to (relative_time + X - midnight_utc) minutes from today's midnight. @@ -1277,10 +1317,16 @@ def fetch_pv_forecast(self): target_minute = minute - minute_offset last_value = pv_forecast_packed.get(minute, last_value) last_value10 = pv_forecast10_packed.get(minute, last_value10) + last_value90 = pv_forecast90_packed.get(minute, last_value90) + last_valueCS = pv_forecastCS_packed.get(minute, last_valueCS) + last_valueHIST = pv_forecastHIST_packed.get(minute, last_valueHIST) pv_forecast_minute[target_minute] = last_value pv_forecast_minute10[target_minute] = last_value10 + pv_forecast_minute90[target_minute] = last_value90 + pv_forecast_minuteCS[target_minute] = last_valueCS + pv_forecast_minuteHIST[target_minute] = last_valueHIST - return pv_forecast_minute, pv_forecast_minute10 + return pv_forecast_minute, pv_forecast_minute10, pv_forecast_minute90, pv_forecast_minuteCS, pv_forecast_minuteHIST def predict_battery_temperature(self, battery_temperature_history, step): """ @@ -2312,6 +2358,7 @@ def fetch_config_options(self): # Clipping peak cost penalty model self.clipping_peak_enable = self.get_arg("clipping_peak_enable") + self.clipping_use_clearsky_peaks = self.get_arg("clipping_use_clearsky_peaks") self.clipping_cost_weight = self.get_arg("clipping_cost_weight") self.clipping_peak_amplification = self.get_arg("clipping_peak_amplification") self.clipping_limit_override = self.get_arg("clipping_limit_override") / MINUTE_WATT if self.get_arg("clipping_limit_override") else 0 diff --git a/apps/predbat/output.py b/apps/predbat/output.py index 0f864b222..29e5bd284 100644 --- a/apps/predbat/output.py +++ b/apps/predbat/output.py @@ -939,6 +939,13 @@ def short_textual_plan(self, soc_min, soc_min_minute, pv_forecast_minute_step, p export_type, self.duration_string(self.export_window_best[export_window_n_next]["start"] - self.minutes_now), self.get_rate_text(self.export_window_best[export_window_n_next]["start"], export=True, with_value=True) ) + # Clipping summary + predict_clipped_best = getattr(self, "predict_clipped_best", {}) + if predict_clipped_best: + clipping_total = predict_clipped_best.get(max(predict_clipped_best.keys()), 0.0) + if clipping_total > 0.01: + sentence += "- Forecast {} kWh clipping, plan penalized to mitigate.\n".format(dp2(clipping_total)) + if publish: self.text_plan = self.get_text_plan_html(sentence) @@ -2439,6 +2446,48 @@ def record_status(self, message, debug="", had_errors=False, notify=False, extra }, ) + # Clipping Status + clipping_status_text = "No clipping forecast." + clipping_mode = getattr(self, "clipping_limit_mode", "Unknown") + clipping_total = 0.0 + + predict_clipped_best = getattr(self, "predict_clipped_best", {}) + if predict_clipped_best: + clipping_total = predict_clipped_best.get(max(predict_clipped_best.keys()), 0.0) + if clipping_total > 0.01: + start_str = "" + end_str = "" + start_stamp = None + end_stamp = None + + # Find start time (first increase) + prev_val = 0.0 + for min_key, val in sorted(predict_clipped_best.items()): + if val > prev_val + 0.001: + if start_stamp is None: + start_stamp = self.midnight_utc + timedelta(minutes=min_key) + end_stamp = self.midnight_utc + timedelta(minutes=min_key) + prev_val = val + + if start_stamp and end_stamp: + start_str = start_stamp.strftime("%H:%M") + end_str = end_stamp.strftime("%H:%M") + + clipping_status_text = "Forecast {} kWh clipping, exceeding {} limit from {} to {}. Plan penalized to mitigate.".format( + dp2(clipping_total), clipping_mode, start_str, end_str + ) + + self.dashboard_item( + self.prefix + ".clipping_status", + state=clipping_status_text, + attributes={ + "friendly_name": "Clipping Status", + "icon": "mdi:solar-power", + "clipping_mode": clipping_mode, + "expected_total_kwh": dp2(clipping_total) + }, + ) + if had_errors: self.log("Warn: record_status {}".format(message + extra)) else: diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index 96090b9e7..c89ee30c5 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -964,26 +964,44 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): # Scaled by amplification factor for safety margin pv_forecast_peak_step = None clipping_limit_effective = 0 + clipping_limit_mode = "Unknown" if self.clipping_peak_enable: pv_forecast_peak_step = self.step_data_history( self.pv_forecast_minute, self.minutes_now, forward=True, cloud_factor=None ) - # Apply amplification factor - if self.clipping_peak_amplification != 1.0: + + # Apply ClearSky or Amplification factor + if getattr(self, "clipping_use_clearsky_peaks", False): + pv_clearsky_step = self.step_data_history( + getattr(self, "pv_forecast_minuteCS", {}), self.minutes_now, forward=True, cloud_factor=None + ) + pv_forecast_peak_step = { + k: max(v, pv_clearsky_step.get(k, 0)) for k, v in pv_forecast_peak_step.items() + } + elif self.clipping_peak_amplification != 1.0: pv_forecast_peak_step = {k: v * self.clipping_peak_amplification for k, v in pv_forecast_peak_step.items()} # Calculate effective clipping limit: most restrictive hardware constraint if self.clipping_limit_override > 0: clipping_limit_effective = self.clipping_limit_override + clipping_limit_mode = "Manual Override" else: limits = [] if self.inverter_limit > 0: - limits.append(self.inverter_limit) + limits.append((self.inverter_limit, "Inverter AC Capacity")) if self.export_limit > 0: - limits.append(self.export_limit) - if self.pv_ac_limit > 0: - limits.append(self.pv_ac_limit) - clipping_limit_effective = min(limits) if limits else 0 + limits.append((self.export_limit, "DNO Export Limit")) + if getattr(self, "pv_ac_limit", 0) > 0: + limits.append((self.pv_ac_limit, "PV AC Limit")) + + if limits: + clipping_limit_effective, clipping_limit_mode = min(limits, key=lambda x: x[0]) + else: + clipping_limit_effective = 0 + clipping_limit_mode = "No Limit" + + self.clipping_limit_effective = clipping_limit_effective + self.clipping_limit_mode = clipping_limit_mode # Save step data for debug self.load_minutes_step = load_minutes_step diff --git a/apps/predbat/solcast.py b/apps/predbat/solcast.py index c4d743057..49022acb9 100644 --- a/apps/predbat/solcast.py +++ b/apps/predbat/solcast.py @@ -330,7 +330,7 @@ async def download_open_meteo_data(self, configs=None): else: self.log("Warn: SolarAPI: Postcode {} could not be resolved to latitude and longitude, using default".format(postcode)) - url = "https://api.open-meteo.com/v1/forecast?latitude={lat}&longitude={lon}&hourly=global_tilted_irradiance,temperature_2m,wind_speed_10m&wind_speed_unit=ms&tilt={tilt}&azimuth={az}&forecast_days=4&timezone=UTC".format( + url = "https://api.open-meteo.com/v1/forecast?latitude={lat}&longitude={lon}&hourly=global_tilted_irradiance,clear_sky_gti,temperature_2m,wind_speed_10m&wind_speed_unit=ms&tilt={tilt}&azimuth={az}&forecast_days=4&timezone=UTC".format( lat=lat, lon=lon, tilt=tilt, az=az ) data = await self.cache_get_url(url, params={}, max_age=self.open_meteo_forecast_max_age * 60) @@ -341,6 +341,7 @@ async def download_open_meteo_data(self, configs=None): hourly = data.get("hourly", {}) times = hourly.get("time", []) gti_values = hourly.get("global_tilted_irradiance", []) + cs_gti_values = hourly.get("clear_sky_gti", []) temp_values = hourly.get("temperature_2m", []) wind_values = hourly.get("wind_speed_10m", []) @@ -353,7 +354,7 @@ async def download_open_meteo_data(self, configs=None): # Pass 1: compute instantaneous kW at each UTC timestamp sample. # Open-Meteo returns point-in-time irradiance (W/m²) at the start of each hour, # so we must integrate over the period rather than treating the sample as the period energy. - instant_kw = {} # datetime stamp -> (pv50_kw, pv10_kw) + instant_kw = {} # datetime stamp -> (pv50_kw, pv10_kw, pv_cs_kw) instant_stamps = [] for idx, ts in enumerate(times): if idx >= len(gti_values): @@ -361,6 +362,7 @@ async def download_open_meteo_data(self, configs=None): gti = gti_values[idx] if gti is None: gti = 0.0 + cs_gti = cs_gti_values[idx] if idx < len(cs_gti_values) and cs_gti_values[idx] is not None else gti temp = temp_values[idx] if idx < len(temp_values) and temp_values[idx] is not None else 25.0 wind = wind_values[idx] if idx < len(wind_values) and wind_values[idx] is not None else 1.0 # Cell temperature via SAPM/PVWatts model: irradiance heats the cell above ambient @@ -370,6 +372,7 @@ async def download_open_meteo_data(self, configs=None): # Cap at 1.1 (10% above STC) to prevent unrealistic gains at very cold temperatures. eta_temp = max(0.5, min(1.1, 1.0 - 0.004 * (t_cell - 25.0))) pv50_inst = dp4((gti / 1000.0) * kwp * eta_temp * (1.0 - system_loss)) + pv_cs_inst = dp4((cs_gti / 1000.0) * kwp * eta_temp * (1.0 - system_loss)) raw_p10 = ensemble_p10.get(ts) # ensemble_p10 was computed without temperature derating; apply eta_temp now pv10_inst = dp4(min(raw_p10 * eta_temp, pv50_inst) if raw_p10 is not None else pv50_inst * 0.7) @@ -378,7 +381,7 @@ async def download_open_meteo_data(self, configs=None): stamp = stamp.replace(tzinfo=pytz.utc) except (ValueError, TypeError): continue - instant_kw[stamp] = (pv50_inst, pv10_inst) + instant_kw[stamp] = (pv50_inst, pv10_inst, pv_cs_inst) instant_stamps.append(stamp) # Pass 2: trapezoidal integration — energy over [T, T+1h] = 0.5*(kW_at_T + kW_at_T+1h). @@ -389,21 +392,24 @@ async def download_open_meteo_data(self, configs=None): next_stamp = instant_stamps[i + 1] if (next_stamp - stamp) != timedelta(hours=1): continue - pv50_start, pv10_start = instant_kw[stamp] - pv50_end, pv10_end = instant_kw[next_stamp] + pv50_start, pv10_start, pv_cs_start = instant_kw[stamp] + pv50_end, pv10_end, pv_cs_end = instant_kw[next_stamp] pv50 = dp4(0.5 * (pv50_start + pv50_end)) pv10 = dp4(0.5 * (pv10_start + pv10_end)) + pv_cs = dp4(0.5 * (pv_cs_start + pv_cs_end)) # Apply per-month site shading correction from Google Solar API if available if shading_factors and len(shading_factors) == 12: shading_month = shading_factors[stamp.month - 1] pv50 = dp4(pv50 * shading_month) pv10 = dp4(pv10 * shading_month) + pv_cs = dp4(pv_cs * shading_month) - data_item = {"period_start": stamp.strftime(TIME_FORMAT), "pv_estimate": pv50, "pv_estimate10": pv10} + data_item = {"period_start": stamp.strftime(TIME_FORMAT), "pv_estimate": pv50, "pv_estimate10": pv10, "pv_clearsky": pv_cs} if stamp in period_data: period_data[stamp]["pv_estimate"] = dp4(period_data[stamp]["pv_estimate"] + pv50) period_data[stamp]["pv_estimate10"] = dp4(period_data[stamp]["pv_estimate10"] + pv10) + period_data[stamp]["pv_clearsky"] = dp4(period_data[stamp]["pv_clearsky"] + pv_cs) else: period_data[stamp] = data_item @@ -658,12 +664,14 @@ async def download_solcast_data(self): pv50 = forecast.get("pv_estimate", 0) / 60 * period_minutes pv10 = forecast.get("pv_estimate10", forecast.get("pv_estimate", 0)) / 60 * period_minutes pv90 = forecast.get("pv_estimate90", forecast.get("pv_estimate", 0)) / 60 * period_minutes + pv_cs = forecast.get("clearsky_estimate", pv90) / 60 * period_minutes - data_item = {"period_start": period_start_stamp.strftime(TIME_FORMAT), "pv_estimate": pv50, "pv_estimate10": pv10, "pv_estimate90": pv90} + data_item = {"period_start": period_start_stamp.strftime(TIME_FORMAT), "pv_estimate": pv50, "pv_estimate10": pv10, "pv_estimate90": pv90, "pv_clearsky": pv_cs} if period_start_stamp in period_data: period_data[period_start_stamp]["pv_estimate"] += pv50 period_data[period_start_stamp]["pv_estimate10"] += pv10 period_data[period_start_stamp]["pv_estimate90"] += pv90 + period_data[period_start_stamp]["pv_clearsky"] += pv_cs else: period_data[period_start_stamp] = data_item @@ -744,17 +752,20 @@ def publish_pv_stats(self, pv_forecast_data, divide_by, period): total_left_today10 = 0 total_left_today90 = 0 total_left_todayCL = 0 + total_left_todayCS = 0 forecast_day = {} total_day = {} total_day10 = {} total_day90 = {} total_dayCL = {} + total_dayCS = {} days = 0 for day in range(7): total_day[day] = 0 total_day10[day] = 0 total_day90[day] = 0 total_dayCL[day] = 0 + total_dayCS[day] = 0 forecast_day[day] = [] midnight_today = self.midnight_utc @@ -765,6 +776,7 @@ def publish_pv_stats(self, pv_forecast_data, divide_by, period): power_now10 = 0 power_now90 = 0 power_nowCL = 0 + power_nowCS = 0 point_gap = period for entry in pv_forecast_data: @@ -782,6 +794,7 @@ def publish_pv_stats(self, pv_forecast_data, divide_by, period): total_day10[day] = 0 total_day90[day] = 0 total_dayCL[day] = 0 + total_dayCS[day] = 0 forecast_day[day] = [] days = max(days, day + 1) @@ -789,22 +802,26 @@ def publish_pv_stats(self, pv_forecast_data, divide_by, period): pv_estimate10 = entry.get("pv_estimate10", pv_estimate) pv_estimate90 = entry.get("pv_estimate90", pv_estimate) pv_estimateCL = entry.get("pv_estimateCL", pv_estimate) + pv_clearsky = entry.get("pv_clearsky", pv_estimate90) pv_estimate /= divide_by pv_estimate10 /= divide_by pv_estimate90 /= divide_by pv_estimateCL /= divide_by + pv_clearsky /= divide_by total_day[day] += pv_estimate total_day10[day] += pv_estimate10 total_day90[day] += pv_estimate90 total_dayCL[day] += pv_estimateCL + total_dayCS[day] += pv_clearsky if day == 0 and this_point > now: total_left_today += pv_estimate total_left_today10 += pv_estimate10 total_left_today90 += pv_estimate90 total_left_todayCL += pv_estimateCL + total_left_todayCS += pv_clearsky next_point = this_point + timedelta(minutes=point_gap) if this_point <= now and next_point > now: @@ -812,6 +829,7 @@ def publish_pv_stats(self, pv_forecast_data, divide_by, period): power_now10 = pv_estimate10 * power_scale power_now90 = pv_estimate90 * power_scale power_nowCL = pv_estimateCL * power_scale + power_nowCS = pv_clearsky * power_scale # Add this slot into the total left today but scaled for the time since this point if day == 0: @@ -820,6 +838,7 @@ def publish_pv_stats(self, pv_forecast_data, divide_by, period): total_left_today10 += pv_estimate10 * left_this_slot_scale total_left_today90 += pv_estimate90 * left_this_slot_scale total_left_todayCL += pv_estimateCL * left_this_slot_scale + total_left_todayCS += pv_clearsky * left_this_slot_scale fentry = { "period_start": entry["period_start"], @@ -827,6 +846,7 @@ def publish_pv_stats(self, pv_forecast_data, divide_by, period): "pv_estimate10": dp2(pv_estimate10 * power_scale), "pv_estimate90": dp2(pv_estimate90 * power_scale), "pv_estimateCL": dp2(pv_estimateCL * power_scale), + "pv_clearsky": dp2(pv_clearsky * power_scale), } forecast_day[day].append(fentry) @@ -836,15 +856,17 @@ def publish_pv_stats(self, pv_forecast_data, divide_by, period): for day in range(days): if day == 0: self.log( - "SolarAPI: PV Forecast for today is {} ({} 10%, {} 90%, {} calibrated) kWh, and PV left today is {} ({} 10%, {} 90%, {} calibrated) kWh".format( + "SolarAPI: PV Forecast for today is {} ({} 10%, {} 90%, {} calibrated, {} clearsky) kWh, and PV left today is {} ({} 10%, {} 90%, {} calibrated, {} clearsky) kWh".format( dp2(total_day[day]), dp2(total_day10[day]), dp2(total_day90[day]), dp2(total_dayCL[day]), + dp2(total_dayCS[day]), dp2(total_left_today), dp2(total_left_today10), dp2(total_left_today90), dp2(total_left_todayCL), + dp2(total_left_todayCS), ) ) self.dashboard_item( @@ -860,10 +882,12 @@ def publish_pv_stats(self, pv_forecast_data, divide_by, period): "total10": dp2(total_day10[day]), "total90": dp2(total_day90[day]), "totalCL": dp2(total_dayCL[day]), + "totalCS": dp2(total_dayCS[day]), "remaining": dp2(total_left_today), "remaining10": dp2(total_left_today10), "remaining90": dp2(total_left_today90), "remainingCL": dp2(total_left_todayCL), + "remainingCS": dp2(total_left_todayCS), "detailedForecast": forecast_day[day], }, app="solar", @@ -881,18 +905,19 @@ def publish_pv_stats(self, pv_forecast_data, divide_by, period): "now10": dp2(power_now10), "now90": dp2(power_now90), "nowCL": dp2(power_nowCL), + "nowCS": dp2(power_nowCS), "remaining": dp2(total_left_today), "remaining10": dp2(total_left_today10), "remaining90": dp2(total_left_today90), "remainingCL": dp2(total_left_todayCL), + "remainingCS": dp2(total_left_todayCS), }, app="solar", ) else: day_name = "tomorrow" if day == 1 else "d{}".format(day) day_name_long = day_name if day == 1 else "day {}".format(day) - self.log("SolarAPI: PV Forecast for day {} is {} ({} 10%, {} 90%, {} calibrated) kWh".format(day_name, dp2(total_day[day]), dp2(total_day10[day]), dp2(total_day90[day]), dp2(total_dayCL[day]))) - + self.log("SolarAPI: PV Forecast for day {} is {} ({} 10%, {} 90%, {} calibrated, {} clearsky) kWh".format(day_name, dp2(total_day[day]), dp2(total_day10[day]), dp2(total_day90[day]), dp2(total_dayCL[day]), dp2(total_dayCS[day]))) self.dashboard_item( "sensor." + self.prefix + "_pv_" + day_name, state=dp2(total_dayCL[day] if calibration_on else total_day[day]), @@ -906,6 +931,7 @@ def publish_pv_stats(self, pv_forecast_data, divide_by, period): "total10": dp2(total_day10[day]), "total90": dp2(total_day90[day]), "totalCL": dp2(total_dayCL[day]), + "totalCS": dp2(total_dayCS[day]), "detailedForecast": forecast_day[day], }, app="solar", @@ -1142,6 +1168,7 @@ def pv_calibration(self, pv_forecast_minute, pv_forecast_minute10, pv_forecast_d pv_estimateCL = {} pv_estimate10 = {} pv_estimate90 = {} + pv_historical = {} # The after scaling cap will be applied, but remember that the input data is # When we have a valid observed peak (from history or forecast history) cap to the lower of # the inverter rating and that observed peak. With no valid history (e.g. all days excluded @@ -1153,6 +1180,10 @@ def pv_calibration(self, pv_forecast_minute, pv_forecast_minute10, pv_forecast_d capped_data = min(max_kwh_cap, observed_cap) else: capped_data = max_kwh_cap + + # Historical max curve + peak_hist_avg = max(pv_power_hist_by_slot.values()) if pv_power_hist_by_slot else 0 + hist_max_scaling = max_pv_power_hist / peak_hist_avg if peak_hist_avg > 0 else 1.0 for minute in range(0, max(pv_forecast_minute.keys()) + 1, self.plan_interval_minutes): pv_value = 0 for offset in range(0, self.plan_interval_minutes, 1): @@ -1162,6 +1193,10 @@ def pv_calibration(self, pv_forecast_minute, pv_forecast_minute10, pv_forecast_d pv_estimate10[minute] = dp4(min(pv_value * worst_day_scaling, capped_data)) pv_estimate90[minute] = dp4(min(pv_value * best_day_scaling, capped_data)) + slot = (int(minute / self.plan_interval_minutes) * self.plan_interval_minutes) % (24 * 60) + pv_max = pv_power_hist_by_slot.get(slot, 0) * hist_max_scaling + pv_historical[minute] = dp4(min(pv_max / 60 * self.plan_interval_minutes, capped_data)) + for entry in pv_forecast_data: period_start = entry.get("period_start", "") if period_start: @@ -1175,9 +1210,11 @@ def pv_calibration(self, pv_forecast_minute, pv_forecast_minute10, pv_forecast_d calibrated = 0 calibrated10 = 0 calibrated90 = 0 + calibratedMAX = 0 has_calibrated = False has_calibrated10 = False has_calibrated90 = False + has_calibratedMAX = False for i in range(slots_per_period): s = slot + i * self.plan_interval_minutes v = pv_estimateCL.get(s, None) @@ -1192,10 +1229,16 @@ def pv_calibration(self, pv_forecast_minute, pv_forecast_minute10, pv_forecast_d if v90 is not None: calibrated90 += v90 has_calibrated90 = True + vMAX = pv_historical.get(s, None) + if vMAX is not None: + calibratedMAX += vMAX + has_calibratedMAX = True # When we store the data we have to reverse the divide_by factor if has_calibrated: entry["pv_estimateCL"] = calibrated * divide_by + if has_calibratedMAX: + entry["pv_historical"] = calibratedMAX * divide_by if create_pv10 and has_calibrated10: entry["pv_estimate10"] = calibrated10 * divide_by if create_pv10 and has_calibrated90: @@ -1212,26 +1255,54 @@ def pv_calibration(self, pv_forecast_minute, pv_forecast_minute10, pv_forecast_d # Do we use calibrated or raw data? if self.get_arg("metric_pv_calibration_enable", default=True): self.log("SolarAPI: PV Calibration: Using calibrated PV data") - return pv_forecast_minute_adjusted, pv_forecast_minute10, pv_forecast_data + return pv_forecast_minute_adjusted, pv_forecast_minute10, pv_forecast_data, pv_historical else: - return pv_forecast_minute, pv_forecast_minute10, pv_forecast_data + return pv_forecast_minute, pv_forecast_minute10, pv_forecast_data, pv_historical - def pack_and_store_forecast(self, pv_forecast_minute, pv_forecast_minute10): + def pack_and_store_forecast(self, pv_forecast_minute, pv_forecast_minute10, pv_forecast_minute90=None, pv_clearsky_minute=None, pv_max_minute=None): pv_forecast_pack = {} pv_forecast_pack10 = {} - + pv_forecast_pack90 = {} + pv_forecast_pack_clearsky = {} + pv_forecast_pack_historical = {} prev_value = -1 prev_value10 = -1 + prev_value90 = -1 + prev_value_clearsky = -1 + prev_value_historical = -1 + + # Pre-fill dictionaries to ensure interpolation for packing + def get_interp_val(data, m): + if not data: + return 0 + if m in data: + return data[m] + # Use plan_interval fallback + last_tick = (m // self.plan_interval_minutes) * self.plan_interval_minutes + return data.get(last_tick, 0) for minute in range(0, self.forecast_days * 24 * 60): current_value = dp4(pv_forecast_minute.get(minute, 0)) current_value10 = dp4(pv_forecast_minute10.get(minute, 0)) + current_value90 = dp4(get_interp_val(pv_forecast_minute90, minute)) + current_value_clearsky = dp4(get_interp_val(pv_clearsky_minute, minute)) + current_value_historical = dp4(get_interp_val(pv_max_minute, minute)) + if current_value != prev_value: pv_forecast_pack[minute] = current_value prev_value = current_value if current_value10 != prev_value10: pv_forecast_pack10[minute] = current_value10 prev_value10 = current_value10 + if current_value90 != prev_value90: + pv_forecast_pack90[minute] = current_value90 + prev_value90 = current_value90 + if current_value_clearsky != prev_value_clearsky: + pv_forecast_pack_clearsky[minute] = current_value_clearsky + prev_value_clearsky = current_value_clearsky + if current_value_historical != prev_value_historical: + pv_forecast_pack_historical[minute] = current_value_historical + prev_value_historical = current_value_historical current_pv_power = dp4(pv_forecast_minute.get(self.minutes_now, 0)) @@ -1244,6 +1315,9 @@ def pack_and_store_forecast(self, pv_forecast_minute, pv_forecast_minute10): "relative_time": self.midnight_utc.strftime(TIME_FORMAT), "forecast": pv_forecast_pack, "forecast10": pv_forecast_pack10, + "forecast90": pv_forecast_pack90, + "forecast_clearsky": pv_forecast_pack_clearsky, + "forecast_historical": pv_forecast_pack_historical, "unit_of_measurement": "kW", "device_class": "power", "state_class": "measurement", @@ -1364,11 +1438,33 @@ async def fetch_pv_forecast(self): scale=self.pv_scaling, spreading=period, ) + pv_forecast_minute90, _ = minute_data( + pv_forecast_data, + self.forecast_days, + self.midnight_utc, + "pv_estimate90", + "period_start", + backwards=False, + divide_by=divide_by, + scale=self.pv_scaling, + spreading=period, + ) + pv_clearsky_minute, _ = minute_data( + pv_forecast_data, + self.forecast_days, + self.midnight_utc, + "pv_clearsky", + "period_start", + backwards=False, + divide_by=divide_by, + scale=self.pv_scaling, + spreading=period, + ) # Run calibration on the data - pv_forecast_minute, pv_forecast_minute10, pv_forecast_data = self.pv_calibration(pv_forecast_minute, pv_forecast_minute10, pv_forecast_data, create_pv10, divide_by / period, max_kwh, self.forecast_days, period) + pv_forecast_minute, pv_forecast_minute10, pv_forecast_data, pv_max_minute = self.pv_calibration(pv_forecast_minute, pv_forecast_minute10, pv_forecast_data, create_pv10, divide_by / period, max_kwh, self.forecast_days, period) self.publish_pv_stats(pv_forecast_data, divide_by / period, period) - self.pack_and_store_forecast(pv_forecast_minute, pv_forecast_minute10) + self.pack_and_store_forecast(pv_forecast_minute, pv_forecast_minute10, pv_forecast_minute90, pv_clearsky_minute, pv_max_minute) self.update_success_timestamp() self.last_fetched_timestamp = self.now_utc_exact else: diff --git a/apps/predbat/web.py b/apps/predbat/web.py index 80514cfef..4e88c86cd 100644 --- a/apps/predbat/web.py +++ b/apps/predbat/web.py @@ -2979,6 +2979,86 @@ def get_chart(self, chart): {"name": "PV Actual", "data": pv_actual, "opacity": "1.0", "stroke_width": "3", "stroke_curve": "stepline", "color": "#f5c43d"}, ] text += self.render_chart(series_data, "kWh", "PV Forecast vs Actual", now_str) + elif chart == "Clipping": + clipping_mode = getattr(self.base, "clipping_limit_mode", "Unknown") + clipping_limit_effective = getattr(self.base, "clipping_limit_effective", 0) + inverter_ac_limit_kw = getattr(self.base, "inverter_limit", 0) + + # Re-fetch PV actuals for overlay + pv_power_hist = history_attribute(self.get_history_wrapper(self.prefix + ".pv_power", 7, required=False)) + pv_power = prune_today(pv_power_hist, self.now_utc, self.midnight_utc, prune=False) + + axis_max = 12.0 + if self.base.soc_max > 12.0 or inverter_ac_limit_kw > 12.0: + axis_max = max(self.base.soc_max, inverter_ac_limit_kw, 12.0) + axis_ticks = 6 + + annotations = [] + if clipping_limit_effective > 0: + annotations.append({"y": clipping_limit_effective, "text": "{} ({} kW)".format(clipping_mode, round(clipping_limit_effective, 2)), "color": "#FF0000"}) + if inverter_ac_limit_kw > 0 and abs(inverter_ac_limit_kw - clipping_limit_effective) > 0.1: + annotations.append({"y": inverter_ac_limit_kw, "text": "Inverter Capacity ({} kW)".format(round(inverter_ac_limit_kw, 2)), "color": "#999999"}) + + # Data series parsing for per-minute data + step_size = getattr(self.base, "plan_interval_minutes", 30) + clipping_forecast_series = {} + if getattr(self.base, "predict_clipped_best", None): + for minute, kwh in self.base.predict_clipped_best.items(): + if minute % step_size == 0: + minute_timestamp = self.midnight_utc + timedelta(minutes=minute) + stamp = minute_timestamp.strftime(TIME_FORMAT) + clipping_forecast_series[stamp] = round(kwh, 2) + + # Raw PV forecast (ClearSky if used, else regular) + raw_pv_series = {} + if getattr(self.base, "clipping_use_clearsky_peaks", False): + raw_pv_data = getattr(self.base, "pv_forecast_minuteCS", {}) + raw_pv_name = "ClearSky Forecast" + else: + raw_pv_data = getattr(self.base, "pv_forecast_minute", {}) + raw_pv_name = "Base Forecast" + if raw_pv_data: + for minute, kw in raw_pv_data.items(): + if minute % step_size == 0: + minute_timestamp = self.midnight_utc + timedelta(minutes=minute) + stamp = minute_timestamp.strftime(TIME_FORMAT) + raw_pv_series[stamp] = round(kw, 2) + + series_data = [ + {"name": "Actual SOC", "data": soc_kw_h0, "opacity": "1.0", "stroke_width": "2", "stroke_curve": "smooth", "color": "#3291a8", "unit": "kWh"}, + {"name": "Target SOC", "data": soc_kw_best, "opacity": "1.0", "stroke_width": "3", "stroke_curve": "stepline", "color": "#9b23eb", "unit": "kWh"}, + {"name": "PV Power Actual", "data": pv_power, "opacity": "1.0", "stroke_width": "3", "stroke_curve": "smooth", "color": "#f5c43d", "unit": "kW"}, + {"name": raw_pv_name, "data": raw_pv_series, "opacity": "0.3", "stroke_width": "2", "stroke_curve": "smooth", "chart_type": "area", "color": "#a8a8a7", "unit": "kW"}, + {"name": "Forecast Clipping Total", "data": clipping_forecast_series, "opacity": "1.0", "stroke_width": "2", "stroke_curve": "smooth", "chart_type": "area", "color": "#FF0000", "unit": "kWh"}, + ] + + secondary_axis = [ + { + "title": "kW", + "series_name": "PV Power Actual", + "decimals": 1, + "opposite": True, + "min": 0, + "max": axis_max, + "tickAmount": axis_ticks, + }, + { + "title": "kW", + "series_name": raw_pv_name, + "show": False, + "min": 0, + "max": axis_max, + "tickAmount": axis_ticks, + }, + ] + + clipping_total = 0 + if getattr(self.base, "predict_clipped_best", None): + clipping_total = self.base.predict_clipped_best.get(max(self.base.predict_clipped_best.keys()), 0.0) + + chart_title = "Clipping Analysis (Expected Total Clipping: {:.2f} kWh)".format(clipping_total) + + text += self.render_chart(series_data, "kWh", chart_title, now_str, yaxis_annotations=annotations, extra_yaxis=secondary_axis, yaxis_min=0, yaxis_max=axis_max, yaxis_tick_amount=axis_ticks) elif chart == "LoadML": load_today_history = self.get_history_with_now_attrs("sensor." + self.prefix + "_load_ml_stats", 7) # Get historical load data for last 24 hours @@ -3280,6 +3360,7 @@ async def html_charts(self, request): text += f'PV' text += f'PV7' text += f'PVAccuracy' + text += f'Clipping' text += f'Savings' text += f'BatteryDegradation' text += f'MarginalCosts' From 4b49d45b59b2b375dcf343fb7260d488620f8aca Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 14 Jun 2026 15:05:40 +0100 Subject: [PATCH 04/74] feat: add pv_forecast_primary and clipping_clearsky_source parameters for flexible clearsky overlay --- apps/predbat/solcast.py | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/apps/predbat/solcast.py b/apps/predbat/solcast.py index 49022acb9..4e99ed411 100644 --- a/apps/predbat/solcast.py +++ b/apps/predbat/solcast.py @@ -1339,7 +1339,9 @@ async def fetch_pv_forecast(self): max_kwh = 9999 using_ha_data = False - if self.forecast_solar: + pv_forecast_primary = self.get_arg("pv_forecast_primary", "auto", indirect=False) + + if (pv_forecast_primary == "forecast_solar") or (pv_forecast_primary == "auto" and self.forecast_solar): self.log("SolarAPI: Obtaining solar forecast from Forecast Solar API") pv_forecast_data, max_kwh = await self.download_forecast_solar_data() divide_by = 30.0 @@ -1348,12 +1350,12 @@ async def fetch_pv_forecast(self): self.log("SolarAPI: Forecast Solar returned no data, falling back to Open-Meteo backup") backup_configs = self.open_meteo_forecast if self.open_meteo_forecast else self.forecast_solar pv_forecast_data, max_kwh = await self.download_open_meteo_data(configs=backup_configs) - elif self.open_meteo_forecast: + elif (pv_forecast_primary == "openmeteo") or (pv_forecast_primary == "auto" and self.open_meteo_forecast): self.log("SolarAPI: Obtaining solar forecast from Open-Meteo API") pv_forecast_data, max_kwh = await self.download_open_meteo_data() divide_by = 30.0 create_pv10 = True - elif self.solcast_host and self.solcast_api_key: + elif (pv_forecast_primary == "solcast_api") or (pv_forecast_primary == "auto" and self.solcast_host and self.solcast_api_key): self.log("SolarAPI: Obtaining solar forecast from Solcast API") pv_forecast_data = await self.download_solcast_data() divide_by = 30.0 @@ -1389,6 +1391,29 @@ async def fetch_pv_forecast(self): self.log("SolarAPI: PV Forecast today adds up to {} kWh, and total sensors add up to {} kWh, factor is {}".format(pv_forecast_total_data, pv_forecast_total_sensor, factor)) if pv_forecast_data: + # Optional overlay of ClearSky data from a secondary source + clipping_clearsky_source = self.get_arg("clipping_clearsky_source", "auto", indirect=False) + if clipping_clearsky_source == "openmeteo" and self.open_meteo_forecast: + self.log("SolarAPI: Overlaying ClearSky data from Open-Meteo API") + om_data, _ = await self.download_open_meteo_data() + if om_data: + om_lookup = {item["period_start"]: item.get("pv_clearsky", item.get("pv_estimate90", 0)) for item in om_data} + for item in pv_forecast_data: + ts = item.get("period_start") + if ts in om_lookup: + item["pv_clearsky"] = om_lookup[ts] + elif clipping_clearsky_source == "solcast_api" and self.solcast_api_key: + self.log("SolarAPI: Overlaying ClearSky data from Solcast API") + sol_data = await self.download_solcast_data() + if sol_data: + sol_lookup = {item["period_start"]: item.get("pv_clearsky", item.get("pv_estimate90", 0)) for item in sol_data} + for item in pv_forecast_data: + ts = item.get("period_start") + if ts in sol_lookup: + item["pv_clearsky"] = sol_lookup[ts] + elif clipping_clearsky_source not in ["auto", "base", ""]: + self.log("Warn: SolarAPI: clipping_clearsky_source '{}' not configured properly, using base data".format(clipping_clearsky_source)) + # Detect the actual period of the forecast data (e.g. 15 or 30 minutes) # by examining the time difference between consecutive entries. # This ensures 15-minute resolution data is handled correctly. From f09c9a00ce0d90867606591a8629a83b150650f8 Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 14 Jun 2026 16:20:30 +0100 Subject: [PATCH 05/74] feat: hybrid clipping buffer logic with cloud model parity --- apps/predbat/config.py | 33 ++++++ apps/predbat/plan.py | 114 ++++++++++++++++++ apps/predbat/predbat.py | 8 ++ apps/predbat/prediction.py | 28 ++++- apps/predbat/tests/test_fetch_pv_forecast.py | 12 +- apps/predbat/tests/test_solcast.py | 8 +- apps/predbat/web.py | 117 +++++++++++++++++++ 7 files changed, 305 insertions(+), 15 deletions(-) diff --git a/apps/predbat/config.py b/apps/predbat/config.py index febc51b84..f3843b584 100644 --- a/apps/predbat/config.py +++ b/apps/predbat/config.py @@ -1479,6 +1479,36 @@ "enable": "clipping_peak_enable", "default": 0, }, + { + "name": "clipping_buffer_max_kwh", + "friendly_name": "Clipping Buffer Max kWh (Manual Override)", + "type": "input_number", + "min": 0, + "max": 50.0, + "step": 0.1, + "unit": "kWh", + "icon": "mdi:battery-50", + "default": 0, + "enable": "clipping_peak_enable", + }, + { + "name": "clipping_buffer_start_time", + "friendly_name": "Clipping Buffer Start Time", + "type": "select", + "options": ["None"] + OPTIONS_TIME, + "icon": "mdi:clock-start", + "default": "None", + "enable": "clipping_peak_enable", + }, + { + "name": "clipping_buffer_end_time", + "friendly_name": "Clipping Buffer End Time", + "type": "select", + "options": ["None"] + OPTIONS_TIME, + "icon": "mdi:clock-end", + "default": "None", + "enable": "clipping_peak_enable", + }, { "name": "inverter_limit_override", "friendly_name": "Inverter Limit Override", @@ -2284,4 +2314,7 @@ "clipping_cost_weight": {"type": "float"}, "clipping_peak_amplification": {"type": "float"}, "clipping_limit_override": {"type": "integer"}, + "clipping_buffer_max_kwh": {"type": "float"}, + "clipping_buffer_start_time": {"type": "string"}, + "clipping_buffer_end_time": {"type": "string"}, } diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index c89ee30c5..002b25f4f 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -1003,6 +1003,47 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): self.clipping_limit_effective = clipping_limit_effective self.clipping_limit_mode = clipping_limit_mode + # Hybrid Clipping: Read Manual Overrides + self.clipping_buffer_kwh = float(self.get_arg("clipping_buffer_max_kwh", default=0.0)) + + start_time_str = self.get_arg("clipping_buffer_start_time", default="None") + end_time_str = self.get_arg("clipping_buffer_end_time", default="None") + + self.clipping_buffer_start = None + if start_time_str and start_time_str != "None": + self.clipping_buffer_start = self.time_to_minutes(start_time_str) + + self.clipping_buffer_end = None + if end_time_str and end_time_str != "None": + self.clipping_buffer_end = self.time_to_minutes(end_time_str) + + # Calculate Implicit Buffer (Physics-based Decay Curve) + self.clipping_buffer_forecast_kwh = {} + self.clipping_remaining_today = 0.0 + self.clipping_tomorrow = 0.0 + + if self.clipping_peak_enable and pv_forecast_peak_step and clipping_limit_effective > 0: + step_size = getattr(self, "plan_interval_minutes", 30) + # Accumulate potential clipping loss per interval + for minute in range(0, self.forecast_minutes, step_size): + kwh_loss = 0.0 + for m in range(minute, min(minute + step_size, self.forecast_minutes)): + pv_power = pv_forecast_peak_step.get(m, 0) + if pv_power > clipping_limit_effective: + kwh_loss += (pv_power - clipping_limit_effective) / 60.0 + + if kwh_loss > 0: + self.clipping_buffer_forecast_kwh[minute] = kwh_loss + # Add to totals + if minute < 1440: + self.clipping_remaining_today += kwh_loss + else: + self.clipping_tomorrow += kwh_loss + + # If manual override is active, reflect it in the UI totals + if self.clipping_buffer_kwh > 0: + self.clipping_remaining_today = max(self.clipping_remaining_today, self.clipping_buffer_kwh) + # Save step data for debug self.load_minutes_step = load_minutes_step self.load_minutes_step10 = load_minutes_step10 @@ -3861,6 +3902,79 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi "icon": "mdi:currency-usd", }, ) + + # Add Clipping Summary Dashboard Items + self.dashboard_item( + self.prefix + ".clipping_remaining_today", + state=dp2(self.clipping_remaining_today), + attributes={ + "friendly_name": "Clipping Remaining Today", + "unit_of_measurement": "kWh", + "device_class": "energy", + "icon": "mdi:solar-power-variant", + }, + ) + self.dashboard_item( + self.prefix + ".clipping_tomorrow", + state=dp2(self.clipping_tomorrow), + attributes={ + "friendly_name": "Clipping Forecast Tomorrow", + "unit_of_measurement": "kWh", + "device_class": "energy", + "icon": "mdi:solar-power-variant-outline", + }, + ) + self.dashboard_item( + self.prefix + ".clipping_mitigated_today", + state=dp2(getattr(self, "clipping_mitigated_today", 0.0)), + attributes={ + "friendly_name": "Clipping Mitigated Today", + "unit_of_measurement": "kWh", + "device_class": "energy", + "icon": "mdi:battery-check", + }, + ) + + clipping_status_text = "No clipping forecast." + clipping_start_iso = None + clipping_end_iso = None + + if self.clipping_buffer_kwh > 0: + def format_time_human(minute): + if minute is None: + return "N/A" + target_dt = self.midnight + timedelta(minutes=minute) + if target_dt.date() == self.midnight.date(): + return target_dt.strftime("%H:%M") + else: + return target_dt.strftime("Tomorrow %H:%M") + start_str = format_time_human(self.clipping_buffer_start) + end_str = format_time_human(self.clipping_buffer_end) + if self.clipping_buffer_start is not None: + clipping_start_iso = (self.midnight_utc + timedelta(minutes=self.clipping_buffer_start)).isoformat() + if self.clipping_buffer_end is not None: + clipping_end_iso = (self.midnight_utc + timedelta(minutes=self.clipping_buffer_end)).isoformat() + + if self.clipping_buffer_start is not None and self.clipping_buffer_end is not None: + clipping_status_text = "{} kWh clipping forecast ({}) between {} and {}.".format(dp2(self.clipping_buffer_kwh), self.clipping_mode, start_str, end_str) + else: + clipping_status_text = "{} kWh manual clipping buffer override active.".format(dp2(self.clipping_buffer_kwh)) + + self.dashboard_item( + self.prefix + ".clipping_status", + state=clipping_status_text, + attributes={ + "friendly_name": "Clipping Buffer Status", + "icon": "mdi:information-outline", + "results": self.filtered_times(self.clipping_buffer_forecast_kwh), + "clipping_start": clipping_start_iso, + "clipping_end": clipping_end_iso, + "clipping_mode": self.clipping_mode, + "clipping_remaining_today": dp2(self.clipping_remaining_today), + "clipping_tomorrow": dp2(self.clipping_tomorrow), + "clipping_mitigated_today": dp2(getattr(self, "clipping_mitigated_today", 0.0)), + }, + ) self.dashboard_item(self.prefix + ".record", state=0.0, attributes={"results": self.filtered_times(record_time), "friendly_name": "Prediction window", "state_class": "measurement"}) self.dashboard_item( self.prefix + ".iboost_best", diff --git a/apps/predbat/predbat.py b/apps/predbat/predbat.py index c6fb32ef6..46e31d27d 100644 --- a/apps/predbat/predbat.py +++ b/apps/predbat/predbat.py @@ -644,6 +644,14 @@ def reset(self): self.isCharging_Target = 0 self.isExporting = False self.isExporting_Target = 0 + self.clipping_buffer_kwh = 0 + self.clipping_buffer_forecast_kwh = {} + self.clipping_buffer_start = None + self.clipping_buffer_end = None + self.clipping_remaining_today = 0.0 + self.clipping_tomorrow = 0.0 + self.clipping_mitigated_today = 0.0 + self.clipping_mode = "Auto" self.savings_today_predbat = 0.0 self.savings_today_predbat_soc = 0.0 self.savings_today_pvbat = 0.0 diff --git a/apps/predbat/prediction.py b/apps/predbat/prediction.py index 8cf4ae85c..c04526b84 100644 --- a/apps/predbat/prediction.py +++ b/apps/predbat/prediction.py @@ -92,7 +92,7 @@ class Prediction: Class to hold prediction input and output data and the run function """ - def __init__(self, base=None, pv_forecast_minute_step=None, pv_forecast_minute10_step=None, load_minutes_step=None, load_minutes_step10=None, soc_kw=None, soc_max=None, pv_forecast_peak_step=None, clipping_limit=0, clipping_cost_weight=0): + def __init__(self, base=None, pv_forecast_minute_step=None, pv_forecast_minute10_step=None, load_minutes_step=None, load_minutes_step10=None, soc_kw=None, soc_max=None, pv_forecast_peak_step=None, clipping_limit=0, clipping_cost_weight=0, clipping_buffer_kwh=0, clipping_buffer_start=None, clipping_buffer_end=None): global PRED_GLOBAL if base: self.minutes_now = base.minutes_now @@ -181,6 +181,12 @@ def __init__(self, base=None, pv_forecast_minute_step=None, pv_forecast_minute10 self.rate_export = base.rate_export self.io_adjusted = base.io_adjusted self.rate_max = base.rate_max + self.clipping_limit = clipping_limit + self.clipping_cost_weight = clipping_cost_weight + self.clipping_buffer_kwh = clipping_buffer_kwh + self.clipping_buffer_start = clipping_buffer_start + self.clipping_buffer_end = clipping_buffer_end + self.pv_forecast_peak_step = pv_forecast_peak_step self.pv_forecast_minute_step = pv_forecast_minute_step self.pv_forecast_minute10_step = pv_forecast_minute10_step self.load_minutes_step = load_minutes_step @@ -195,9 +201,6 @@ def __init__(self, base=None, pv_forecast_minute_step=None, pv_forecast_minute10 self.prediction_cache = {} self.plan_interval_minutes = base.plan_interval_minutes self.charge_scaling10 = base.charge_scaling10 - self.pv_forecast_peak_step = pv_forecast_peak_step - self.clipping_limit = clipping_limit - self.clipping_cost_weight = clipping_cost_weight # Store this dictionary in global so we can reconstruct it in the thread without passing the data PRED_GLOBAL["dict"] = self.__dict__.copy() @@ -669,7 +672,22 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi # Clipping peak cost penalty: check if worst-case PV would exceed the clipping limit # and add a cost to the metric if the battery can't absorb the excess. # This makes the optimizer prefer plans that leave battery headroom during peak solar. - if pv_forecast_peak_step and clipping_limit_step > 0 and clipping_cost_weight > 0: + manual_buffer_active = False + if self.clipping_buffer_kwh > 0: + if self.clipping_buffer_start is None or self.clipping_buffer_end is None: + manual_buffer_active = True + elif self.clipping_buffer_start <= minute_absolute < self.clipping_buffer_end: + manual_buffer_active = True + + if manual_buffer_active: + target_headroom = self.clipping_buffer_kwh + battery_headroom = max(soc_max - soc, 0) * battery_loss + if battery_headroom < target_headroom: + # Apply a severe penalty to force the optimizer to respect the manual buffer limit + clipping_penalty = (target_headroom - battery_headroom) * export_rate * clipping_cost_weight * 5 + metric += clipping_penalty + clipping_penalty_total += clipping_penalty + elif pv_forecast_peak_step and clipping_limit_step > 0 and clipping_cost_weight > 0: peak_pv = pv_forecast_peak_step.get(minute, 0) if peak_pv > clipping_limit_step: potential_clip = peak_pv - clipping_limit_step diff --git a/apps/predbat/tests/test_fetch_pv_forecast.py b/apps/predbat/tests/test_fetch_pv_forecast.py index a257150df..c33bcd67a 100644 --- a/apps/predbat/tests/test_fetch_pv_forecast.py +++ b/apps/predbat/tests/test_fetch_pv_forecast.py @@ -107,7 +107,7 @@ def test_fetch_pv_forecast_with_relative_time(): ) # Call fetch_pv_forecast - pv_forecast_minute, pv_forecast_minute10 = fetch.fetch_pv_forecast() + pv_forecast_minute, pv_forecast_minute10, _, _, _ = fetch.fetch_pv_forecast() # With the corrected formula (target = stored_minute - offset), a +120 min offset # (relative_time is 2 hours before midnight_utc) shifts data BACK by 120 minutes: @@ -171,7 +171,7 @@ def test_fetch_pv_forecast_no_relative_time(): ) # Call fetch_pv_forecast - pv_forecast_minute, pv_forecast_minute10 = fetch.fetch_pv_forecast() + pv_forecast_minute, pv_forecast_minute10, _, _, _ = fetch.fetch_pv_forecast() # With no relative_time, it should fall back to midnight_utc # minute_offset = 0, so forecast data should map directly @@ -226,7 +226,7 @@ def test_fetch_pv_forecast_invalid_relative_time(): ) # Call fetch_pv_forecast - pv_forecast_minute, pv_forecast_minute10 = fetch.fetch_pv_forecast() + pv_forecast_minute, pv_forecast_minute10, _, _, _ = fetch.fetch_pv_forecast() # With invalid relative_time, it should fall back to midnight_utc # minute_offset = 0 @@ -279,7 +279,7 @@ def test_fetch_pv_forecast_relative_time_same_as_midnight(): ) # Call fetch_pv_forecast - pv_forecast_minute, pv_forecast_minute10 = fetch.fetch_pv_forecast() + pv_forecast_minute, pv_forecast_minute10, _, _, _ = fetch.fetch_pv_forecast() # minute_offset = 0 (same time), so data maps directly assert pv_forecast_minute[0] == 0.0, f"Expected minute 0 to be 0.0, got {pv_forecast_minute[0]}" @@ -339,7 +339,7 @@ def test_fetch_pv_forecast_previous_day(): ) # Call fetch_pv_forecast - pv_forecast_minute, pv_forecast_minute10 = fetch.fetch_pv_forecast() + pv_forecast_minute, pv_forecast_minute10, _, _, _ = fetch.fetch_pv_forecast() # With the corrected formula (target = stored_minute - offset), a +1440 min offset # maps yesterday's stored minutes to today-relative minutes: @@ -408,7 +408,7 @@ def test_fetch_pv_forecast_negative_offset(): ) # Call fetch_pv_forecast - pv_forecast_minute, pv_forecast_minute10 = fetch.fetch_pv_forecast() + pv_forecast_minute, pv_forecast_minute10, _, _, _ = fetch.fetch_pv_forecast() # With the corrected formula (target = stored_minute - offset), a -60 min offset # (relative_time is 1 hour AFTER midnight_utc) shifts data FORWARD by 60 minutes: diff --git a/apps/predbat/tests/test_solcast.py b/apps/predbat/tests/test_solcast.py index aac1440a8..4e5441ef0 100644 --- a/apps/predbat/tests/test_solcast.py +++ b/apps/predbat/tests/test_solcast.py @@ -2355,7 +2355,7 @@ def mock_minute_data_import_export(max_days_previous, now_utc, key, scale=1.0, r pv_forecast_minute10 = {m: 0.04 for m in range(total_minutes)} pv_forecast_data = [{"period_start": base.midnight_utc.strftime("%Y-%m-%dT%H:%M:%S+0000"), "pv_estimate": 0.05}] - adj_minute, adj_minute10, adj_data = solar.pv_calibration(pv_forecast_minute, pv_forecast_minute10, pv_forecast_data, create_pv10=False, divide_by=1.0, max_kwh=5.0, forecast_days=solar.forecast_days) + adj_minute, adj_minute10, adj_data, _ = solar.pv_calibration(pv_forecast_minute, pv_forecast_minute10, pv_forecast_data, create_pv10=False, divide_by=1.0, max_kwh=5.0, forecast_days=solar.forecast_days) # Returned minute data must be non-negative if any(v < 0 for v in adj_minute.values()): @@ -2575,7 +2575,7 @@ def mock_minute_data_import_export(max_days_previous, now_utc, key, scale=1.0, r pv_forecast_data.append({"period_start": ts.strftime("%Y-%m-%dT%H:%M:%S+0000"), "pv_estimate": 3.0 * plan_interval / 60}) max_kwh = 2.0 # panel peak output cap in kW - adj_minute, adj_minute10, adj_data = solar.pv_calibration(pv_forecast_minute, pv_forecast_minute10, pv_forecast_data, create_pv10=False, divide_by=1.0, max_kwh=max_kwh, forecast_days=solar.forecast_days) + adj_minute, adj_minute10, adj_data, _ = solar.pv_calibration(pv_forecast_minute, pv_forecast_minute10, pv_forecast_data, create_pv10=False, divide_by=1.0, max_kwh=max_kwh, forecast_days=solar.forecast_days) # capped_data = min(max(max_pv_power_hist, max_pv_power_forecast), max_kwh) * plan_interval / 60 # max_pv_power_hist ≈ 1 kW (per minute), max_pv_power_forecast ≈ 3/60 kW per minute @@ -2764,7 +2764,7 @@ def mock_minute_import_export(max_days_prev, now_utc, key, scale=1.0, required_u # synthetic pv_forecast dict without going through the real h0 pipeline # (which relies on now_utc_exact returning the mocked time). with patch("solcast.history_attribute_to_minute_data", return_value=(pv_forecast_hist, days_back)): - adj_m, adj_m10, adj_data = solar.pv_calibration(pv_m, pv_m10, pv_data, create_pv10=True, divide_by=1.0, max_kwh=5.0, forecast_days=solar.forecast_days) + adj_m, adj_m10, adj_data, _ = solar.pv_calibration(pv_m, pv_m10, pv_data, create_pv10=True, divide_by=1.0, max_kwh=5.0, forecast_days=solar.forecast_days) result = { "total_adj": solar.pv_calibration_total_adjustment, "avg_scaling": getattr(solar, "pv_calibration_average_scaling", None), @@ -2973,7 +2973,7 @@ def mock_minute_import_export(max_days_prev, now_utc, key, scale=1.0, required_u pv_forecast_hist[minutes_ago] = float(FORECAST_KW) with patch("solcast.history_attribute_to_minute_data", return_value=(pv_forecast_hist, days)): - adj_m, adj_m10, adj_data = solar.pv_calibration(pv_m, pv_m10, pv_data, create_pv10=True, divide_by=divide_by_factor, max_kwh=10.0, forecast_days=solar.forecast_days, period=FORECAST_PERIOD) + adj_m, adj_m10, adj_data, _ = solar.pv_calibration(pv_m, pv_m10, pv_data, create_pv10=True, divide_by=divide_by_factor, max_kwh=10.0, forecast_days=solar.forecast_days, period=FORECAST_PERIOD) # Each annotated entry should cover the full FORECAST_PERIOD minutes. # Expected calibrated kWh per entry ≈ FORECAST_KW * FORECAST_PERIOD / 60 = 2.0 kWh. diff --git a/apps/predbat/web.py b/apps/predbat/web.py index 4e88c86cd..6970c7ed2 100644 --- a/apps/predbat/web.py +++ b/apps/predbat/web.py @@ -2960,6 +2960,123 @@ def get_chart(self, chart): {"name": "Forecast CL", "data": pv_today_forecastCL, "opacity": "0.3", "stroke_width": "2", "stroke_curve": "smooth", "chart_type": "area", "color": "#e90a0a"}, ] text += self.render_chart(series_data, "kW", "Solar Forecast", now_str) + elif chart == "Clipping": + pv_power = prune_today(self.get_history_wrapper("sensor." + self.prefix + "_pv_power", 2, "state_class", "measurement"), self.now_utc, self.midnight_utc) + clipping_limit_kw = getattr(self.base, "clipping_limit_effective", 0.0) / 1000.0 + clipping_mode = getattr(self.base, "clipping_limit_mode", "Unknown") + inverter_ac_limit_kw = self.base.inverter_limit / 1000.0 if self.base.inverter_limit else clipping_limit_kw + + soc_kw_best_raw = history_attribute(self.get_history_wrapper(self.prefix + ".soc_kw_best", 2)) + soc_kw_best = prune_today(soc_kw_best_raw, self.now_utc, self.midnight_utc) + + clipping_forecast_raw = history_attribute(self.get_history_wrapper("sensor." + self.prefix + "_pv_forecast_peak", 2, "state_class", "measurement")) + clipping_forecast = prune_today(clipping_forecast_raw, self.now_utc, self.midnight_utc, prune=False, intermediate=True) + clipping_forecast_type = getattr(self.base, "pv_forecast_primary", "unknown") + if getattr(self.base, "clipping_use_clearsky_peaks", False): + clipping_forecast_type = getattr(self.base, "clipping_clearsky_source", "ClearSky") + + # Ceiling for both axes (kWh and kW) to ensure ticks line up + axis_max = 12.0 + if self.base.soc_max > 12.0 or inverter_ac_limit_kw > 12.0: + axis_max = max(self.base.soc_max, inverter_ac_limit_kw, 12.0) + + axis_ticks = 6 + + annotations = [] + if clipping_limit_kw > 0: + annotations.append({"y": clipping_limit_kw, "text": "{} ({} kW)".format(clipping_mode, round(clipping_limit_kw, 2)), "color": "#FF0000"}) + + if inverter_ac_limit_kw > 0 and abs(inverter_ac_limit_kw - clipping_limit_kw) > 0.1: + annotations.append({"y": inverter_ac_limit_kw, "text": "Inverter Capacity ({} kW)".format(round(inverter_ac_limit_kw, 2)), "color": "#999999"}) + + # New X-Axis annotations for clipping window + xaxis_annotations = [] + clipping_start = getattr(self.base, "clipping_buffer_start", None) + clipping_end = getattr(self.base, "clipping_buffer_end", None) + + if clipping_start is not None and clipping_end is not None: + start_dt = self.midnight_utc + timedelta(minutes=clipping_start) + end_dt = self.midnight_utc + timedelta(minutes=clipping_end) + xaxis_annotations.append({"x": start_dt.strftime(TIME_FORMAT), "text": "Today Buffer Start", "color": "#FF9800"}) + xaxis_annotations.append({"x": end_dt.strftime(TIME_FORMAT), "text": "Today Buffer End", "color": "#FF9800"}) + + # Tomorrow's window (if available in forecast) + if getattr(self.base, "clipping_buffer_forecast_kwh", None): + forecast = self.base.clipping_buffer_forecast_kwh + tomorrow_start = 1440 + if any(forecast.get(m, 0) > 0 for m in range(tomorrow_start, tomorrow_start + 1440)): + mid_tomorrow = self.midnight_utc + timedelta(days=1) + xaxis_annotations.append({"x": mid_tomorrow.strftime(TIME_FORMAT), "text": "Tomorrow Buffer Active", "color": "#FF9800"}) + + # Clipping Remaining Time-Series + clipping_remaining_series = {} + if getattr(self.base, "clipping_buffer_forecast_kwh", None): + forecast = self.base.clipping_buffer_forecast_kwh + step_size = getattr(self.base, "plan_interval_minutes", 30) + for minute, kwh in forecast.items(): + if minute % step_size == 0: + minute_timestamp = self.midnight_utc + timedelta(minutes=minute) + stamp = minute_timestamp.strftime(TIME_FORMAT) + clipping_remaining_series[stamp] = round(kwh, 2) + + # Clipping Target SOC series (The "Red Line") + clipping_target_series = {} + if getattr(self.base, "clipping_buffer_forecast_kwh", None): + forecast = self.base.clipping_buffer_forecast_kwh + step_size = getattr(self.base, "plan_interval_minutes", 30) + for minute, kwh in forecast.items(): + if minute % step_size == 0: + minute_timestamp = self.midnight_utc + timedelta(minutes=minute) + stamp = minute_timestamp.strftime(TIME_FORMAT) + # Target is the "Ceiling" we want to stay under + target_val = max(0, self.base.soc_max - kwh) + if kwh > 0: + clipping_target_series[stamp] = round(target_val, 2) + else: + clipping_target_series[stamp] = None # Don't plot the ceiling when there is no buffer + + series_data = [ + {"name": "Clipping Ceiling", "data": clipping_target_series, "opacity": "1.0", "stroke_width": "3", "stroke_curve": "smooth", "color": "#eb2323", "unit": "kWh"}, + {"name": "Actual SOC", "data": soc_kw_best, "opacity": "1.0", "stroke_width": "3", "stroke_curve": "stepline", "color": "#9b23eb", "unit": "kWh"}, + {"name": "Clipping Remaining", "data": clipping_remaining_series, "opacity": "0.4", "stroke_width": "2", "stroke_curve": "smooth", "color": "#2196F3", "unit": "kWh"}, + {"name": "PV Power", "data": pv_power, "opacity": "1.0", "stroke_width": "3", "stroke_curve": "smooth", "color": "#f5c43d", "unit": "kW"}, + {"name": "Clipping Forecast (" + clipping_forecast_type + ")", "data": clipping_forecast, "opacity": "0.3", "stroke_width": "2", "stroke_curve": "smooth", "chart_type": "area", "color": "#a8a8a7", "unit": "kW"}, + ] + + secondary_axis = [ + { + "title": "kW", + "series_name": "PV Power", + "decimals": 1, + "opposite": True, + "min": 0, + "max": axis_max, + "tickAmount": axis_ticks, + }, + { + "title": "kW", + "series_name": "Clipping Forecast (" + clipping_forecast_type + ")", + "show": False, + "min": 0, + "max": axis_max, + "tickAmount": axis_ticks, + }, + { + "title": "kWh", + "series_name": "Clipping Remaining", + "show": False, + "min": 0, + "max": axis_max, + "tickAmount": axis_ticks, + }, + ] + + # Append dynamic stats to chart title + clipping_remaining = getattr(self.base, "clipping_remaining_today", 0.0) + clipping_mitigated = getattr(self.base, "clipping_mitigated_today", 0.0) + chart_title = "Clipping Analysis (Remaining: {:.2f} kWh, Mitigated: {:.2f} kWh)".format(clipping_remaining, clipping_mitigated) + + text += self.render_chart(series_data, "kWh", chart_title, now_str, yaxis_annotations=annotations, xaxis_annotations=xaxis_annotations, extra_yaxis=secondary_axis, yaxis_min=0, yaxis_max=axis_max, yaxis_tick_amount=axis_ticks) elif chart == "PVAccuracy": # Get pv_today history once and extract total and remaining attributes per timestamp pv_today_hist = self.get_history_wrapper("sensor." + self.prefix + "_pv_today", 7, required=False) From afaad6f412c8fd157884e525f9af7c84c59b3b5e Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 14 Jun 2026 17:07:13 +0100 Subject: [PATCH 06/74] fix: update ML component fetch_pv_forecast unpack for 5 return values --- apps/predbat/load_ml_component.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/predbat/load_ml_component.py b/apps/predbat/load_ml_component.py index d0bc62358..82609fea3 100644 --- a/apps/predbat/load_ml_component.py +++ b/apps/predbat/load_ml_component.py @@ -404,7 +404,7 @@ async def _fetch_load_data(self): energy = self.get_from_incrementing(pv_data_cumulative, m, PREDICT_STEP, backwards=True) pv_data[m] = dp4(energy) - pv_forecast_minute, pv_forecast_minute10 = self.base.fetch_pv_forecast() + pv_forecast_minute, pv_forecast_minute10, pv_forecast_minute90, pv_forecast_minuteCS, pv_forecast_minuteHIST = self.base.fetch_pv_forecast() # Add future PV forecast as per-5-min energy with negative keys (negative = future) # key -5 = first future step, -10 = second, etc. if pv_forecast_minute: From 4e07fe5d61e6d2272f8e561dcd8ed90f56bb2b7a Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 14 Jun 2026 18:08:09 +0100 Subject: [PATCH 07/74] fix: synthesize clear sky curve from P90 ensemble for Open-Meteo --- apps/predbat/solcast.py | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/apps/predbat/solcast.py b/apps/predbat/solcast.py index 4e99ed411..79ae6838a 100644 --- a/apps/predbat/solcast.py +++ b/apps/predbat/solcast.py @@ -255,21 +255,22 @@ def convert_azimuth(self, az): async def download_open_meteo_ensemble_data(self, lat, lon, tilt, az, kwp, system_loss): """ - Download Open-Meteo ensemble data for P10 solar estimate. - Returns a dict mapping ISO timestamp strings to P10 kW values. + Download Open-Meteo ensemble data for P10 and P90 solar estimates. + Returns a tuple of (p10_dict, p90_dict) mapping ISO timestamp strings to kW values. """ url = "https://ensemble-api.open-meteo.com/v1/ensemble?models=icon_seamless&latitude={lat}&longitude={lon}&hourly=global_tilted_irradiance&tilt={tilt}&azimuth={az}&forecast_days=4&timezone=UTC".format(lat=lat, lon=lon, tilt=tilt, az=az) data = await self.cache_get_url(url, params={}, max_age=self.open_meteo_forecast_max_age * 60) if not data: - return {} + return {}, {} hourly = data.get("hourly", {}) times = hourly.get("time", []) member_keys = [k for k in hourly if k.startswith("global_tilted_irradiance_member")] if not member_keys or not times: - return {} + return {}, {} - result = {} + result_p10 = {} + result_p90 = {} for idx, ts in enumerate(times): values = [] for k in member_keys: @@ -277,13 +278,17 @@ async def download_open_meteo_ensemble_data(self, lat, lon, tilt, az, kwp, syste if val is not None: values.append(val) if not values: - result[ts] = 0.0 + result_p10[ts] = 0.0 + result_p90[ts] = 0.0 continue values.sort() p10_idx = max(0, math.ceil(len(values) * 0.10) - 1) + p90_idx = min(len(values) - 1, math.floor(len(values) * 0.90)) gti_p10 = values[p10_idx] - result[ts] = dp4((gti_p10 / 1000.0) * kwp * (1.0 - system_loss)) - return result + gti_p90 = values[p90_idx] + result_p10[ts] = dp4((gti_p10 / 1000.0) * kwp * (1.0 - system_loss)) + result_p90[ts] = dp4((gti_p90 / 1000.0) * kwp * (1.0 - system_loss)) + return result_p10, result_p90 async def download_open_meteo_data(self, configs=None): """ @@ -349,7 +354,7 @@ async def download_open_meteo_data(self, configs=None): self.log("Warn: SolarAPI: Open-Meteo data for lat {} lon {} has no hourly data".format(lat, lon)) continue - ensemble_p10 = await self.download_open_meteo_ensemble_data(lat, lon, tilt, az, kwp, system_loss) + ensemble_p10, ensemble_p90 = await self.download_open_meteo_ensemble_data(lat, lon, tilt, az, kwp, system_loss) # Pass 1: compute instantaneous kW at each UTC timestamp sample. # Open-Meteo returns point-in-time irradiance (W/m²) at the start of each hour, @@ -362,7 +367,6 @@ async def download_open_meteo_data(self, configs=None): gti = gti_values[idx] if gti is None: gti = 0.0 - cs_gti = cs_gti_values[idx] if idx < len(cs_gti_values) and cs_gti_values[idx] is not None else gti temp = temp_values[idx] if idx < len(temp_values) and temp_values[idx] is not None else 25.0 wind = wind_values[idx] if idx < len(wind_values) and wind_values[idx] is not None else 1.0 # Cell temperature via SAPM/PVWatts model: irradiance heats the cell above ambient @@ -372,10 +376,18 @@ async def download_open_meteo_data(self, configs=None): # Cap at 1.1 (10% above STC) to prevent unrealistic gains at very cold temperatures. eta_temp = max(0.5, min(1.1, 1.0 - 0.004 * (t_cell - 25.0))) pv50_inst = dp4((gti / 1000.0) * kwp * eta_temp * (1.0 - system_loss)) - pv_cs_inst = dp4((cs_gti / 1000.0) * kwp * eta_temp * (1.0 - system_loss)) + raw_p10 = ensemble_p10.get(ts) - # ensemble_p10 was computed without temperature derating; apply eta_temp now + raw_p90 = ensemble_p90.get(ts) + # ensemble_p10/p90 were computed without temperature derating; apply eta_temp now pv10_inst = dp4(min(raw_p10 * eta_temp, pv50_inst) if raw_p10 is not None else pv50_inst * 0.7) + + if idx < len(cs_gti_values) and cs_gti_values[idx] is not None: + cs_gti = cs_gti_values[idx] + pv_cs_inst = dp4((cs_gti / 1000.0) * kwp * eta_temp * (1.0 - system_loss)) + else: + # Fallback to P90 ensemble if Clear Sky GTI is unavailable (which is always true for Open-Meteo right now) + pv_cs_inst = dp4(max(raw_p90 * eta_temp, pv50_inst) if raw_p90 is not None else pv50_inst * 1.3) try: stamp = datetime.strptime(ts, "%Y-%m-%dT%H:%M") stamp = stamp.replace(tzinfo=pytz.utc) From 374125fbd33a1bea2d34467dbf12991d18b05c80 Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 14 Jun 2026 19:23:04 +0100 Subject: [PATCH 08/74] feat: apply clipping_peak_amplification to clear sky curve --- apps/predbat/plan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index 002b25f4f..eb4d0fe56 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -976,7 +976,7 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): getattr(self, "pv_forecast_minuteCS", {}), self.minutes_now, forward=True, cloud_factor=None ) pv_forecast_peak_step = { - k: max(v, pv_clearsky_step.get(k, 0)) for k, v in pv_forecast_peak_step.items() + k: max(v, pv_clearsky_step.get(k, 0) * self.clipping_peak_amplification) for k, v in pv_forecast_peak_step.items() } elif self.clipping_peak_amplification != 1.0: pv_forecast_peak_step = {k: v * self.clipping_peak_amplification for k, v in pv_forecast_peak_step.items()} From 49bd3f3915258dce78a537957dad5db7612de304 Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 14 Jun 2026 19:49:30 +0100 Subject: [PATCH 09/74] fix: remove invalid clear_sky_gti from Open-Meteo URL --- apps/predbat/solcast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/predbat/solcast.py b/apps/predbat/solcast.py index 79ae6838a..cf51aaf5a 100644 --- a/apps/predbat/solcast.py +++ b/apps/predbat/solcast.py @@ -335,7 +335,7 @@ async def download_open_meteo_data(self, configs=None): else: self.log("Warn: SolarAPI: Postcode {} could not be resolved to latitude and longitude, using default".format(postcode)) - url = "https://api.open-meteo.com/v1/forecast?latitude={lat}&longitude={lon}&hourly=global_tilted_irradiance,clear_sky_gti,temperature_2m,wind_speed_10m&wind_speed_unit=ms&tilt={tilt}&azimuth={az}&forecast_days=4&timezone=UTC".format( + url = "https://api.open-meteo.com/v1/forecast?latitude={lat}&longitude={lon}&hourly=global_tilted_irradiance,temperature_2m,wind_speed_10m&wind_speed_unit=ms&tilt={tilt}&azimuth={az}&forecast_days=4&timezone=UTC".format( lat=lat, lon=lon, tilt=tilt, az=az ) data = await self.cache_get_url(url, params={}, max_age=self.open_meteo_forecast_max_age * 60) From dc68891b9a2872aded1cc481cadf8122e807ce99 Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 14 Jun 2026 21:11:04 +0100 Subject: [PATCH 10/74] fix: 500 error in clipping chart and publish peak forecast --- apps/predbat/output.py | 15 +++++++++++++++ apps/predbat/web.py | 32 +++++++++++++++++++------------- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/apps/predbat/output.py b/apps/predbat/output.py index 29e5bd284..79d8cab10 100644 --- a/apps/predbat/output.py +++ b/apps/predbat/output.py @@ -2488,6 +2488,21 @@ def record_status(self, message, debug="", had_errors=False, notify=False, extra }, ) + pv_forecast_peak_step = getattr(self, "pv_forecast_peak_step", {}) + if pv_forecast_peak_step: + self.dashboard_item( + self.prefix + ".pv_forecast_peak", + state=dp2(pv_forecast_peak_step.get(self.minutes_now, 0)), + attributes={ + "results": self.filtered_times(pv_forecast_peak_step), + "friendly_name": "Solar peak forecast", + "state_class": "measurement", + "unit_of_measurement": "kW", + "icon": "mdi:solar-power", + }, + ) + + if had_errors: self.log("Warn: record_status {}".format(message + extra)) else: diff --git a/apps/predbat/web.py b/apps/predbat/web.py index 6970c7ed2..101284c25 100644 --- a/apps/predbat/web.py +++ b/apps/predbat/web.py @@ -2961,19 +2961,25 @@ def get_chart(self, chart): ] text += self.render_chart(series_data, "kW", "Solar Forecast", now_str) elif chart == "Clipping": - pv_power = prune_today(self.get_history_wrapper("sensor." + self.prefix + "_pv_power", 2, "state_class", "measurement"), self.now_utc, self.midnight_utc) - clipping_limit_kw = getattr(self.base, "clipping_limit_effective", 0.0) / 1000.0 - clipping_mode = getattr(self.base, "clipping_limit_mode", "Unknown") - inverter_ac_limit_kw = self.base.inverter_limit / 1000.0 if self.base.inverter_limit else clipping_limit_kw - - soc_kw_best_raw = history_attribute(self.get_history_wrapper(self.prefix + ".soc_kw_best", 2)) - soc_kw_best = prune_today(soc_kw_best_raw, self.now_utc, self.midnight_utc) - - clipping_forecast_raw = history_attribute(self.get_history_wrapper("sensor." + self.prefix + "_pv_forecast_peak", 2, "state_class", "measurement")) - clipping_forecast = prune_today(clipping_forecast_raw, self.now_utc, self.midnight_utc, prune=False, intermediate=True) - clipping_forecast_type = getattr(self.base, "pv_forecast_primary", "unknown") - if getattr(self.base, "clipping_use_clearsky_peaks", False): - clipping_forecast_type = getattr(self.base, "clipping_clearsky_source", "ClearSky") + try: + pv_power = prune_today(self.get_history_wrapper("sensor." + self.prefix + "_pv_power", 2, "state_class", "measurement"), self.now_utc, self.midnight_utc) + clipping_limit_kw = getattr(self.base, "clipping_limit_effective", 0.0) / 1000.0 + clipping_mode = getattr(self.base, "clipping_limit_mode", "Unknown") + inverter_ac_limit_kw = self.base.inverter_limit / 1000.0 if self.base.inverter_limit else clipping_limit_kw + + soc_kw_best_raw = history_attribute(self.get_history_wrapper(self.prefix + ".soc_kw_best", 2)) + soc_kw_best = prune_today(soc_kw_best_raw, self.now_utc, self.midnight_utc) + + clipping_forecast_raw = history_attribute(self.get_history_wrapper("sensor." + self.prefix + "_pv_forecast_peak", 2, "state_class", "measurement")) + clipping_forecast = prune_today(clipping_forecast_raw, self.now_utc, self.midnight_utc, prune=False, intermediate=True) + clipping_forecast_type = getattr(self.base, "pv_forecast_primary", "unknown") + if getattr(self.base, "clipping_use_clearsky_peaks", False): + clipping_forecast_type = getattr(self.base, "clipping_clearsky_source", "ClearSky") + except Exception as e: + import traceback + self.log("ERROR IN CLIPPING CHART: " + str(e)) + self.log(traceback.format_exc()) + raise e # Ceiling for both axes (kWh and kW) to ensure ticks line up axis_max = 12.0 From 627a36585ada25bd180e2ed6ee64f8b41588eccf Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 14 Jun 2026 21:17:55 +0100 Subject: [PATCH 11/74] feat: Add clipping times and mode to the short_textual_plan --- apps/predbat/output.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/apps/predbat/output.py b/apps/predbat/output.py index 79d8cab10..12eb5f718 100644 --- a/apps/predbat/output.py +++ b/apps/predbat/output.py @@ -944,7 +944,27 @@ def short_textual_plan(self, soc_min, soc_min_minute, pv_forecast_minute_step, p if predict_clipped_best: clipping_total = predict_clipped_best.get(max(predict_clipped_best.keys()), 0.0) if clipping_total > 0.01: - sentence += "- Forecast {} kWh clipping, plan penalized to mitigate.\n".format(dp2(clipping_total)) + clipping_mode = getattr(self, "clipping_limit_mode", "Unknown") + start_str = "" + end_str = "" + start_stamp = None + end_stamp = None + + prev_val = 0.0 + for min_key, val in sorted(predict_clipped_best.items()): + if val > prev_val + 0.001: + if start_stamp is None: + start_stamp = self.midnight_utc + timedelta(minutes=min_key) + end_stamp = self.midnight_utc + timedelta(minutes=min_key) + prev_val = val + + if start_stamp and end_stamp: + start_str = start_stamp.strftime("%H:%M") + end_str = end_stamp.strftime("%H:%M") + + sentence += "- Forecast {} kWh clipping, exceeding {} limit from {} to {}. Plan penalized to mitigate.\n".format( + dp2(clipping_total), clipping_mode, start_str, end_str + ) if publish: self.text_plan = self.get_text_plan_html(sentence) From 62a3480ccb24aa84b2e2dd99a842f6c3fc36b9b5 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 15 Jun 2026 08:50:12 +0100 Subject: [PATCH 12/74] Fix legacy clipping entities and implement auto-tuning --- apps/predbat/config.py | 8 +++++ apps/predbat/fetch.py | 1 + apps/predbat/plan.py | 80 ++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 86 insertions(+), 3 deletions(-) diff --git a/apps/predbat/config.py b/apps/predbat/config.py index f3843b584..811dfe736 100644 --- a/apps/predbat/config.py +++ b/apps/predbat/config.py @@ -1443,6 +1443,14 @@ "icon": "mdi:weather-sunny-alert", "enable": "clipping_peak_enable", }, + { + "name": "clipping_peak_auto_tune", + "friendly_name": "Clipping Peak Auto-Tune", + "type": "switch", + "default": True, + "icon": "mdi:auto-fix", + "enable": "clipping_peak_enable", + }, { "name": "clipping_cost_weight", "friendly_name": "Clipping Cost Weight", diff --git a/apps/predbat/fetch.py b/apps/predbat/fetch.py index 750cd759c..67b6c07a0 100644 --- a/apps/predbat/fetch.py +++ b/apps/predbat/fetch.py @@ -2359,6 +2359,7 @@ def fetch_config_options(self): # Clipping peak cost penalty model self.clipping_peak_enable = self.get_arg("clipping_peak_enable") self.clipping_use_clearsky_peaks = self.get_arg("clipping_use_clearsky_peaks") + self.clipping_peak_auto_tune = self.get_arg("clipping_peak_auto_tune") self.clipping_cost_weight = self.get_arg("clipping_cost_weight") self.clipping_peak_amplification = self.get_arg("clipping_peak_amplification") self.clipping_limit_override = self.get_arg("clipping_limit_override") / MINUTE_WATT if self.get_arg("clipping_limit_override") else 0 diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index eb4d0fe56..4736935fc 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -970,16 +970,73 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): self.pv_forecast_minute, self.minutes_now, forward=True, cloud_factor=None ) + # Apply Auto-Tuning if enabled + auto_amp = 1.0 + if getattr(self, "clipping_peak_auto_tune", False): + import os + import json + auto_tune_file = os.path.join(self.config_root, "clipping_auto_tune.json") + last_tune_day = None + if os.path.exists(auto_tune_file): + try: + with open(auto_tune_file, "r") as f: + data = json.load(f) + auto_amp = data.get("auto_amp", 1.0) + last_tune_day = data.get("last_tune_day", None) + except Exception: + pass + + # Check if we should tune today (only once per day) + current_day = self.now_utc.strftime("%Y-%m-%d") + if current_day != last_tune_day: + # Retrieve last 24h of pv_power + days = 1 + pv_today_hist = self.base.minute_data_import_export(days, self.now_utc, "pv_today", required_unit="kWh", pad=False) + max_pv_power = 0.0 + if pv_today_hist: + pv_today_hist_max_minute = max(pv_today_hist.keys()) + current_value = None + for minute in range(pv_today_hist_max_minute - 5, -5, -5): + current_value = pv_today_hist.get(minute, current_value) + next_value = pv_today_hist.get(minute - 5, current_value) + if current_value is not None and next_value is not None: + power_amount = max(0, next_value - current_value) * 60.0 / 5.0 + if power_amount > max_pv_power: + max_pv_power = power_amount + + # Check inverter limit + limit = 0.0 + if self.inverter_limit > 0: + limit = self.inverter_limit / MINUTE_WATT + + if limit > 0: + if max_pv_power >= limit * 0.98: + auto_amp = min(2.5, auto_amp + 0.05) + self.log("Clipping auto-tuner: Real clipping detected (max PV {} kW >= limit {} kW). Increased auto_amp to {}".format(dp2(max_pv_power), dp2(limit), auto_amp)) + else: + auto_amp = max(1.0, auto_amp - 0.01) + self.log("Clipping auto-tuner: No clipping detected. Decreased auto_amp to {}".format(auto_amp)) + + try: + with open(auto_tune_file, "w") as f: + json.dump({"auto_amp": auto_amp, "last_tune_day": current_day}, f) + except Exception: + pass + + self.dashboard_item(self.prefix + ".clipping_auto_amplification", state=dp2(auto_amp), attributes={"friendly_name": "Clipping Auto Amplification Factor", "icon": "mdi:auto-fix"}) + + effective_amplification = self.clipping_peak_amplification * auto_amp + # Apply ClearSky or Amplification factor if getattr(self, "clipping_use_clearsky_peaks", False): pv_clearsky_step = self.step_data_history( getattr(self, "pv_forecast_minuteCS", {}), self.minutes_now, forward=True, cloud_factor=None ) pv_forecast_peak_step = { - k: max(v, pv_clearsky_step.get(k, 0) * self.clipping_peak_amplification) for k, v in pv_forecast_peak_step.items() + k: max(v, pv_clearsky_step.get(k, 0) * effective_amplification) for k, v in pv_forecast_peak_step.items() } - elif self.clipping_peak_amplification != 1.0: - pv_forecast_peak_step = {k: v * self.clipping_peak_amplification for k, v in pv_forecast_peak_step.items()} + elif effective_amplification != 1.0: + pv_forecast_peak_step = {k: v * effective_amplification for k, v in pv_forecast_peak_step.items()} # Calculate effective clipping limit: most restrictive hardware constraint if self.clipping_limit_override > 0: @@ -3903,6 +3960,23 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi }, ) + # Calculate legacy clipping entities from the cloud model's clipping curve + midnight_today_minute = 24 * 60 - self.minutes_now + clipping_today = 0.0 + clipping_tomorrow = 0.0 + + if hasattr(self, "predict_clipped_best") and self.predict_clipped_best: + # Find closest key <= midnight_today_minute for clipping_today + keys = sorted(self.predict_clipped_best.keys()) + key_today = next((k for k in reversed(keys) if k <= midnight_today_minute), 0) + clipping_today = self.predict_clipped_best.get(key_today, 0.0) + clipping_total = self.predict_clipped_best.get(keys[-1], 0.0) + clipping_tomorrow = max(0.0, clipping_total - clipping_today) + + self.clipping_remaining_today = clipping_today + self.clipping_tomorrow = clipping_tomorrow + self.clipping_mitigated_today = clipping_today + # Add Clipping Summary Dashboard Items self.dashboard_item( self.prefix + ".clipping_remaining_today", From b841222994928c1ed9b777f86936d766186f67ec Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 15 Jun 2026 08:57:58 +0100 Subject: [PATCH 13/74] Refine auto-tuning logic --- apps/predbat/plan.py | 105 ++++++++++++++++++++++--------------------- 1 file changed, 54 insertions(+), 51 deletions(-) diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index 4736935fc..fefac8f3c 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -970,62 +970,65 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): self.pv_forecast_minute, self.minutes_now, forward=True, cloud_factor=None ) - # Apply Auto-Tuning if enabled + # Auto-Tuner: Always runs to gather data and recommend a factor auto_amp = 1.0 - if getattr(self, "clipping_peak_auto_tune", False): - import os - import json - auto_tune_file = os.path.join(self.config_root, "clipping_auto_tune.json") - last_tune_day = None - if os.path.exists(auto_tune_file): + import os + import json + auto_tune_file = os.path.join(self.config_root, "clipping_auto_tune.json") + last_tune_day = None + if os.path.exists(auto_tune_file): + try: + with open(auto_tune_file, "r") as f: + data = json.load(f) + auto_amp = data.get("auto_amp", 1.0) + last_tune_day = data.get("last_tune_day", None) + except Exception: + pass + + # Check if we should tune today (only once per day) + current_day = self.now_utc.strftime("%Y-%m-%d") + if current_day != last_tune_day: + # Retrieve last 24h of pv_power + days = 1 + pv_today_hist = self.base.minute_data_import_export(days, self.now_utc, "pv_today", required_unit="kWh", pad=False) + max_pv_power = 0.0 + if pv_today_hist: + pv_today_hist_max_minute = max(pv_today_hist.keys()) + current_value = None + for minute in range(pv_today_hist_max_minute - 5, -5, -5): + current_value = pv_today_hist.get(minute, current_value) + next_value = pv_today_hist.get(minute - 5, current_value) + if current_value is not None and next_value is not None: + power_amount = max(0, next_value - current_value) * 60.0 / 5.0 + if power_amount > max_pv_power: + max_pv_power = power_amount + + # Check inverter limit + limit = 0.0 + if self.inverter_limit > 0: + limit = self.inverter_limit / MINUTE_WATT + + if limit > 0: + if max_pv_power >= limit * 0.98: + auto_amp = min(2.5, auto_amp + 0.05) + self.log("Clipping auto-tuner: Real clipping detected (max PV {} kW >= limit {} kW). Increased recommended auto_amp to {}".format(dp2(max_pv_power), dp2(limit), auto_amp)) + else: + auto_amp = max(1.0, auto_amp - 0.01) + self.log("Clipping auto-tuner: No clipping detected. Decreased recommended auto_amp to {}".format(auto_amp)) + try: - with open(auto_tune_file, "r") as f: - data = json.load(f) - auto_amp = data.get("auto_amp", 1.0) - last_tune_day = data.get("last_tune_day", None) + with open(auto_tune_file, "w") as f: + json.dump({"auto_amp": auto_amp, "last_tune_day": current_day}, f) except Exception: pass - - # Check if we should tune today (only once per day) - current_day = self.now_utc.strftime("%Y-%m-%d") - if current_day != last_tune_day: - # Retrieve last 24h of pv_power - days = 1 - pv_today_hist = self.base.minute_data_import_export(days, self.now_utc, "pv_today", required_unit="kWh", pad=False) - max_pv_power = 0.0 - if pv_today_hist: - pv_today_hist_max_minute = max(pv_today_hist.keys()) - current_value = None - for minute in range(pv_today_hist_max_minute - 5, -5, -5): - current_value = pv_today_hist.get(minute, current_value) - next_value = pv_today_hist.get(minute - 5, current_value) - if current_value is not None and next_value is not None: - power_amount = max(0, next_value - current_value) * 60.0 / 5.0 - if power_amount > max_pv_power: - max_pv_power = power_amount - - # Check inverter limit - limit = 0.0 - if self.inverter_limit > 0: - limit = self.inverter_limit / MINUTE_WATT - - if limit > 0: - if max_pv_power >= limit * 0.98: - auto_amp = min(2.5, auto_amp + 0.05) - self.log("Clipping auto-tuner: Real clipping detected (max PV {} kW >= limit {} kW). Increased auto_amp to {}".format(dp2(max_pv_power), dp2(limit), auto_amp)) - else: - auto_amp = max(1.0, auto_amp - 0.01) - self.log("Clipping auto-tuner: No clipping detected. Decreased auto_amp to {}".format(auto_amp)) - - try: - with open(auto_tune_file, "w") as f: - json.dump({"auto_amp": auto_amp, "last_tune_day": current_day}, f) - except Exception: - pass - - self.dashboard_item(self.prefix + ".clipping_auto_amplification", state=dp2(auto_amp), attributes={"friendly_name": "Clipping Auto Amplification Factor", "icon": "mdi:auto-fix"}) - effective_amplification = self.clipping_peak_amplification * auto_amp + self.dashboard_item(self.prefix + ".clipping_recommended_amplification", state=dp2(auto_amp), attributes={"friendly_name": "Clipping Recommended Amplification", "icon": "mdi:auto-fix"}) + + # Apply Manual vs Auto Factor + if getattr(self, "clipping_peak_auto_tune", False): + effective_amplification = auto_amp + else: + effective_amplification = self.clipping_peak_amplification # Apply ClearSky or Amplification factor if getattr(self, "clipping_use_clearsky_peaks", False): From 70a752710d1444e0abdb8ce601fcd48eb6cf9240 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 15 Jun 2026 10:37:40 +0100 Subject: [PATCH 14/74] Fix AttributeError in auto-tuner --- apps/predbat/plan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index fefac8f3c..01349d2ed 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -990,7 +990,7 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): if current_day != last_tune_day: # Retrieve last 24h of pv_power days = 1 - pv_today_hist = self.base.minute_data_import_export(days, self.now_utc, "pv_today", required_unit="kWh", pad=False) + pv_today_hist = self.minute_data_import_export(days, self.now_utc, "pv_today", required_unit="kWh", pad=False) max_pv_power = 0.0 if pv_today_hist: pv_today_hist_max_minute = max(pv_today_hist.keys()) From 11c68149d0c51dcea2c79c34fdb8303bc2516da2 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 15 Jun 2026 13:03:45 +0100 Subject: [PATCH 15/74] Initialize auto_amp with manual amplification value instead of 1.0 --- apps/predbat/plan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index 01349d2ed..1bdfe183e 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -971,7 +971,7 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): ) # Auto-Tuner: Always runs to gather data and recommend a factor - auto_amp = 1.0 + auto_amp = getattr(self, "clipping_peak_amplification", 1.0) import os import json auto_tune_file = os.path.join(self.config_root, "clipping_auto_tune.json") From 9df1dde4215acc2513f13cbd2400febcbb707352 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 15 Jun 2026 14:18:26 +0100 Subject: [PATCH 16/74] Fix clipping chart 500 error and add clipping status to textual plan --- apps/predbat/output.py | 4 ++ apps/predbat/web.py | 122 ----------------------------------------- 2 files changed, 4 insertions(+), 122 deletions(-) diff --git a/apps/predbat/output.py b/apps/predbat/output.py index 12eb5f718..2cc86ae77 100644 --- a/apps/predbat/output.py +++ b/apps/predbat/output.py @@ -940,6 +940,10 @@ def short_textual_plan(self, soc_min, soc_min_minute, pv_forecast_minute_step, p ) # Clipping summary + if getattr(self, "clipping_peak_enable", False): + clipping_status = getattr(self, "clipping_status", "No clipping forecast.") + sentence += "- Clipping status: {}\n".format(clipping_status) + predict_clipped_best = getattr(self, "predict_clipped_best", {}) if predict_clipped_best: clipping_total = predict_clipped_best.get(max(predict_clipped_best.keys()), 0.0) diff --git a/apps/predbat/web.py b/apps/predbat/web.py index 101284c25..b1f0a70a1 100644 --- a/apps/predbat/web.py +++ b/apps/predbat/web.py @@ -2960,129 +2960,7 @@ def get_chart(self, chart): {"name": "Forecast CL", "data": pv_today_forecastCL, "opacity": "0.3", "stroke_width": "2", "stroke_curve": "smooth", "chart_type": "area", "color": "#e90a0a"}, ] text += self.render_chart(series_data, "kW", "Solar Forecast", now_str) - elif chart == "Clipping": - try: - pv_power = prune_today(self.get_history_wrapper("sensor." + self.prefix + "_pv_power", 2, "state_class", "measurement"), self.now_utc, self.midnight_utc) - clipping_limit_kw = getattr(self.base, "clipping_limit_effective", 0.0) / 1000.0 - clipping_mode = getattr(self.base, "clipping_limit_mode", "Unknown") - inverter_ac_limit_kw = self.base.inverter_limit / 1000.0 if self.base.inverter_limit else clipping_limit_kw - - soc_kw_best_raw = history_attribute(self.get_history_wrapper(self.prefix + ".soc_kw_best", 2)) - soc_kw_best = prune_today(soc_kw_best_raw, self.now_utc, self.midnight_utc) - - clipping_forecast_raw = history_attribute(self.get_history_wrapper("sensor." + self.prefix + "_pv_forecast_peak", 2, "state_class", "measurement")) - clipping_forecast = prune_today(clipping_forecast_raw, self.now_utc, self.midnight_utc, prune=False, intermediate=True) - clipping_forecast_type = getattr(self.base, "pv_forecast_primary", "unknown") - if getattr(self.base, "clipping_use_clearsky_peaks", False): - clipping_forecast_type = getattr(self.base, "clipping_clearsky_source", "ClearSky") - except Exception as e: - import traceback - self.log("ERROR IN CLIPPING CHART: " + str(e)) - self.log(traceback.format_exc()) - raise e - - # Ceiling for both axes (kWh and kW) to ensure ticks line up - axis_max = 12.0 - if self.base.soc_max > 12.0 or inverter_ac_limit_kw > 12.0: - axis_max = max(self.base.soc_max, inverter_ac_limit_kw, 12.0) - - axis_ticks = 6 - - annotations = [] - if clipping_limit_kw > 0: - annotations.append({"y": clipping_limit_kw, "text": "{} ({} kW)".format(clipping_mode, round(clipping_limit_kw, 2)), "color": "#FF0000"}) - - if inverter_ac_limit_kw > 0 and abs(inverter_ac_limit_kw - clipping_limit_kw) > 0.1: - annotations.append({"y": inverter_ac_limit_kw, "text": "Inverter Capacity ({} kW)".format(round(inverter_ac_limit_kw, 2)), "color": "#999999"}) - - # New X-Axis annotations for clipping window - xaxis_annotations = [] - clipping_start = getattr(self.base, "clipping_buffer_start", None) - clipping_end = getattr(self.base, "clipping_buffer_end", None) - - if clipping_start is not None and clipping_end is not None: - start_dt = self.midnight_utc + timedelta(minutes=clipping_start) - end_dt = self.midnight_utc + timedelta(minutes=clipping_end) - xaxis_annotations.append({"x": start_dt.strftime(TIME_FORMAT), "text": "Today Buffer Start", "color": "#FF9800"}) - xaxis_annotations.append({"x": end_dt.strftime(TIME_FORMAT), "text": "Today Buffer End", "color": "#FF9800"}) - - # Tomorrow's window (if available in forecast) - if getattr(self.base, "clipping_buffer_forecast_kwh", None): - forecast = self.base.clipping_buffer_forecast_kwh - tomorrow_start = 1440 - if any(forecast.get(m, 0) > 0 for m in range(tomorrow_start, tomorrow_start + 1440)): - mid_tomorrow = self.midnight_utc + timedelta(days=1) - xaxis_annotations.append({"x": mid_tomorrow.strftime(TIME_FORMAT), "text": "Tomorrow Buffer Active", "color": "#FF9800"}) - - # Clipping Remaining Time-Series - clipping_remaining_series = {} - if getattr(self.base, "clipping_buffer_forecast_kwh", None): - forecast = self.base.clipping_buffer_forecast_kwh - step_size = getattr(self.base, "plan_interval_minutes", 30) - for minute, kwh in forecast.items(): - if minute % step_size == 0: - minute_timestamp = self.midnight_utc + timedelta(minutes=minute) - stamp = minute_timestamp.strftime(TIME_FORMAT) - clipping_remaining_series[stamp] = round(kwh, 2) - - # Clipping Target SOC series (The "Red Line") - clipping_target_series = {} - if getattr(self.base, "clipping_buffer_forecast_kwh", None): - forecast = self.base.clipping_buffer_forecast_kwh - step_size = getattr(self.base, "plan_interval_minutes", 30) - for minute, kwh in forecast.items(): - if minute % step_size == 0: - minute_timestamp = self.midnight_utc + timedelta(minutes=minute) - stamp = minute_timestamp.strftime(TIME_FORMAT) - # Target is the "Ceiling" we want to stay under - target_val = max(0, self.base.soc_max - kwh) - if kwh > 0: - clipping_target_series[stamp] = round(target_val, 2) - else: - clipping_target_series[stamp] = None # Don't plot the ceiling when there is no buffer - - series_data = [ - {"name": "Clipping Ceiling", "data": clipping_target_series, "opacity": "1.0", "stroke_width": "3", "stroke_curve": "smooth", "color": "#eb2323", "unit": "kWh"}, - {"name": "Actual SOC", "data": soc_kw_best, "opacity": "1.0", "stroke_width": "3", "stroke_curve": "stepline", "color": "#9b23eb", "unit": "kWh"}, - {"name": "Clipping Remaining", "data": clipping_remaining_series, "opacity": "0.4", "stroke_width": "2", "stroke_curve": "smooth", "color": "#2196F3", "unit": "kWh"}, - {"name": "PV Power", "data": pv_power, "opacity": "1.0", "stroke_width": "3", "stroke_curve": "smooth", "color": "#f5c43d", "unit": "kW"}, - {"name": "Clipping Forecast (" + clipping_forecast_type + ")", "data": clipping_forecast, "opacity": "0.3", "stroke_width": "2", "stroke_curve": "smooth", "chart_type": "area", "color": "#a8a8a7", "unit": "kW"}, - ] - - secondary_axis = [ - { - "title": "kW", - "series_name": "PV Power", - "decimals": 1, - "opposite": True, - "min": 0, - "max": axis_max, - "tickAmount": axis_ticks, - }, - { - "title": "kW", - "series_name": "Clipping Forecast (" + clipping_forecast_type + ")", - "show": False, - "min": 0, - "max": axis_max, - "tickAmount": axis_ticks, - }, - { - "title": "kWh", - "series_name": "Clipping Remaining", - "show": False, - "min": 0, - "max": axis_max, - "tickAmount": axis_ticks, - }, - ] - - # Append dynamic stats to chart title - clipping_remaining = getattr(self.base, "clipping_remaining_today", 0.0) - clipping_mitigated = getattr(self.base, "clipping_mitigated_today", 0.0) - chart_title = "Clipping Analysis (Remaining: {:.2f} kWh, Mitigated: {:.2f} kWh)".format(clipping_remaining, clipping_mitigated) - text += self.render_chart(series_data, "kWh", chart_title, now_str, yaxis_annotations=annotations, xaxis_annotations=xaxis_annotations, extra_yaxis=secondary_axis, yaxis_min=0, yaxis_max=axis_max, yaxis_tick_amount=axis_ticks) elif chart == "PVAccuracy": # Get pv_today history once and extract total and remaining attributes per timestamp pv_today_hist = self.get_history_wrapper("sensor." + self.prefix + "_pv_today", 7, required=False) From 51dfdbe78633ca66037971b6eebb693d3f00de18 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 15 Jun 2026 14:46:46 +0100 Subject: [PATCH 17/74] Fix NameError for undefined SOC variables in clipping chart --- apps/predbat/web.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/apps/predbat/web.py b/apps/predbat/web.py index b1f0a70a1..9605c852b 100644 --- a/apps/predbat/web.py +++ b/apps/predbat/web.py @@ -2985,6 +2985,20 @@ def get_chart(self, chart): clipping_limit_effective = getattr(self.base, "clipping_limit_effective", 0) inverter_ac_limit_kw = getattr(self.base, "inverter_limit", 0) + # Fetch Target SOC (Best Plan) + soc_kw_best_raw = history_attribute(self.get_history_wrapper(self.prefix + ".soc_kw_best", 2)) + soc_kw_best = prune_today(soc_kw_best_raw, self.now_utc, self.midnight_utc) + + # Fetch Actual SOC history + soc_kw_h0 = {} + hist = self.base.soc_kw_h0 + if hist: + for minute in hist: + minute_timestamp = self.midnight_utc + timedelta(minutes=minute) + stamp = minute_timestamp.strftime(TIME_FORMAT) + soc_kw_h0[stamp] = hist.get(self.minutes_now - minute, 0) + soc_kw_h0[now_str] = self.base.soc_kw + # Re-fetch PV actuals for overlay pv_power_hist = history_attribute(self.get_history_wrapper(self.prefix + ".pv_power", 7, required=False)) pv_power = prune_today(pv_power_hist, self.now_utc, self.midnight_utc, prune=False) From 2664ad5291061680ba678294db2a951b68d4f4b6 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 15 Jun 2026 16:21:37 +0100 Subject: [PATCH 18/74] Fix AttributeError in Clipping chart --- apps/predbat/web.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/predbat/web.py b/apps/predbat/web.py index 9605c852b..74a631234 100644 --- a/apps/predbat/web.py +++ b/apps/predbat/web.py @@ -2991,9 +2991,9 @@ def get_chart(self, chart): # Fetch Actual SOC history soc_kw_h0 = {} - hist = self.base.soc_kw_h0 - if hist: - for minute in hist: + if self.base.soc_kwh_history: + hist = self.base.soc_kwh_history + for minute in range(0, self.minutes_now, self.plan_interval_minutes): minute_timestamp = self.midnight_utc + timedelta(minutes=minute) stamp = minute_timestamp.strftime(TIME_FORMAT) soc_kw_h0[stamp] = hist.get(self.minutes_now - minute, 0) From 776c02feae71950190e62af898761b7995d2a0eb Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 15 Jun 2026 16:26:08 +0100 Subject: [PATCH 19/74] Use getattr for soc_kwh_history to fix AttributeError --- apps/predbat/web.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/predbat/web.py b/apps/predbat/web.py index 74a631234..61fa2cdff 100644 --- a/apps/predbat/web.py +++ b/apps/predbat/web.py @@ -2991,8 +2991,8 @@ def get_chart(self, chart): # Fetch Actual SOC history soc_kw_h0 = {} - if self.base.soc_kwh_history: - hist = self.base.soc_kwh_history + hist = getattr(self.base, "soc_kwh_history", {}) + if hist: for minute in range(0, self.minutes_now, self.plan_interval_minutes): minute_timestamp = self.midnight_utc + timedelta(minutes=minute) stamp = minute_timestamp.strftime(TIME_FORMAT) From 4872978b7cc0a74ac20783001eae0bd4ba0311e6 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 15 Jun 2026 17:02:23 +0100 Subject: [PATCH 20/74] Remove unsupported annotations kwargs from render_chart --- apps/predbat/web.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/apps/predbat/web.py b/apps/predbat/web.py index 61fa2cdff..648ae20db 100644 --- a/apps/predbat/web.py +++ b/apps/predbat/web.py @@ -3008,11 +3008,7 @@ def get_chart(self, chart): axis_max = max(self.base.soc_max, inverter_ac_limit_kw, 12.0) axis_ticks = 6 - annotations = [] - if clipping_limit_effective > 0: - annotations.append({"y": clipping_limit_effective, "text": "{} ({} kW)".format(clipping_mode, round(clipping_limit_effective, 2)), "color": "#FF0000"}) - if inverter_ac_limit_kw > 0 and abs(inverter_ac_limit_kw - clipping_limit_effective) > 0.1: - annotations.append({"y": inverter_ac_limit_kw, "text": "Inverter Capacity ({} kW)".format(round(inverter_ac_limit_kw, 2)), "color": "#999999"}) + # Data series parsing for per-minute data step_size = getattr(self.base, "plan_interval_minutes", 30) @@ -3073,7 +3069,7 @@ def get_chart(self, chart): chart_title = "Clipping Analysis (Expected Total Clipping: {:.2f} kWh)".format(clipping_total) - text += self.render_chart(series_data, "kWh", chart_title, now_str, yaxis_annotations=annotations, extra_yaxis=secondary_axis, yaxis_min=0, yaxis_max=axis_max, yaxis_tick_amount=axis_ticks) + text += self.render_chart(series_data, "kWh", chart_title, now_str, extra_yaxis=secondary_axis) elif chart == "LoadML": load_today_history = self.get_history_with_now_attrs("sensor." + self.prefix + "_load_ml_stats", 7) # Get historical load data for last 24 hours From 3fc257601206876d47a7d988e462b9647fb59dd9 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 15 Jun 2026 17:27:15 +0100 Subject: [PATCH 21/74] Remove secondary_axis from clipping chart to fix apexcharts axis mapping bug --- apps/predbat/web.py | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/apps/predbat/web.py b/apps/predbat/web.py index 648ae20db..2cd7c4e2a 100644 --- a/apps/predbat/web.py +++ b/apps/predbat/web.py @@ -3043,25 +3043,7 @@ def get_chart(self, chart): {"name": "Forecast Clipping Total", "data": clipping_forecast_series, "opacity": "1.0", "stroke_width": "2", "stroke_curve": "smooth", "chart_type": "area", "color": "#FF0000", "unit": "kWh"}, ] - secondary_axis = [ - { - "title": "kW", - "series_name": "PV Power Actual", - "decimals": 1, - "opposite": True, - "min": 0, - "max": axis_max, - "tickAmount": axis_ticks, - }, - { - "title": "kW", - "series_name": raw_pv_name, - "show": False, - "min": 0, - "max": axis_max, - "tickAmount": axis_ticks, - }, - ] + clipping_total = 0 if getattr(self.base, "predict_clipped_best", None): @@ -3069,7 +3051,7 @@ def get_chart(self, chart): chart_title = "Clipping Analysis (Expected Total Clipping: {:.2f} kWh)".format(clipping_total) - text += self.render_chart(series_data, "kWh", chart_title, now_str, extra_yaxis=secondary_axis) + text += self.render_chart(series_data, "kWh", chart_title, now_str) elif chart == "LoadML": load_today_history = self.get_history_with_now_attrs("sensor." + self.prefix + "_load_ml_stats", 7) # Get historical load data for last 24 hours From b566a66967c38f2cc8ee8f9690751ac59767fcbe Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 15 Jun 2026 20:18:21 +0100 Subject: [PATCH 22/74] Implement clipping remaining, clipping ceiling, and chart annotations to match legacy mockup --- apps/predbat/plan.py | 41 ++++++++++++++++++++ apps/predbat/web.py | 91 +++++++++++++++++++++++++++++++++++++++----- 2 files changed, 123 insertions(+), 9 deletions(-) diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index 1bdfe183e..4f0c5e81d 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -3976,6 +3976,47 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi clipping_total = self.predict_clipped_best.get(keys[-1], 0.0) clipping_tomorrow = max(0.0, clipping_total - clipping_today) + # Generate Clipping Visual Series (Remaining and Ceiling) for web.py charts + predict_clipping_remaining_best = {} + predict_clipping_ceiling_best = {} + + clipping_limit_step = self.clipping_limit * (self.minutes_now_step / 60) + pv_forecast_peak_step = getattr(self, 'pv_forecast_minute_stepCS', getattr(self, 'pv_forecast_minute_step90', None)) + + manual_buffer_active = False + if self.clipping_buffer_kwh > 0: + if self.clipping_buffer_start is None or self.clipping_buffer_end is None: + manual_buffer_active = True + + cumulative_clip = 0.0 + max_minute = self.forecast_minutes + + buffer_start = self.clipping_buffer_start if self.clipping_buffer_start else 0 + buffer_end = self.clipping_buffer_end if self.clipping_buffer_end else 24 * 60 + + for minute in range(max_minute, -self.minutes_now_step, -self.minutes_now_step): + remaining = 0.0 + minute_absolute = minute + self.minutes_now + + if manual_buffer_active or (self.clipping_buffer_kwh > 0 and buffer_start <= minute_absolute < buffer_end): + # Calculate linear decay for manual mode visual graph + if buffer_end > buffer_start: + progress = max(0.0, min(1.0, (minute_absolute - buffer_start) / (buffer_end - buffer_start))) + remaining = self.clipping_buffer_kwh * (1.0 - progress) + else: + remaining = self.clipping_buffer_kwh + elif pv_forecast_peak_step and clipping_limit_step > 0 and self.clipping_cost_weight > 0: + peak_pv = pv_forecast_peak_step.get(minute, 0) + if peak_pv > clipping_limit_step: + cumulative_clip += (peak_pv - clipping_limit_step) + remaining = cumulative_clip + + predict_clipping_remaining_best[minute] = round(remaining, 4) + predict_clipping_ceiling_best[minute] = round(self.soc_max - remaining, 4) + + self.predict_clipping_remaining_best = predict_clipping_remaining_best + self.predict_clipping_ceiling_best = predict_clipping_ceiling_best + self.clipping_remaining_today = clipping_today self.clipping_tomorrow = clipping_tomorrow self.clipping_mitigated_today = clipping_today diff --git a/apps/predbat/web.py b/apps/predbat/web.py index 2cd7c4e2a..87249beb3 100644 --- a/apps/predbat/web.py +++ b/apps/predbat/web.py @@ -1564,7 +1564,7 @@ def get_chart_series(self, name, results, chart_type, color): text += " }\n" return text - def render_chart(self, series_data, yaxis_name, chart_name, now_str, tagname="chart", daily_chart=True, extra_yaxis=None): + def render_chart(self, series_data, yaxis_name, chart_name, now_str, tagname="chart", daily_chart=True, extra_yaxis=None, xaxis_annotations=None, yaxis_annotations=None): """ Render a chart """ @@ -1744,6 +1744,22 @@ def render_chart(self, series_data, yaxis_name, chart_name, now_str, tagname="ch text += " },\n" text += " annotations: {\n" text += " xaxis: [\n" + if xaxis_annotations: + for item in xaxis_annotations: + text += " {\n" + text += " x: new Date('{}').getTime(),\n".format(item.get("x", "")) + text += " borderColor: '{}',\n".format(item.get("borderColor", "#000000")) + text += " strokeDashArray: {},\n".format(item.get("strokeDashArray", 0)) + text += " textAnchor: '{}',\n".format(item.get("textAnchor", "middle")) + text += " label: {\n" + text += " text: '{}',\n".format(item.get("text", "")) + text += " orientation: '{}',\n".format(item.get("orientation", "horizontal")) + text += " style: {\n" + text += " color: '{}',\n".format(item.get("textColor", "#fff")) + text += " background: '{}',\n".format(item.get("backgroundColor", "#775DD0")) + text += " }\n" + text += " }\n" + text += " },\n" text += " {\n" text += " x: new Date('{}').getTime(),\n".format(now_str) text += " borderColor: '#775DD0',\n" @@ -1760,6 +1776,22 @@ def render_chart(self, series_data, yaxis_name, chart_name, now_str, tagname="ch text += " text: 'midnight'\n" text += " }\n" text += " }\n" + text += " ],\n" + text += " yaxis: [\n" + if yaxis_annotations: + for item in yaxis_annotations: + text += " {\n" + text += " y: {},\n".format(item.get("y", 0)) + text += " borderColor: '{}',\n".format(item.get("borderColor", "#000000")) + text += " strokeDashArray: {},\n".format(item.get("strokeDashArray", 0)) + text += " label: {\n" + text += " text: '{}',\n".format(item.get("text", "")) + text += " style: {\n" + text += " color: '{}',\n".format(item.get("textColor", "#fff")) + text += " background: '{}',\n".format(item.get("backgroundColor", "#FF0000")) + text += " }\n" + text += " }\n" + text += " },\n" text += " ]\n" text += " }\n" text += "}\n" @@ -3012,13 +3044,21 @@ def get_chart(self, chart): # Data series parsing for per-minute data step_size = getattr(self.base, "plan_interval_minutes", 30) - clipping_forecast_series = {} - if getattr(self.base, "predict_clipped_best", None): - for minute, kwh in self.base.predict_clipped_best.items(): + clipping_remaining_series = {} + if getattr(self.base, "predict_clipping_remaining_best", None): + for minute, kwh in self.base.predict_clipping_remaining_best.items(): if minute % step_size == 0: minute_timestamp = self.midnight_utc + timedelta(minutes=minute) stamp = minute_timestamp.strftime(TIME_FORMAT) - clipping_forecast_series[stamp] = round(kwh, 2) + clipping_remaining_series[stamp] = round(kwh, 2) + + clipping_ceiling_series = {} + if getattr(self.base, "predict_clipping_ceiling_best", None): + for minute, kwh in self.base.predict_clipping_ceiling_best.items(): + if minute % step_size == 0: + minute_timestamp = self.midnight_utc + timedelta(minutes=minute) + stamp = minute_timestamp.strftime(TIME_FORMAT) + clipping_ceiling_series[stamp] = round(kwh, 2) # Raw PV forecast (ClearSky if used, else regular) raw_pv_series = {} @@ -3036,14 +3076,47 @@ def get_chart(self, chart): raw_pv_series[stamp] = round(kw, 2) series_data = [ + {"name": "Clipping Remaining", "data": clipping_remaining_series, "opacity": "0.5", "stroke_width": "2", "stroke_curve": "smooth", "chart_type": "area", "color": "#1877f2", "unit": "kWh"}, {"name": "Actual SOC", "data": soc_kw_h0, "opacity": "1.0", "stroke_width": "2", "stroke_curve": "smooth", "color": "#3291a8", "unit": "kWh"}, {"name": "Target SOC", "data": soc_kw_best, "opacity": "1.0", "stroke_width": "3", "stroke_curve": "stepline", "color": "#9b23eb", "unit": "kWh"}, + {"name": "Clipping Ceiling", "data": clipping_ceiling_series, "opacity": "1.0", "stroke_width": "2", "stroke_curve": "stepline", "color": "#eb3434", "unit": "kWh"}, {"name": "PV Power Actual", "data": pv_power, "opacity": "1.0", "stroke_width": "3", "stroke_curve": "smooth", "color": "#f5c43d", "unit": "kW"}, - {"name": raw_pv_name, "data": raw_pv_series, "opacity": "0.3", "stroke_width": "2", "stroke_curve": "smooth", "chart_type": "area", "color": "#a8a8a7", "unit": "kW"}, - {"name": "Forecast Clipping Total", "data": clipping_forecast_series, "opacity": "1.0", "stroke_width": "2", "stroke_curve": "smooth", "chart_type": "area", "color": "#FF0000", "unit": "kWh"}, + {"name": "Clipping Forecast ({})".format(raw_pv_name), "data": raw_pv_series, "opacity": "0.3", "stroke_width": "2", "stroke_curve": "smooth", "chart_type": "area", "color": "#a8a8a7", "unit": "kW"}, ] - + yaxis_annotations = [] + if inverter_ac_limit_kw > 0: + yaxis_annotations.append({ + "y": inverter_ac_limit_kw, + "borderColor": "#FF0000", + "strokeDashArray": 4, + "text": "Inverter AC Capacity ({} kW)".format(inverter_ac_limit_kw), + "textColor": "#fff", + "backgroundColor": "#FF0000" + }) + + xaxis_annotations = [] + buffer_start = getattr(self.base, "clipping_buffer_start", None) + buffer_end = getattr(self.base, "clipping_buffer_end", None) + if getattr(self.base, "clipping_buffer_kwh", 0) > 0 and buffer_start is not None and buffer_end is not None: + start_stamp = (self.midnight_utc + timedelta(minutes=buffer_start)).strftime(TIME_FORMAT) + end_stamp = (self.midnight_utc + timedelta(minutes=buffer_end)).strftime(TIME_FORMAT) + xaxis_annotations.append({ + "x": start_stamp, + "borderColor": "#ffa500", + "strokeDashArray": 4, + "text": "Today Buffer Start", + "orientation": "vertical", + "backgroundColor": "#ffa500" + }) + xaxis_annotations.append({ + "x": end_stamp, + "borderColor": "#ffa500", + "strokeDashArray": 4, + "text": "Today Buffer End", + "orientation": "vertical", + "backgroundColor": "#ffa500" + }) clipping_total = 0 if getattr(self.base, "predict_clipped_best", None): @@ -3051,7 +3124,7 @@ def get_chart(self, chart): chart_title = "Clipping Analysis (Expected Total Clipping: {:.2f} kWh)".format(clipping_total) - text += self.render_chart(series_data, "kWh", chart_title, now_str) + text += self.render_chart(series_data, "kWh", chart_title, now_str, xaxis_annotations=xaxis_annotations, yaxis_annotations=yaxis_annotations) elif chart == "LoadML": load_today_history = self.get_history_with_now_attrs("sensor." + self.prefix + "_load_ml_stats", 7) # Get historical load data for last 24 hours From f459a222a64ad7c8f32228d18088155efc4fd3cb Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 15 Jun 2026 20:36:45 +0100 Subject: [PATCH 23/74] Extend clipping chart to show 48 hours of historical data --- apps/predbat/web.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/apps/predbat/web.py b/apps/predbat/web.py index 87249beb3..136978873 100644 --- a/apps/predbat/web.py +++ b/apps/predbat/web.py @@ -3018,22 +3018,24 @@ def get_chart(self, chart): inverter_ac_limit_kw = getattr(self.base, "inverter_limit", 0) # Fetch Target SOC (Best Plan) - soc_kw_best_raw = history_attribute(self.get_history_wrapper(self.prefix + ".soc_kw_best", 2)) - soc_kw_best = prune_today(soc_kw_best_raw, self.now_utc, self.midnight_utc) + soc_kw_best_raw = history_attribute(self.get_history_wrapper(self.prefix + ".soc_kw_best", 3)) + soc_kw_best = prune_today(soc_kw_best_raw, self.now_utc, self.midnight_utc, prune_past_days=2) - # Fetch Actual SOC history + # Fetch Actual SOC history (48 hours into the past) soc_kw_h0 = {} hist = getattr(self.base, "soc_kwh_history", {}) if hist: - for minute in range(0, self.minutes_now, self.plan_interval_minutes): - minute_timestamp = self.midnight_utc + timedelta(minutes=minute) - stamp = minute_timestamp.strftime(TIME_FORMAT) - soc_kw_h0[stamp] = hist.get(self.minutes_now - minute, 0) - soc_kw_h0[now_str] = self.base.soc_kw + for minute in range(-2 * 24 * 60, self.minutes_now, self.plan_interval_minutes): + val = hist.get(self.minutes_now - minute, None) + if val is not None: + minute_timestamp = self.midnight_utc + timedelta(minutes=minute) + stamp = minute_timestamp.strftime(TIME_FORMAT) + soc_kw_h0[stamp] = round(val, 2) + soc_kw_h0[now_str] = round(self.base.soc_kw, 2) - # Re-fetch PV actuals for overlay - pv_power_hist = history_attribute(self.get_history_wrapper(self.prefix + ".pv_power", 7, required=False)) - pv_power = prune_today(pv_power_hist, self.now_utc, self.midnight_utc, prune=False) + # Re-fetch PV actuals for overlay (48 hours into the past) + pv_power_hist = history_attribute(self.get_history_wrapper(self.prefix + ".pv_power", 3, required=False)) + pv_power = prune_today(pv_power_hist, self.now_utc, self.midnight_utc, prune=True, prune_past_days=2) axis_max = 12.0 if self.base.soc_max > 12.0 or inverter_ac_limit_kw > 12.0: From 81d282c9fe752353ec184684fb183d6033b604f6 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 15 Jun 2026 20:56:04 +0100 Subject: [PATCH 24/74] Expose clipping remaining and clipping ceiling arrays as HA sensors to enable historical tracking --- apps/predbat/plan.py | 16 ++++++++++++++-- apps/predbat/web.py | 11 +++++++---- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index 4f0c5e81d..2e545715a 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -4023,15 +4023,27 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi # Add Clipping Summary Dashboard Items self.dashboard_item( - self.prefix + ".clipping_remaining_today", + self.prefix + ".clipping_remaining", state=dp2(self.clipping_remaining_today), attributes={ - "friendly_name": "Clipping Remaining Today", + "results": self.filtered_times(self.predict_clipping_remaining_best), + "friendly_name": "Clipping Remaining", "unit_of_measurement": "kWh", "device_class": "energy", "icon": "mdi:solar-power-variant", }, ) + self.dashboard_item( + self.prefix + ".clipping_ceiling", + state=dp2(self.predict_clipping_ceiling_best.get(0, self.soc_max)), + attributes={ + "results": self.filtered_times(self.predict_clipping_ceiling_best), + "friendly_name": "Clipping Ceiling", + "unit_of_measurement": "kWh", + "device_class": "energy", + "icon": "mdi:arrow-collapse-up", + }, + ) self.dashboard_item( self.prefix + ".clipping_tomorrow", state=dp2(self.clipping_tomorrow), diff --git a/apps/predbat/web.py b/apps/predbat/web.py index 136978873..ebad8d22f 100644 --- a/apps/predbat/web.py +++ b/apps/predbat/web.py @@ -3046,16 +3046,19 @@ def get_chart(self, chart): # Data series parsing for per-minute data step_size = getattr(self.base, "plan_interval_minutes", 30) - clipping_remaining_series = {} - if getattr(self.base, "predict_clipping_remaining_best", None): + clipping_remaining_raw = history_attribute(self.get_history_wrapper(self.prefix + ".clipping_remaining", 3)) + clipping_remaining_series = prune_today(clipping_remaining_raw, self.now_utc, self.midnight_utc, prune_past_days=2) + # If history is missing/empty, fallback to the base predictor + if not clipping_remaining_series and getattr(self.base, "predict_clipping_remaining_best", None): for minute, kwh in self.base.predict_clipping_remaining_best.items(): if minute % step_size == 0: minute_timestamp = self.midnight_utc + timedelta(minutes=minute) stamp = minute_timestamp.strftime(TIME_FORMAT) clipping_remaining_series[stamp] = round(kwh, 2) - clipping_ceiling_series = {} - if getattr(self.base, "predict_clipping_ceiling_best", None): + clipping_ceiling_raw = history_attribute(self.get_history_wrapper(self.prefix + ".clipping_ceiling", 3)) + clipping_ceiling_series = prune_today(clipping_ceiling_raw, self.now_utc, self.midnight_utc, prune_past_days=2) + if not clipping_ceiling_series and getattr(self.base, "predict_clipping_ceiling_best", None): for minute, kwh in self.base.predict_clipping_ceiling_best.items(): if minute % step_size == 0: minute_timestamp = self.midnight_utc + timedelta(minutes=minute) From 8ff9bd42b522c1cfbb8ad5224750151c8d8f3850 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 15 Jun 2026 21:45:48 +0100 Subject: [PATCH 25/74] Fix clipping_limit_effective attribute error --- apps/predbat/plan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index 2e545715a..54354b41f 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -3980,7 +3980,7 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi predict_clipping_remaining_best = {} predict_clipping_ceiling_best = {} - clipping_limit_step = self.clipping_limit * (self.minutes_now_step / 60) + clipping_limit_step = getattr(self, "clipping_limit_effective", 0) * (self.minutes_now_step / 60.0) pv_forecast_peak_step = getattr(self, 'pv_forecast_minute_stepCS', getattr(self, 'pv_forecast_minute_step90', None)) manual_buffer_active = False From 06587bd3fb6f989f41533650712756966cc1f78b Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 15 Jun 2026 21:51:32 +0100 Subject: [PATCH 26/74] Fix step usage in backward loop --- apps/predbat/plan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index 54354b41f..1e8b87aa0 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -3980,7 +3980,7 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi predict_clipping_remaining_best = {} predict_clipping_ceiling_best = {} - clipping_limit_step = getattr(self, "clipping_limit_effective", 0) * (self.minutes_now_step / 60.0) + clipping_limit_step = getattr(self, "clipping_limit_effective", 0) * (step / 60.0) pv_forecast_peak_step = getattr(self, 'pv_forecast_minute_stepCS', getattr(self, 'pv_forecast_minute_step90', None)) manual_buffer_active = False @@ -3994,7 +3994,7 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi buffer_start = self.clipping_buffer_start if self.clipping_buffer_start else 0 buffer_end = self.clipping_buffer_end if self.clipping_buffer_end else 24 * 60 - for minute in range(max_minute, -self.minutes_now_step, -self.minutes_now_step): + for minute in range(max_minute, -step, -step): remaining = 0.0 minute_absolute = minute + self.minutes_now From 585cddce1c6172f06f17025513b1aa76d591993e Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 16 Jun 2026 08:37:36 +0100 Subject: [PATCH 27/74] Fix clipping functionality, correct plan limit looping, and update tests --- apps/predbat/output.py | 57 +-------------------------- apps/predbat/plan.py | 3 ++ apps/predbat/tests/test_download.py | 4 +- apps/predbat/tests/test_load_ml.py | 2 +- apps/predbat/tests/test_open_meteo.py | 18 ++++----- apps/predbat/tests/test_solcast.py | 2 +- apps/predbat/web.py | 8 +++- 7 files changed, 24 insertions(+), 70 deletions(-) diff --git a/apps/predbat/output.py b/apps/predbat/output.py index 8a05b9486..ae1c8214b 100644 --- a/apps/predbat/output.py +++ b/apps/predbat/output.py @@ -2470,61 +2470,8 @@ def record_status(self, message, debug="", had_errors=False, notify=False, extra }, ) - # Clipping Status - clipping_status_text = "No clipping forecast." - clipping_mode = getattr(self, "clipping_limit_mode", "Unknown") - clipping_total = 0.0 - - predict_clipped_best = getattr(self, "predict_clipped_best", {}) - if predict_clipped_best: - clipping_total = predict_clipped_best.get(max(predict_clipped_best.keys()), 0.0) - if clipping_total > 0.01: - start_str = "" - end_str = "" - start_stamp = None - end_stamp = None - - # Find start time (first increase) - prev_val = 0.0 - for min_key, val in sorted(predict_clipped_best.items()): - if val > prev_val + 0.001: - if start_stamp is None: - start_stamp = self.midnight_utc + timedelta(minutes=min_key) - end_stamp = self.midnight_utc + timedelta(minutes=min_key) - prev_val = val - - if start_stamp and end_stamp: - start_str = start_stamp.strftime("%H:%M") - end_str = end_stamp.strftime("%H:%M") - - clipping_status_text = "Forecast {} kWh clipping, exceeding {} limit from {} to {}. Plan penalized to mitigate.".format( - dp2(clipping_total), clipping_mode, start_str, end_str - ) - - self.dashboard_item( - self.prefix + ".clipping_status", - state=clipping_status_text, - attributes={ - "friendly_name": "Clipping Status", - "icon": "mdi:solar-power", - "clipping_mode": clipping_mode, - "expected_total_kwh": dp2(clipping_total) - }, - ) - - pv_forecast_peak_step = getattr(self, "pv_forecast_peak_step", {}) - if pv_forecast_peak_step: - self.dashboard_item( - self.prefix + ".pv_forecast_peak", - state=dp2(pv_forecast_peak_step.get(self.minutes_now, 0)), - attributes={ - "results": self.filtered_times(pv_forecast_peak_step), - "friendly_name": "Solar peak forecast", - "state_class": "measurement", - "unit_of_measurement": "kW", - "icon": "mdi:solar-power", - }, - ) + # Clipping Status and PV Peak Forecast sensors are published in plan.py run_prediction(save="best") + # to avoid duplicate dashboard_item writes to the same sensor. if had_errors: diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index 1e8b87aa0..14d1aa058 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -1120,6 +1120,9 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): pv_forecast_peak_step=pv_forecast_peak_step, clipping_limit=clipping_limit_effective, clipping_cost_weight=self.clipping_cost_weight if self.clipping_peak_enable else 0, + clipping_buffer_kwh=self.clipping_buffer_kwh, + clipping_buffer_start=self.clipping_buffer_start, + clipping_buffer_end=self.clipping_buffer_end, ) # Check if LoadML is active and disable thread pools as it causes lockup due to race conditions with NumPy diff --git a/apps/predbat/tests/test_download.py b/apps/predbat/tests/test_download.py index de386381c..6e6549959 100644 --- a/apps/predbat/tests/test_download.py +++ b/apps/predbat/tests/test_download.py @@ -257,8 +257,8 @@ def _test_compute_file_sha1(my_predbat): Test Git blob SHA1 hash computation (matches GitHub's SHA) """ # Create a temporary file with known content - with tempfile.NamedTemporaryFile(mode="w", delete=False) as f: - f.write("test content\n") + with tempfile.NamedTemporaryFile(mode="wb", delete=False) as f: + f.write(b"test content\n") temp_path = f.name try: diff --git a/apps/predbat/tests/test_load_ml.py b/apps/predbat/tests/test_load_ml.py index 2a5ab241f..16372b5c9 100644 --- a/apps/predbat/tests/test_load_ml.py +++ b/apps/predbat/tests/test_load_ml.py @@ -1853,7 +1853,7 @@ def get_state_wrapper(self, entity_id, default=None, attribute=None, refresh=Fal def fetch_pv_forecast(self): """Mock fetch_pv_forecast - returns empty forecasts""" - return {}, {} + return {}, {}, {}, {}, {} def minute_data_import_export(self, days, now_utc, entity, scale=1.0, increment=False, smoothing=False, required_unit=None): """Mock minute_data_import_export - returns empty dict""" diff --git a/apps/predbat/tests/test_open_meteo.py b/apps/predbat/tests/test_open_meteo.py index fb9bec7c6..10e7da779 100644 --- a/apps/predbat/tests/test_open_meteo.py +++ b/apps/predbat/tests/test_open_meteo.py @@ -76,17 +76,17 @@ def create_mock_session(*args, **kwargs): kwp = 3.0 system_loss = 0.0 # simplify: 0% loss so kW = GTI_kWm2 * kwp with patch("solcast.aiohttp.ClientSession", side_effect=create_mock_session): - result = run_async(test_api.solar.download_open_meteo_ensemble_data(51.5, -0.1, 35, 0, kwp, system_loss)) + result_p10, result_p90 = run_async(test_api.solar.download_open_meteo_ensemble_data(51.5, -0.1, 35, 0, kwp, system_loss)) # For 3 members at 2025-06-15T12:00: [400, 450, 480] sorted # p10_idx = max(0, int(3 * 0.1) - 1) = 0 -> gti_p10 = 400 # kW = (400 / 1000) * 3.0 * (1 - 0.0) = 1.2 expected_12 = round((400.0 / 1000.0) * kwp * (1.0 - system_loss), 4) - if "2025-06-15T12:00" not in result: - print("ERROR: Expected key '2025-06-15T12:00' in ensemble result") + if "2025-06-15T12:00" not in result_p10: + print("ERROR: Expected key '2025-06-15T12:00' in ensemble result_p10") failed = True - elif abs(result["2025-06-15T12:00"] - expected_12) > 0.001: - print(f"ERROR: ensemble p10 at 12:00: expected {expected_12}, got {result['2025-06-15T12:00']}") + elif abs(result_p10["2025-06-15T12:00"] - expected_12) > 0.001: + print(f"ERROR: ensemble p10 at 12:00: expected {expected_12}, got {result_p10['2025-06-15T12:00']}") failed = True finally: test_api.cleanup() @@ -114,8 +114,8 @@ def create_mock_session(*args, **kwargs): with patch("solcast.aiohttp.ClientSession", side_effect=create_mock_session): result = run_async(test_api.solar.download_open_meteo_ensemble_data(51.5, -0.1, 35, 0, 3.0, 0.14)) - if result != {}: - print(f"ERROR: Expected empty dict, got {result}") + if result != ({}, {}): + print(f"ERROR: Expected empty dict tuple, got {result}") failed = True finally: test_api.cleanup() @@ -141,8 +141,8 @@ def create_mock_session(*args, **kwargs): with patch("solcast.aiohttp.ClientSession", side_effect=create_mock_session): result = run_async(test_api.solar.download_open_meteo_ensemble_data(51.5, -0.1, 35, 0, 3.0, 0.14)) - if result != {}: - print(f"ERROR: Expected empty dict on HTTP failure, got {result}") + if result != ({}, {}): + print(f"ERROR: Expected empty dict tuple on HTTP failure, got {result}") failed = True finally: test_api.cleanup() diff --git a/apps/predbat/tests/test_solcast.py b/apps/predbat/tests/test_solcast.py index 4e5441ef0..4a113b8c1 100644 --- a/apps/predbat/tests/test_solcast.py +++ b/apps/predbat/tests/test_solcast.py @@ -3096,7 +3096,7 @@ def mock_minute_import_export(max_days_prev, now_utc, key, scale=1.0, required_u pv_forecast_hist[minutes_ago] = float(FORECAST_KW) with patch("solcast.history_attribute_to_minute_data", return_value=(pv_forecast_hist, days)): - adj_m, adj_m10, adj_data = solar.pv_calibration(pv_m, pv_m10, pv_data, create_pv10=True, divide_by=divide_by_factor, max_kwh=10.0, forecast_days=solar.forecast_days, period=FORECAST_PERIOD) + adj_m, adj_m10, adj_data, _ = solar.pv_calibration(pv_m, pv_m10, pv_data, create_pv10=True, divide_by=divide_by_factor, max_kwh=10.0, forecast_days=solar.forecast_days, period=FORECAST_PERIOD) # Each 15-min entry should be annotated with the single 30-min plan slot that # starts at the entry timestamp. slots_per_period=max(1,round(15/30))=1, so diff --git a/apps/predbat/web.py b/apps/predbat/web.py index ebad8d22f..ec3d90a0f 100644 --- a/apps/predbat/web.py +++ b/apps/predbat/web.py @@ -2970,26 +2970,32 @@ def get_chart(self, chart): pv_power = prune_today(pv_power_hist, self.now_utc, self.midnight_utc, prune=chart == "PV") pv_forecast_hist = history_attribute(self.get_history_wrapper("sensor." + self.prefix + "_pv_forecast_h0", 7, required=False)) pv_forecast_histCL = history_attribute(self.get_history_wrapper("sensor." + self.prefix + "_pv_forecast_h0", 7, required=False), attributes=True, state_key="nowCL") + pv_forecast_histCS = history_attribute(self.get_history_wrapper("sensor." + self.prefix + "_pv_forecast_h0", 7, required=False), attributes=True, state_key="nowCS") pv_forecast = prune_today(pv_forecast_hist, self.now_utc, self.midnight_utc, prune=chart == "PV", intermediate=True) pv_forecastCL = prune_today(pv_forecast_histCL, self.now_utc, self.midnight_utc, prune=chart == "PV", intermediate=True) + pv_forecastCS = prune_today(pv_forecast_histCS, self.now_utc, self.midnight_utc, prune=chart == "PV", intermediate=True) pv_today_forecast = prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + "_pv_today", "pv_estimate"), self.now_utc, self.midnight_utc, prune=False, intermediate=True) pv_today_forecast10 = prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + "_pv_today", "pv_estimate10"), self.now_utc, self.midnight_utc, prune=False, intermediate=True) pv_today_forecast90 = prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + "_pv_today", "pv_estimate90"), self.now_utc, self.midnight_utc, prune=False, intermediate=True) pv_today_forecastCL = prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + "_pv_today", "pv_estimateCL"), self.now_utc, self.midnight_utc, prune=False, intermediate=True) + pv_today_forecastCS = prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + "_pv_today", "pv_clearsky"), self.now_utc, self.midnight_utc, prune=False, intermediate=True) pv_today_forecast.update(prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + "_pv_tomorrow", "pv_estimate"), self.now_utc, self.midnight_utc, prune=False, intermediate=True)) pv_today_forecast10.update(prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + "_pv_tomorrow", "pv_estimate10"), self.now_utc, self.midnight_utc, prune=False, intermediate=True)) pv_today_forecast90.update(prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + "_pv_tomorrow", "pv_estimate90"), self.now_utc, self.midnight_utc, prune=False, intermediate=True)) pv_today_forecastCL.update(prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + "_pv_tomorrow", "pv_estimateCL"), self.now_utc, self.midnight_utc, prune=False, intermediate=True)) + pv_today_forecastCS.update(prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + "_pv_tomorrow", "pv_clearsky"), self.now_utc, self.midnight_utc, prune=False, intermediate=True)) series_data = [ {"name": "PV Power", "data": pv_power, "opacity": "1.0", "stroke_width": "3", "stroke_curve": "smooth", "color": "#f5c43d"}, {"name": "Forecast History", "data": pv_forecast, "opacity": "0.3", "stroke_width": "3", "stroke_curve": "smooth", "color": "#a8a8a7", "chart_type": "area"}, {"name": "Forecast History CL", "data": pv_forecastCL, "opacity": "0.3", "stroke_width": "3", "stroke_curve": "smooth", "color": "#e90a0a", "chart_type": "area"}, + {"name": "Forecast History CS", "data": pv_forecastCS, "opacity": "0.3", "stroke_width": "3", "stroke_curve": "smooth", "color": "#1877f2", "chart_type": "area"}, {"name": "Forecast", "data": pv_today_forecast, "opacity": "0.3", "stroke_width": "2", "stroke_curve": "smooth", "chart_type": "area", "color": "#a8a8a7"}, {"name": "Forecast 10%", "data": pv_today_forecast10, "opacity": "0.3", "stroke_width": "2", "stroke_curve": "smooth", "chart_type": "area", "color": "#6b6b6b"}, {"name": "Forecast 90%", "data": pv_today_forecast90, "opacity": "0.3", "stroke_width": "2", "stroke_curve": "smooth", "chart_type": "area", "color": "#cccccc"}, {"name": "Forecast CL", "data": pv_today_forecastCL, "opacity": "0.3", "stroke_width": "2", "stroke_curve": "smooth", "chart_type": "area", "color": "#e90a0a"}, + {"name": "Forecast CS", "data": pv_today_forecastCS, "opacity": "0.3", "stroke_width": "2", "stroke_curve": "smooth", "chart_type": "area", "color": "#1877f2"}, ] text += self.render_chart(series_data, "kW", "Solar Forecast", now_str) @@ -3042,8 +3048,6 @@ def get_chart(self, chart): axis_max = max(self.base.soc_max, inverter_ac_limit_kw, 12.0) axis_ticks = 6 - - # Data series parsing for per-minute data step_size = getattr(self.base, "plan_interval_minutes", 30) clipping_remaining_raw = history_attribute(self.get_history_wrapper(self.prefix + ".clipping_remaining", 3)) From 91e2f766e5040c02a07ba4240ed9e48b8ea1eb12 Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 16 Jun 2026 09:30:04 +0100 Subject: [PATCH 28/74] Fix clipping chart inverter limit and enhance visualization --- apps/predbat/web.py | 39 +++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/apps/predbat/web.py b/apps/predbat/web.py index ec3d90a0f..32efb4d52 100644 --- a/apps/predbat/web.py +++ b/apps/predbat/web.py @@ -1634,6 +1634,7 @@ def render_chart(self, series_data, yaxis_name, chart_name, now_str, tagname="ch opacity = [] stroke_width = [] stroke_curve = [] + stroke_dasharray = [] series_units = [] for series in series_data: name = series.get("name") @@ -1641,6 +1642,7 @@ def render_chart(self, series_data, yaxis_name, chart_name, now_str, tagname="ch opacity_value = series.get("opacity", "1.0") stroke_width_value = series.get("stroke_width", "1") stroke_curve_value = series.get("stroke_curve", "smooth") + stroke_dasharray_value = series.get("stroke_dasharray", "0") chart_type = series.get("chart_type", "line") color = series.get("color", "") unit_name = series.get("unit", yaxis_name) or "" @@ -1653,6 +1655,7 @@ def render_chart(self, series_data, yaxis_name, chart_name, now_str, tagname="ch opacity.append(opacity_value) stroke_width.append(stroke_width_value) stroke_curve.append("'{}'".format(stroke_curve_value)) + stroke_dasharray.append(stroke_dasharray_value) series_units.append(unit_name) units_array = ",".join("'{}'".format(unit) for unit in series_units) @@ -1664,6 +1667,7 @@ def render_chart(self, series_data, yaxis_name, chart_name, now_str, tagname="ch text += " stroke: {\n" text += " width: [{}],\n".format(",".join(stroke_width)) text += " curve: [{}],\n".format(",".join(stroke_curve)) + text += " dashArray: [{}],\n".format(",".join(stroke_dasharray)) text += " },\n" text += " xaxis: {\n" text += " type: 'datetime',\n" @@ -3021,7 +3025,7 @@ def get_chart(self, chart): elif chart == "Clipping": clipping_mode = getattr(self.base, "clipping_limit_mode", "Unknown") clipping_limit_effective = getattr(self.base, "clipping_limit_effective", 0) - inverter_ac_limit_kw = getattr(self.base, "inverter_limit", 0) + inverter_ac_limit_kw = getattr(self.base, "inverter_limit", 0) * 60.0 / 1000.0 # Fetch Target SOC (Best Plan) soc_kw_best_raw = history_attribute(self.get_history_wrapper(self.prefix + ".soc_kw_best", 3)) @@ -3069,14 +3073,9 @@ def get_chart(self, chart): stamp = minute_timestamp.strftime(TIME_FORMAT) clipping_ceiling_series[stamp] = round(kwh, 2) - # Raw PV forecast (ClearSky if used, else regular) + # Raw PV forecast (Base) raw_pv_series = {} - if getattr(self.base, "clipping_use_clearsky_peaks", False): - raw_pv_data = getattr(self.base, "pv_forecast_minuteCS", {}) - raw_pv_name = "ClearSky Forecast" - else: - raw_pv_data = getattr(self.base, "pv_forecast_minute", {}) - raw_pv_name = "Base Forecast" + raw_pv_data = getattr(self.base, "pv_forecast_minute", {}) if raw_pv_data: for minute, kw in raw_pv_data.items(): if minute % step_size == 0: @@ -3084,14 +3083,26 @@ def get_chart(self, chart): stamp = minute_timestamp.strftime(TIME_FORMAT) raw_pv_series[stamp] = round(kw, 2) + # ClearSky PV forecast + cs_pv_series = {} + cs_pv_data = getattr(self.base, "pv_forecast_minuteCS", {}) + if cs_pv_data: + for minute, kw in cs_pv_data.items(): + if minute % step_size == 0: + minute_timestamp = self.midnight_utc + timedelta(minutes=minute) + stamp = minute_timestamp.strftime(TIME_FORMAT) + cs_pv_series[stamp] = round(kw, 2) + series_data = [ - {"name": "Clipping Remaining", "data": clipping_remaining_series, "opacity": "0.5", "stroke_width": "2", "stroke_curve": "smooth", "chart_type": "area", "color": "#1877f2", "unit": "kWh"}, - {"name": "Actual SOC", "data": soc_kw_h0, "opacity": "1.0", "stroke_width": "2", "stroke_curve": "smooth", "color": "#3291a8", "unit": "kWh"}, - {"name": "Target SOC", "data": soc_kw_best, "opacity": "1.0", "stroke_width": "3", "stroke_curve": "stepline", "color": "#9b23eb", "unit": "kWh"}, - {"name": "Clipping Ceiling", "data": clipping_ceiling_series, "opacity": "1.0", "stroke_width": "2", "stroke_curve": "stepline", "color": "#eb3434", "unit": "kWh"}, - {"name": "PV Power Actual", "data": pv_power, "opacity": "1.0", "stroke_width": "3", "stroke_curve": "smooth", "color": "#f5c43d", "unit": "kW"}, - {"name": "Clipping Forecast ({})".format(raw_pv_name), "data": raw_pv_series, "opacity": "0.3", "stroke_width": "2", "stroke_curve": "smooth", "chart_type": "area", "color": "#a8a8a7", "unit": "kW"}, + {"name": "Clipping Remaining", "data": clipping_remaining_series, "opacity": "1.0", "stroke_width": "3", "stroke_curve": "smooth", "color": "#FF1493", "unit": "kWh"}, + {"name": "Actual SOC", "data": soc_kw_h0, "opacity": "1.0", "stroke_width": "3", "stroke_curve": "smooth", "color": "#00BFFF", "unit": "kWh"}, + {"name": "Target SOC", "data": soc_kw_best, "opacity": "0.7", "stroke_width": "2", "stroke_curve": "stepline", "stroke_dasharray": "5", "color": "#1E90FF", "unit": "kWh"}, + {"name": "Clipping Ceiling", "data": clipping_ceiling_series, "opacity": "0.7", "stroke_width": "2", "stroke_curve": "stepline", "stroke_dasharray": "5", "color": "#C71585", "unit": "kWh"}, + {"name": "PV Power Actual", "data": pv_power, "opacity": "1.0", "stroke_width": "3", "stroke_curve": "smooth", "color": "#FFA500", "unit": "kW"}, + {"name": "PV Forecast (Base)", "data": raw_pv_series, "opacity": "0.7", "stroke_width": "2", "stroke_curve": "smooth", "stroke_dasharray": "5", "color": "#FFD700", "unit": "kW"}, ] + if cs_pv_series: + series_data.append({"name": "PV Forecast (ClearSky)", "data": cs_pv_series, "opacity": "0.7", "stroke_width": "2", "stroke_curve": "smooth", "stroke_dasharray": "5", "color": "#FF8C00", "unit": "kW"}) yaxis_annotations = [] if inverter_ac_limit_kw > 0: From 206480acb0c79e739903f7705bc2268f19b6a647 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 17 Jun 2026 22:53:52 +0100 Subject: [PATCH 29/74] feat: integrate ha-solcast-clearsky and robust clipping baseline --- apps/predbat/components.py | 6 +- apps/predbat/config.py | 4 + apps/predbat/fetch.py | 4 +- apps/predbat/plan.py | 78 ++++++++++++++++ apps/predbat/solcast.py | 26 ++++++ apps/predbat/tests/test_clipping.py | 135 ++++++++++++++++++++++++++++ apps/predbat/unit_test.py | 2 + apps/predbat/web.py | 8 +- templates/example_chart.yml | 30 +++++-- 9 files changed, 280 insertions(+), 13 deletions(-) create mode 100644 apps/predbat/tests/test_clipping.py diff --git a/apps/predbat/components.py b/apps/predbat/components.py index 3236c1845..cbe40c367 100644 --- a/apps/predbat/components.py +++ b/apps/predbat/components.py @@ -108,11 +108,15 @@ "pv_forecast_tomorrow": {"required": False, "config": "pv_forecast_tomorrow"}, "pv_forecast_d3": {"required": False, "config": "pv_forecast_d3"}, "pv_forecast_d4": {"required": False, "config": "pv_forecast_d4"}, + "pv_clearsky_today": {"required": False, "config": "pv_clearsky_today"}, + "pv_clearsky_tomorrow": {"required": False, "config": "pv_clearsky_tomorrow"}, + "pv_clearsky_d3": {"required": False, "config": "pv_clearsky_d3"}, + "pv_clearsky_d4": {"required": False, "config": "pv_clearsky_d4"}, "pv_scaling": {"required": False, "config": "pv_scaling", "default": 1.0}, "open_meteo_forecast": {"required": False, "config": "open_meteo_forecast", "default": False}, "open_meteo_forecast_max_age": {"required": False, "config": "open_meteo_forecast_max_age", "default": 4}, }, - "required_or": ["solcast_api_key", "forecast_solar", "pv_forecast_today", "open_meteo_forecast"], + "required_or": ["solcast_api_key", "forecast_solar", "pv_forecast_today", "open_meteo_forecast", "pv_clearsky_today"], "phase": 2, # Solar component moved to phase 2 so that any Predbat cloud components (such as GEcloud) have been started and initialised pv_today, etc }, "gecloud": { diff --git a/apps/predbat/config.py b/apps/predbat/config.py index 811dfe736..53706167a 100644 --- a/apps/predbat/config.py +++ b/apps/predbat/config.py @@ -2325,4 +2325,8 @@ "clipping_buffer_max_kwh": {"type": "float"}, "clipping_buffer_start_time": {"type": "string"}, "clipping_buffer_end_time": {"type": "string"}, + "pv_clearsky_today": {"type": "sensor", "sensor_type": "float"}, + "pv_clearsky_tomorrow": {"type": "sensor", "sensor_type": "float"}, + "pv_clearsky_d3": {"type": "sensor", "sensor_type": "float"}, + "pv_clearsky_d4": {"type": "sensor", "sensor_type": "float"}, } diff --git a/apps/predbat/fetch.py b/apps/predbat/fetch.py index 74f9270d8..e3f3883f3 100644 --- a/apps/predbat/fetch.py +++ b/apps/predbat/fetch.py @@ -572,7 +572,7 @@ def minute_data_import_export(self, max_days_previous, now_utc, key, scale=1.0, if history and len(history) > 0 and len(history[0]) > 0: item = history[0][0] try: - last_updated_time = str2time(item["last_updated"]) + last_updated_time = str2time(item.get("last_updated", item.get("state_class", ""))) except (ValueError, TypeError): last_updated_time = now_utc age_days = max_days_previous @@ -639,7 +639,7 @@ def minute_data_load(self, now_utc, entity_name, max_days_previous, load_scaling if isinstance(history, list) and history and history[0]: item = history[0][0] try: - last_updated_time = str2time(item["last_updated"]) + last_updated_time = str2time(item.get("last_updated", item.get("state_class", ""))) except (ValueError, TypeError): last_updated_time = now_utc age = now_utc - last_updated_time diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index 14d1aa058..b362af238 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -844,6 +844,81 @@ def in_charge_window(self, charge_window, minute_abs): window_n += 1 return -1 + def inject_clipping_export_windows(self): + """ + Inject candidate export windows before predicted clipping to allow the optimizer to evaluate forced exports to create headroom. + """ + if not getattr(self, "clipping_peak_enable", False): + self.log("inject_clipping_export_windows: Aborting because clipping_peak_enable is False") + return + + forecast = getattr(self, "clipping_buffer_forecast_kwh", {}) + if not forecast: + self.log("inject_clipping_export_windows: Aborting because clipping_buffer_forecast_kwh is empty") + return + + self.log("inject_clipping_export_windows: Proceeding with clipping blocks: {}".format(forecast)) + + # Find contiguous blocks of clipping in the forecast + clipping_blocks = [] + current_block = None + for minute, kwh_loss in sorted(forecast.items()): + if kwh_loss > 0: + if current_block is None: + current_block = {"start": minute, "end": minute + 30} + elif minute <= current_block["end"] + 60: + current_block["end"] = max(current_block["end"], minute + 30) + else: + clipping_blocks.append(current_block) + current_block = {"start": minute, "end": minute + 30} + if current_block: + clipping_blocks.append(current_block) + + # Helper to check if a window intersects the target range + def intersects(w, start, end): + return not (w["end"] <= start or w["start"] >= end) + + for block in clipping_blocks: + peak_start, peak_end = block["start"], block["end"] + + # Start the continuous export window from around 08:00 to catch morning export rates, + # but no more than 12 hours before the peak, and never in the past. + morning_start = max(self.minutes_now, min(peak_start - 300, max(self.minutes_now, peak_start - 720))) + morning_start = int(morning_start / 30) * 30 # Align to nearest 30 mins + + if morning_start >= peak_start: + continue + + new_window = { + "start": morning_start, + "end": peak_end, + "average": self.export_rate.get(morning_start, 0.0) + } + + # We MUST remove any fragmented export windows within this period to ensure a single, continuous + # target SOC can be applied across the entire morning and peak, allowing the battery to slowly + # absorb the clipped excess without resetting its target. + + # Clean export_window_best + for w in [w for w in self.export_window_best if intersects(w, morning_start, peak_end)]: + self.log("Dropped fragmented export window {} to {} in favor of unified spillover window".format(self.time_abs_str(w["start"]), self.time_abs_str(w["end"]))) + + self.export_window_best = [w for w in self.export_window_best if not intersects(w, morning_start, peak_end)] + + # Clean high_export_rates + if getattr(self, "high_export_rates", None) is not None: + self.high_export_rates = [w for w in self.high_export_rates if not intersects(w, morning_start, peak_end)] + + # Inject our new unified window + if not any(w.get("start") <= new_window["start"] and w.get("end") >= new_window["end"] for w in self.export_window_best): + self.export_window_best.append(new_window) + if getattr(self, "high_export_rates", None) is not None: + self.high_export_rates.append(copy.deepcopy(new_window)) + + # Regenerate export limits + self.export_limits_best = [100.0] * len(self.export_window_best) + self.log("Injected continuous anti-clipping candidate export window {} to {} to allow spillover absorption".format(self.time_abs_str(morning_start), self.time_abs_str(peak_end))) + def calculate_plan(self, recompute=True, debug_mode=False, publish=True): """ Calculate the new plan (best) @@ -922,6 +997,9 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): # Pre-fill best export enable with Off self.export_limits_best = [100.0 for i in range(len(self.export_window_best))] + # Inject export windows to create headroom for clipping peaks + self.inject_clipping_export_windows() + self.end_record = self.forecast_minutes # Show best windows self.log("Best charge window {}".format(self.window_as_text(self.charge_window_best, calc_percent_limit(self.charge_limit_best, self.soc_max)))) diff --git a/apps/predbat/solcast.py b/apps/predbat/solcast.py index cf51aaf5a..1e3465061 100644 --- a/apps/predbat/solcast.py +++ b/apps/predbat/solcast.py @@ -74,6 +74,10 @@ def initialize( pv_forecast_tomorrow, pv_forecast_d3, pv_forecast_d4, + pv_clearsky_today, + pv_clearsky_tomorrow, + pv_clearsky_d3, + pv_clearsky_d4, pv_scaling, open_meteo_forecast, open_meteo_forecast_max_age, @@ -90,6 +94,10 @@ def initialize( self.pv_forecast_tomorrow = pv_forecast_tomorrow self.pv_forecast_d3 = pv_forecast_d3 self.pv_forecast_d4 = pv_forecast_d4 + self.pv_clearsky_today = pv_clearsky_today + self.pv_clearsky_tomorrow = pv_clearsky_tomorrow + self.pv_clearsky_d3 = pv_clearsky_d3 + self.pv_clearsky_d4 = pv_clearsky_d4 self.pv_scaling = pv_scaling self.open_meteo_forecast = open_meteo_forecast self.open_meteo_forecast_max_age = open_meteo_forecast_max_age @@ -1423,6 +1431,24 @@ async def fetch_pv_forecast(self): ts = item.get("period_start") if ts in sol_lookup: item["pv_clearsky"] = sol_lookup[ts] + elif clipping_clearsky_source == "ha_solcast_clearsky": + self.log("SolarAPI: Overlaying ClearSky data from Home Assistant integration") + ha_data = [] + for argname in ["pv_clearsky_today", "pv_clearsky_tomorrow", "pv_clearsky_d3", "pv_clearsky_d4"]: + entity_id = getattr(self, argname, None) + if entity_id: + data, _, _ = self.fetch_pv_datapoints(argname, entity_id) + if data: + ha_data += data + if ha_data: + # Apply factor and divide_by logic like the main PV fetcher + ha_lookup = {item["period_start"]: (item.get("pv_estimate", 0) * factor / divide_by) for item in ha_data} + for item in pv_forecast_data: + ts = item.get("period_start") + if ts in ha_lookup: + item["pv_clearsky"] = ha_lookup[ts] + elif clipping_clearsky_source == "pv90_scaled": + self.log("SolarAPI: Overlaying ClearSky data explicitly disabled, PV90 will be scaled and used as base for clipping mitigation") elif clipping_clearsky_source not in ["auto", "base", ""]: self.log("Warn: SolarAPI: clipping_clearsky_source '{}' not configured properly, using base data".format(clipping_clearsky_source)) diff --git a/apps/predbat/tests/test_clipping.py b/apps/predbat/tests/test_clipping.py new file mode 100644 index 000000000..2281d045f --- /dev/null +++ b/apps/predbat/tests/test_clipping.py @@ -0,0 +1,135 @@ +# ----------------------------------------------------------------------------- +# Predbat Home Battery System +# Copyright Trefor Southwell 2026 - All Rights Reserved +# This application maybe used for personal use only and not for commercial use +# ----------------------------------------------------------------------------- +# fmt off +# pylint: disable=consider-using-f-string +# pylint: disable=line-too-long +# pylint: disable=attribute-defined-outside-init +from tests.test_infra import reset_inverter +import copy + +def run_clipping_tests(my_predbat): + """ + Tests for inject_clipping_export_windows method + """ + failed = False + failed |= test_inject_aborts_if_disabled(my_predbat) + failed |= test_inject_aborts_if_empty_forecast(my_predbat) + failed |= test_inject_creates_contiguous_window(my_predbat) + failed |= test_inject_cleans_fragmented_windows(my_predbat) + return failed + +def setup(my_predbat): + reset_inverter(my_predbat) + my_predbat.clipping_peak_enable = True + my_predbat.clipping_buffer_forecast_kwh = {} + my_predbat.minutes_now = 0 + my_predbat.export_rate = {} + my_predbat.export_window_best = [] + my_predbat.high_export_rates = [] + # Adding log mock to avoid exceptions if not using MockBase + if not hasattr(my_predbat, "log"): + my_predbat.log = lambda x: print(x) + if not hasattr(my_predbat, "time_abs_str"): + my_predbat.time_abs_str = lambda x: str(x) + +def test_inject_aborts_if_disabled(my_predbat): + print("**** test_inject_aborts_if_disabled ****") + failed = False + setup(my_predbat) + my_predbat.clipping_peak_enable = False + my_predbat.clipping_buffer_forecast_kwh = {720: 1.0} + + my_predbat.inject_clipping_export_windows() + + if len(my_predbat.export_window_best) > 0: + print("ERROR: Window was injected when clipping was disabled!") + failed = True + + if not failed: print("PASS") + return failed + +def test_inject_aborts_if_empty_forecast(my_predbat): + print("**** test_inject_aborts_if_empty_forecast ****") + failed = False + setup(my_predbat) + my_predbat.clipping_peak_enable = True + my_predbat.clipping_buffer_forecast_kwh = {} + + my_predbat.inject_clipping_export_windows() + + if len(my_predbat.export_window_best) > 0: + print("ERROR: Window was injected when forecast was empty!") + failed = True + + if not failed: print("PASS") + return failed + +def test_inject_creates_contiguous_window(my_predbat): + print("**** test_inject_creates_contiguous_window ****") + failed = False + setup(my_predbat) + my_predbat.minutes_now = 240 # 04:00 + # Peak from 13:00 to 14:00 (780 to 840) + my_predbat.clipping_buffer_forecast_kwh = { + 780: 1.0, + 810: 2.0 + } + + my_predbat.inject_clipping_export_windows() + + if len(my_predbat.export_window_best) != 1: + print("ERROR: Expected exactly 1 window injected, got {}".format(len(my_predbat.export_window_best))) + return True + + w = my_predbat.export_window_best[0] + + if w["start"] != 240: + print("ERROR: Expected window start at 240, got {}".format(w["start"])) + failed = True + + if w["end"] != 840: + print("ERROR: Expected window end at 840, got {}".format(w["end"])) + failed = True + + if not failed: print("PASS") + return failed + +def test_inject_cleans_fragmented_windows(my_predbat): + print("**** test_inject_cleans_fragmented_windows ****") + failed = False + setup(my_predbat) + my_predbat.minutes_now = 0 + # Peak from 780 to 810 + my_predbat.clipping_buffer_forecast_kwh = { + 780: 1.0, + } + + # Inject intersecting fragmented windows + my_predbat.export_window_best = [ + {"start": 30, "end": 50, "average": 0}, # Before morning_start (60) - should KEEP + {"start": 100, "end": 120, "average": 0}, # Inside the new window - should DROP + {"start": 700, "end": 800, "average": 0}, # Intersecting the new window - should DROP + {"start": 900, "end": 960, "average": 0} # After peak_end - should KEEP + ] + + my_predbat.inject_clipping_export_windows() + + # We expect 3 windows: 2 kept + 1 injected + if len(my_predbat.export_window_best) != 3: + print("ERROR: Expected 3 windows (2 kept + 1 new), got {}".format(len(my_predbat.export_window_best))) + return True + + starts = [w["start"] for w in my_predbat.export_window_best] + if 100 in starts or 700 in starts: + print("ERROR: Fragmented windows were not cleaned!") + failed = True + + if 60 not in starts: # Injected window start + print("ERROR: Injected window not found!") + failed = True + + if not failed: print("PASS") + return failed diff --git a/apps/predbat/unit_test.py b/apps/predbat/unit_test.py index b1b39c6c1..2693549dd 100644 --- a/apps/predbat/unit_test.py +++ b/apps/predbat/unit_test.py @@ -118,6 +118,7 @@ from tests.test_band_rate_text import test_band_rate_text from tests.test_kraken import run_kraken_tests from tests.test_kraken_auth_mixin import run_kraken_auth_mixin_tests +from tests.test_clipping import run_clipping_tests from tests.test_clip_export_slots import run_clip_export_slots_tests from tests.test_clip_charge_slots import run_clip_charge_slots_tests from tests.test_discard_unused_charge_slots import run_discard_unused_charge_slots_tests @@ -301,6 +302,7 @@ def main(): # Kraken Energy (EDF/E.ON) tests ("kraken", run_kraken_tests, "Kraken API tests (init, GraphQL, tariff discovery, rate fetching, run lifecycle)", False), ("kraken_auth", run_kraken_auth_mixin_tests, "Kraken auth mixin tests (API key, email, refresh, 401 handling)", False), + ("clipping", run_clipping_tests, "Clipping logic tests", False), ("clip_export_slots", run_clip_export_slots_tests, "Clip export slots tests", False), ("clip_charge_slots", run_clip_charge_slots_tests, "Clip charge slots tests", False), ("discard_unused_charge_slots", run_discard_unused_charge_slots_tests, "Discard unused charge slots tests", False), diff --git a/apps/predbat/web.py b/apps/predbat/web.py index 32efb4d52..d299f942d 100644 --- a/apps/predbat/web.py +++ b/apps/predbat/web.py @@ -3023,9 +3023,7 @@ def get_chart(self, chart): ] text += self.render_chart(series_data, "kWh", "PV Forecast vs Actual", now_str) elif chart == "Clipping": - clipping_mode = getattr(self.base, "clipping_limit_mode", "Unknown") - clipping_limit_effective = getattr(self.base, "clipping_limit_effective", 0) - inverter_ac_limit_kw = getattr(self.base, "inverter_limit", 0) * 60.0 / 1000.0 + inverter_ac_limit_kw = getattr(self.base, "inverter_limit", 0) / 1000.0 # Fetch Target SOC (Best Plan) soc_kw_best_raw = history_attribute(self.get_history_wrapper(self.prefix + ".soc_kw_best", 3)) @@ -3081,7 +3079,7 @@ def get_chart(self, chart): if minute % step_size == 0: minute_timestamp = self.midnight_utc + timedelta(minutes=minute) stamp = minute_timestamp.strftime(TIME_FORMAT) - raw_pv_series[stamp] = round(kw, 2) + raw_pv_series[stamp] = round(kw * 60.0, 2) # ClearSky PV forecast cs_pv_series = {} @@ -3091,7 +3089,7 @@ def get_chart(self, chart): if minute % step_size == 0: minute_timestamp = self.midnight_utc + timedelta(minutes=minute) stamp = minute_timestamp.strftime(TIME_FORMAT) - cs_pv_series[stamp] = round(kw, 2) + cs_pv_series[stamp] = round(kw * 60.0, 2) series_data = [ {"name": "Clipping Remaining", "data": clipping_remaining_series, "opacity": "1.0", "stroke_width": "3", "stroke_curve": "smooth", "color": "#FF1493", "unit": "kWh"}, diff --git a/templates/example_chart.yml b/templates/example_chart.yml index bc15927b1..547fd2aab 100644 --- a/templates/example_chart.yml +++ b/templates/example_chart.yml @@ -20,10 +20,14 @@ now: yaxis: - min: 0 max: 9.54 +apex_config: + stroke: + dashArray: [0, 0, 0, 0, 4, 4, 4, 0, 0, 0] series: - entity: predbat.soc_kw_h0 - stroke_width: 1 + stroke_width: 2 curve: stepline + color: '#000000' name: actual extend_to: now show: @@ -42,6 +46,7 @@ series: - entity: predbat.soc_kw_best stroke_width: 3 curve: smooth + color: '#1f77b4' name: best data_generator: > let res = []; for (const [key, value] of @@ -62,11 +67,12 @@ series: show: in_header: raw - entity: predbat.best_charge_limit_kw - stroke_width: 4 + stroke_width: 3 curve: stepline + color: '#2ca02c' name: charge_limit_best - type: area - opacity: 0.2 + type: line + opacity: 0.8 data_generator: > let res = []; for (const [key, value] of Object.entries(entity.attributes.results)) { res.push([new @@ -75,8 +81,9 @@ series: show: in_header: raw - entity: predbat.charge_limit_kw - stroke_width: 1 + stroke_width: 2 curve: stepline + color: '#98df8a' name: charge_limit_base data_generator: > let res = []; for (const [key, value] of @@ -88,6 +95,7 @@ series: - entity: predbat.best_export_limit_kw stroke_width: 2 curve: stepline + color: '#d62728' name: export_best data_generator: > let res = []; for (const [key, value] of @@ -121,6 +129,18 @@ series: b[0] }) show: in_header: raw + - entity: predbat.clipping_remaining + name: clipping + type: column + color: '#ff9800' + opacity: 0.5 + data_generator: > + let res = []; for (const [key, value] of + Object.entries(entity.attributes.results)) { res.push([new + Date(key).getTime(), value]); } return res.sort((a, b) => { return a[0] - + b[0] }) + show: + in_header: raw ############################################### # This is a cost chart ############################################### From a6b0e4f9262aa80f79075dc41046db1d88e477a0 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 17 Jun 2026 23:12:54 +0100 Subject: [PATCH 30/74] fix: Inverter AC Capacity conversion in web chart --- apps/predbat/web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/predbat/web.py b/apps/predbat/web.py index d299f942d..1824d8691 100644 --- a/apps/predbat/web.py +++ b/apps/predbat/web.py @@ -3023,7 +3023,7 @@ def get_chart(self, chart): ] text += self.render_chart(series_data, "kWh", "PV Forecast vs Actual", now_str) elif chart == "Clipping": - inverter_ac_limit_kw = getattr(self.base, "inverter_limit", 0) / 1000.0 + inverter_ac_limit_kw = getattr(self.base, "inverter_limit", 0) * 60.0 # Fetch Target SOC (Best Plan) soc_kw_best_raw = history_attribute(self.get_history_wrapper(self.prefix + ".soc_kw_best", 3)) From 9b2a80c369f8f4af7fdf19cc9ed6cc858476b153 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 17 Jun 2026 23:15:41 +0100 Subject: [PATCH 31/74] fix: render future plan on Clipping Analysis chart --- apps/predbat/web.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/apps/predbat/web.py b/apps/predbat/web.py index 1824d8691..bade0663f 100644 --- a/apps/predbat/web.py +++ b/apps/predbat/web.py @@ -3028,6 +3028,7 @@ def get_chart(self, chart): # Fetch Target SOC (Best Plan) soc_kw_best_raw = history_attribute(self.get_history_wrapper(self.prefix + ".soc_kw_best", 3)) soc_kw_best = prune_today(soc_kw_best_raw, self.now_utc, self.midnight_utc, prune_past_days=2) + soc_kw_best.update(self.get_entity_results(self.prefix + ".soc_kw_best")) # Fetch Actual SOC history (48 hours into the past) soc_kw_h0 = {} @@ -3054,19 +3055,20 @@ def get_chart(self, chart): step_size = getattr(self.base, "plan_interval_minutes", 30) clipping_remaining_raw = history_attribute(self.get_history_wrapper(self.prefix + ".clipping_remaining", 3)) clipping_remaining_series = prune_today(clipping_remaining_raw, self.now_utc, self.midnight_utc, prune_past_days=2) - # If history is missing/empty, fallback to the base predictor - if not clipping_remaining_series and getattr(self.base, "predict_clipping_remaining_best", None): + + if getattr(self.base, "predict_clipping_remaining_best", None): for minute, kwh in self.base.predict_clipping_remaining_best.items(): - if minute % step_size == 0: + if minute % step_size == 0 and minute >= self.minutes_now: minute_timestamp = self.midnight_utc + timedelta(minutes=minute) stamp = minute_timestamp.strftime(TIME_FORMAT) clipping_remaining_series[stamp] = round(kwh, 2) clipping_ceiling_raw = history_attribute(self.get_history_wrapper(self.prefix + ".clipping_ceiling", 3)) clipping_ceiling_series = prune_today(clipping_ceiling_raw, self.now_utc, self.midnight_utc, prune_past_days=2) - if not clipping_ceiling_series and getattr(self.base, "predict_clipping_ceiling_best", None): + + if getattr(self.base, "predict_clipping_ceiling_best", None): for minute, kwh in self.base.predict_clipping_ceiling_best.items(): - if minute % step_size == 0: + if minute % step_size == 0 and minute >= self.minutes_now: minute_timestamp = self.midnight_utc + timedelta(minutes=minute) stamp = minute_timestamp.strftime(TIME_FORMAT) clipping_ceiling_series[stamp] = round(kwh, 2) From 36cead1e938b9b6839d0680d1dfce8854ebc8cd4 Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 18 Jun 2026 06:52:17 +0100 Subject: [PATCH 32/74] fix: typo export_rate -> rate_export in plan.py --- apps/predbat/plan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index b362af238..81e3cc9dd 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -892,7 +892,7 @@ def intersects(w, start, end): new_window = { "start": morning_start, "end": peak_end, - "average": self.export_rate.get(morning_start, 0.0) + "average": self.rate_export.get(morning_start, 0.0) } # We MUST remove any fragmented export windows within this period to ensure a single, continuous From 823c31971590abedad2c54139bcf0aa83baea59e Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 18 Jun 2026 09:27:53 +0100 Subject: [PATCH 33/74] fix: align with upstream architectural principles (cache auto-tuner & defensive None casting) --- apps/predbat/plan.py | 27 +++++++++++++++++---------- apps/predbat/solcast.py | 2 +- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index 81e3cc9dd..eb1e64b62 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -1049,19 +1049,24 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): ) # Auto-Tuner: Always runs to gather data and recommend a factor - auto_amp = getattr(self, "clipping_peak_amplification", 1.0) import os import json auto_tune_file = os.path.join(self.config_root, "clipping_auto_tune.json") - last_tune_day = None - if os.path.exists(auto_tune_file): - try: - with open(auto_tune_file, "r") as f: - data = json.load(f) - auto_amp = data.get("auto_amp", 1.0) - last_tune_day = data.get("last_tune_day", None) - except Exception: - pass + + if not hasattr(self, "clipping_auto_amp"): + self.clipping_auto_amp = getattr(self, "clipping_peak_amplification", 1.0) + self.clipping_last_tune_day = None + if os.path.exists(auto_tune_file): + try: + with open(auto_tune_file, "r") as f: + data = json.load(f) + self.clipping_auto_amp = data.get("auto_amp", self.clipping_auto_amp) + self.clipping_last_tune_day = data.get("last_tune_day", None) + except Exception: + pass + + auto_amp = self.clipping_auto_amp + last_tune_day = self.clipping_last_tune_day # Check if we should tune today (only once per day) current_day = self.now_utc.strftime("%Y-%m-%d") @@ -1097,6 +1102,8 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): try: with open(auto_tune_file, "w") as f: json.dump({"auto_amp": auto_amp, "last_tune_day": current_day}, f) + self.clipping_auto_amp = auto_amp + self.clipping_last_tune_day = current_day except Exception: pass diff --git a/apps/predbat/solcast.py b/apps/predbat/solcast.py index 1e3465061..5ef056d03 100644 --- a/apps/predbat/solcast.py +++ b/apps/predbat/solcast.py @@ -1442,7 +1442,7 @@ async def fetch_pv_forecast(self): ha_data += data if ha_data: # Apply factor and divide_by logic like the main PV fetcher - ha_lookup = {item["period_start"]: (item.get("pv_estimate", 0) * factor / divide_by) for item in ha_data} + ha_lookup = {item["period_start"]: (float(item.get("pv_estimate") or 0.0) * factor / divide_by) for item in ha_data} for item in pv_forecast_data: ts = item.get("period_start") if ts in ha_lookup: From 3e6058c62dd183171a9df2cc960749eedc73e99b Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 18 Jun 2026 16:13:45 +0100 Subject: [PATCH 34/74] Fix SolarAPI.initialize mock in unit tests to include clearsky parameters --- apps/predbat/tests/test_solcast.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/predbat/tests/test_solcast.py b/apps/predbat/tests/test_solcast.py index 4a113b8c1..a05902e0f 100644 --- a/apps/predbat/tests/test_solcast.py +++ b/apps/predbat/tests/test_solcast.py @@ -172,6 +172,10 @@ def __init__(self): pv_forecast_tomorrow=None, pv_forecast_d3=None, pv_forecast_d4=None, + pv_clearsky_today=None, + pv_clearsky_tomorrow=None, + pv_clearsky_d3=None, + pv_clearsky_d4=None, pv_scaling=1.0, open_meteo_forecast=None, open_meteo_forecast_max_age=1.0, From c4c1bf9657c5197fe984fa09a5f439dbbd7a310f Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 18 Jun 2026 16:22:11 +0100 Subject: [PATCH 35/74] UI: Move clipping config adjacent to cloud config and expose clear-sky selector --- apps/predbat/config.py | 187 +++++++++++++++++++++-------------------- 1 file changed, 98 insertions(+), 89 deletions(-) diff --git a/apps/predbat/config.py b/apps/predbat/config.py index 53706167a..22f749d50 100644 --- a/apps/predbat/config.py +++ b/apps/predbat/config.py @@ -608,6 +608,104 @@ "type": "switch", "default": True, }, + { + "name": "clipping_peak_enable", + "friendly_name": "Clipping Peak Enable", + "type": "switch", + "default": False, + "icon": "mdi:chart-bell-curve-cumulative", + }, + { + "name": "clipping_clearsky_source", + "friendly_name": "Clipping Clear-Sky Source", + "type": "select", + "options": ["auto", "ha_solcast_clearsky", "solcast_api", "openmeteo"], + "icon": "mdi:cloud-search", + "default": "auto", + "enable": "clipping_peak_enable", + }, + { + "name": "clipping_use_clearsky_peaks", + "friendly_name": "Clipping Use ClearSky Peaks", + "type": "switch", + "default": True, + "icon": "mdi:weather-sunny-alert", + "enable": "clipping_peak_enable", + }, + { + "name": "clipping_peak_auto_tune", + "friendly_name": "Clipping Peak Auto-Tune", + "type": "switch", + "default": True, + "icon": "mdi:auto-fix", + "enable": "clipping_peak_enable", + }, + { + "name": "clipping_cost_weight", + "friendly_name": "Clipping Cost Weight", + "type": "input_number", + "min": 0, + "max": 10.0, + "step": 0.1, + "unit": "x", + "icon": "mdi:multiplication", + "enable": "clipping_peak_enable", + "default": 1.0, + }, + { + "name": "clipping_peak_amplification", + "friendly_name": "Clipping Peak Amplification", + "type": "input_number", + "min": 0.5, + "max": 3.0, + "step": 0.1, + "unit": "x", + "icon": "mdi:arrow-expand-vertical", + "enable": "clipping_peak_enable", + "default": 1.0, + }, + { + "name": "clipping_limit_override", + "friendly_name": "Clipping Limit Override", + "type": "input_number", + "min": 0, + "max": 50000, + "step": 100, + "unit": "W", + "icon": "mdi:flash-alert", + "enable": "clipping_peak_enable", + "default": 0, + }, + { + "name": "clipping_buffer_max_kwh", + "friendly_name": "Clipping Buffer Max kWh (Manual Override)", + "type": "input_number", + "min": 0, + "max": 50.0, + "step": 0.1, + "unit": "kWh", + "icon": "mdi:battery-50", + "default": 0, + "enable": "clipping_peak_enable", + }, + { + "name": "clipping_buffer_start_time", + "friendly_name": "Clipping Buffer Start Time", + "type": "select", + "options": ["None"] + OPTIONS_TIME, + "icon": "mdi:clock-start", + "default": "None", + "enable": "clipping_peak_enable", + }, + { + "name": "clipping_buffer_end_time", + "friendly_name": "Clipping Buffer End Time", + "type": "select", + "options": ["None"] + OPTIONS_TIME, + "icon": "mdi:clock-end", + "default": "None", + "enable": "clipping_peak_enable", + }, { "name": "car_energy_reported_load", "friendly_name": "Car energy is reported in load data (inside CT clamp)", @@ -1428,95 +1526,6 @@ "default": 20, "enable": "predheat_enable", }, - { - "name": "clipping_peak_enable", - "friendly_name": "Clipping Peak Enable", - "type": "switch", - "default": False, - "icon": "mdi:chart-bell-curve-cumulative", - }, - { - "name": "clipping_use_clearsky_peaks", - "friendly_name": "Clipping Use ClearSky Peaks", - "type": "switch", - "default": True, - "icon": "mdi:weather-sunny-alert", - "enable": "clipping_peak_enable", - }, - { - "name": "clipping_peak_auto_tune", - "friendly_name": "Clipping Peak Auto-Tune", - "type": "switch", - "default": True, - "icon": "mdi:auto-fix", - "enable": "clipping_peak_enable", - }, - { - "name": "clipping_cost_weight", - "friendly_name": "Clipping Cost Weight", - "type": "input_number", - "min": 0, - "max": 10.0, - "step": 0.1, - "unit": "x", - "icon": "mdi:multiplication", - "enable": "clipping_peak_enable", - "default": 1.0, - }, - { - "name": "clipping_peak_amplification", - "friendly_name": "Clipping Peak Amplification", - "type": "input_number", - "min": 0.5, - "max": 3.0, - "step": 0.1, - "unit": "x", - "icon": "mdi:arrow-expand-vertical", - "enable": "clipping_peak_enable", - "default": 1.0, - }, - { - "name": "clipping_limit_override", - "friendly_name": "Clipping Limit Override", - "type": "input_number", - "min": 0, - "max": 50000, - "step": 100, - "unit": "W", - "icon": "mdi:flash-alert", - "enable": "clipping_peak_enable", - "default": 0, - }, - { - "name": "clipping_buffer_max_kwh", - "friendly_name": "Clipping Buffer Max kWh (Manual Override)", - "type": "input_number", - "min": 0, - "max": 50.0, - "step": 0.1, - "unit": "kWh", - "icon": "mdi:battery-50", - "default": 0, - "enable": "clipping_peak_enable", - }, - { - "name": "clipping_buffer_start_time", - "friendly_name": "Clipping Buffer Start Time", - "type": "select", - "options": ["None"] + OPTIONS_TIME, - "icon": "mdi:clock-start", - "default": "None", - "enable": "clipping_peak_enable", - }, - { - "name": "clipping_buffer_end_time", - "friendly_name": "Clipping Buffer End Time", - "type": "select", - "options": ["None"] + OPTIONS_TIME, - "icon": "mdi:clock-end", - "default": "None", - "enable": "clipping_peak_enable", - }, { "name": "inverter_limit_override", "friendly_name": "Inverter Limit Override", From d45dcc9c9b95e8a8088ae9b2300d307936529ce9 Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 18 Jun 2026 16:35:37 +0100 Subject: [PATCH 36/74] Rename clipping configuration variables for better clarity --- apps/predbat/config.py | 34 +++++++++++++------------- apps/predbat/fetch.py | 6 ++--- apps/predbat/output.py | 2 +- apps/predbat/plan.py | 16 ++++++------ apps/predbat/tests/compare_clipping.py | 8 +++--- apps/predbat/tests/test_clipping.py | 6 ++--- apps/predbat/tests/test_infra.py | 22 ++++++++--------- apps/predbat/tests/test_model.py | 8 +++--- 8 files changed, 51 insertions(+), 51 deletions(-) diff --git a/apps/predbat/config.py b/apps/predbat/config.py index 22f749d50..2110b4813 100644 --- a/apps/predbat/config.py +++ b/apps/predbat/config.py @@ -609,8 +609,8 @@ "default": True, }, { - "name": "clipping_peak_enable", - "friendly_name": "Clipping Peak Enable", + "name": "clipping_buffer_enable", + "friendly_name": "Clipping Buffer Enable", "type": "switch", "default": False, "icon": "mdi:chart-bell-curve-cumulative", @@ -622,7 +622,7 @@ "options": ["auto", "ha_solcast_clearsky", "solcast_api", "openmeteo"], "icon": "mdi:cloud-search", "default": "auto", - "enable": "clipping_peak_enable", + "enable": "clipping_buffer_enable", }, { "name": "clipping_use_clearsky_peaks", @@ -630,15 +630,15 @@ "type": "switch", "default": True, "icon": "mdi:weather-sunny-alert", - "enable": "clipping_peak_enable", + "enable": "clipping_buffer_enable", }, { - "name": "clipping_peak_auto_tune", - "friendly_name": "Clipping Peak Auto-Tune", + "name": "clipping_auto_tune", + "friendly_name": "Clipping Auto-Tune", "type": "switch", "default": True, "icon": "mdi:auto-fix", - "enable": "clipping_peak_enable", + "enable": "clipping_buffer_enable", }, { "name": "clipping_cost_weight", @@ -649,19 +649,19 @@ "step": 0.1, "unit": "x", "icon": "mdi:multiplication", - "enable": "clipping_peak_enable", + "enable": "clipping_buffer_enable", "default": 1.0, }, { - "name": "clipping_peak_amplification", - "friendly_name": "Clipping Peak Amplification", + "name": "clipping_amplification", + "friendly_name": "Clipping Amplification", "type": "input_number", "min": 0.5, "max": 3.0, "step": 0.1, "unit": "x", "icon": "mdi:arrow-expand-vertical", - "enable": "clipping_peak_enable", + "enable": "clipping_buffer_enable", "default": 1.0, }, { @@ -673,7 +673,7 @@ "step": 100, "unit": "W", "icon": "mdi:flash-alert", - "enable": "clipping_peak_enable", + "enable": "clipping_buffer_enable", "default": 0, }, { @@ -686,7 +686,7 @@ "unit": "kWh", "icon": "mdi:battery-50", "default": 0, - "enable": "clipping_peak_enable", + "enable": "clipping_buffer_enable", }, { "name": "clipping_buffer_start_time", @@ -695,7 +695,7 @@ "options": ["None"] + OPTIONS_TIME, "icon": "mdi:clock-start", "default": "None", - "enable": "clipping_peak_enable", + "enable": "clipping_buffer_enable", }, { "name": "clipping_buffer_end_time", @@ -704,7 +704,7 @@ "options": ["None"] + OPTIONS_TIME, "icon": "mdi:clock-end", "default": "None", - "enable": "clipping_peak_enable", + "enable": "clipping_buffer_enable", }, { "name": "car_energy_reported_load", @@ -2326,10 +2326,10 @@ "gateway_mqtt_host": {"type": "string", "empty": False}, "gateway_mqtt_port": {"type": "integer", "zero": False}, "gateway_mqtt_token": {"type": "string", "empty": False}, - "clipping_peak_enable": {"type": "boolean"}, + "clipping_buffer_enable": {"type": "boolean"}, "clipping_use_clearsky_peaks": {"type": "boolean"}, "clipping_cost_weight": {"type": "float"}, - "clipping_peak_amplification": {"type": "float"}, + "clipping_amplification": {"type": "float"}, "clipping_limit_override": {"type": "integer"}, "clipping_buffer_max_kwh": {"type": "float"}, "clipping_buffer_start_time": {"type": "string"}, diff --git a/apps/predbat/fetch.py b/apps/predbat/fetch.py index e3f3883f3..79064d137 100644 --- a/apps/predbat/fetch.py +++ b/apps/predbat/fetch.py @@ -2364,11 +2364,11 @@ def fetch_config_options(self): self.carbon_metric = self.get_arg("carbon_metric") # Clipping peak cost penalty model - self.clipping_peak_enable = self.get_arg("clipping_peak_enable") + self.clipping_buffer_enable = self.get_arg("clipping_buffer_enable") self.clipping_use_clearsky_peaks = self.get_arg("clipping_use_clearsky_peaks") - self.clipping_peak_auto_tune = self.get_arg("clipping_peak_auto_tune") + self.clipping_auto_tune = self.get_arg("clipping_auto_tune") self.clipping_cost_weight = self.get_arg("clipping_cost_weight") - self.clipping_peak_amplification = self.get_arg("clipping_peak_amplification") + self.clipping_amplification = self.get_arg("clipping_amplification") self.clipping_limit_override = self.get_arg("clipping_limit_override") / MINUTE_WATT if self.get_arg("clipping_limit_override") else 0 # iBoost solar diverter model diff --git a/apps/predbat/output.py b/apps/predbat/output.py index ae1c8214b..ef1dd2032 100644 --- a/apps/predbat/output.py +++ b/apps/predbat/output.py @@ -940,7 +940,7 @@ def short_textual_plan(self, soc_min, soc_min_minute, pv_forecast_minute_step, p ) # Clipping summary - if getattr(self, "clipping_peak_enable", False): + if getattr(self, "clipping_buffer_enable", False): clipping_status = getattr(self, "clipping_status", "No clipping forecast.") sentence += "- Clipping status: {}\n".format(clipping_status) diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index eb1e64b62..8a79fafec 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -848,8 +848,8 @@ def inject_clipping_export_windows(self): """ Inject candidate export windows before predicted clipping to allow the optimizer to evaluate forced exports to create headroom. """ - if not getattr(self, "clipping_peak_enable", False): - self.log("inject_clipping_export_windows: Aborting because clipping_peak_enable is False") + if not getattr(self, "clipping_buffer_enable", False): + self.log("inject_clipping_export_windows: Aborting because clipping_buffer_enable is False") return forecast = getattr(self, "clipping_buffer_forecast_kwh", {}) @@ -1043,7 +1043,7 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): pv_forecast_peak_step = None clipping_limit_effective = 0 clipping_limit_mode = "Unknown" - if self.clipping_peak_enable: + if self.clipping_buffer_enable: pv_forecast_peak_step = self.step_data_history( self.pv_forecast_minute, self.minutes_now, forward=True, cloud_factor=None ) @@ -1054,7 +1054,7 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): auto_tune_file = os.path.join(self.config_root, "clipping_auto_tune.json") if not hasattr(self, "clipping_auto_amp"): - self.clipping_auto_amp = getattr(self, "clipping_peak_amplification", 1.0) + self.clipping_auto_amp = getattr(self, "clipping_amplification", 1.0) self.clipping_last_tune_day = None if os.path.exists(auto_tune_file): try: @@ -1110,10 +1110,10 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): self.dashboard_item(self.prefix + ".clipping_recommended_amplification", state=dp2(auto_amp), attributes={"friendly_name": "Clipping Recommended Amplification", "icon": "mdi:auto-fix"}) # Apply Manual vs Auto Factor - if getattr(self, "clipping_peak_auto_tune", False): + if getattr(self, "clipping_auto_tune", False): effective_amplification = auto_amp else: - effective_amplification = self.clipping_peak_amplification + effective_amplification = self.clipping_amplification # Apply ClearSky or Amplification factor if getattr(self, "clipping_use_clearsky_peaks", False): @@ -1167,7 +1167,7 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): self.clipping_remaining_today = 0.0 self.clipping_tomorrow = 0.0 - if self.clipping_peak_enable and pv_forecast_peak_step and clipping_limit_effective > 0: + if self.clipping_buffer_enable and pv_forecast_peak_step and clipping_limit_effective > 0: step_size = getattr(self, "plan_interval_minutes", 30) # Accumulate potential clipping loss per interval for minute in range(0, self.forecast_minutes, step_size): @@ -1204,7 +1204,7 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): self, pv_forecast_minute_step, pv_forecast_minute10_step, load_minutes_step, load_minutes_step10, pv_forecast_peak_step=pv_forecast_peak_step, clipping_limit=clipping_limit_effective, - clipping_cost_weight=self.clipping_cost_weight if self.clipping_peak_enable else 0, + clipping_cost_weight=self.clipping_cost_weight if self.clipping_buffer_enable else 0, clipping_buffer_kwh=self.clipping_buffer_kwh, clipping_buffer_start=self.clipping_buffer_start, clipping_buffer_end=self.clipping_buffer_end, diff --git a/apps/predbat/tests/compare_clipping.py b/apps/predbat/tests/compare_clipping.py index d7e212cde..e1bd9d578 100644 --- a/apps/predbat/tests/compare_clipping.py +++ b/apps/predbat/tests/compare_clipping.py @@ -1,7 +1,7 @@ # ----------------------------------------------------------------------------- # Clipping Approach Comparison: Cloud-Model Penalty vs Baseline # -# Runs identical scenarios with and without clipping_peak_enable and compares +# Runs identical scenarios with and without clipping_buffer_enable and compares # the optimizer metrics, final SoC, and clipping amounts. # # Usage: python apps\predbat\tests\compare_clipping.py @@ -162,7 +162,7 @@ def run_comparison(my_predbat): my_predbat, assert_final_metric=0, assert_final_soc=0, - clipping_peak_enable=False, + clipping_buffer_enable=False, **common_args, ) time_a = time.perf_counter() - t_start @@ -177,9 +177,9 @@ def run_comparison(my_predbat): my_predbat, assert_final_metric=0, assert_final_soc=0, - clipping_peak_enable=True, + clipping_buffer_enable=True, clipping_cost_weight=1.0, - clipping_peak_amplification=1.0, + clipping_amplification=1.0, **common_args, ) time_b = time.perf_counter() - t_start diff --git a/apps/predbat/tests/test_clipping.py b/apps/predbat/tests/test_clipping.py index 2281d045f..00b6b7743 100644 --- a/apps/predbat/tests/test_clipping.py +++ b/apps/predbat/tests/test_clipping.py @@ -23,7 +23,7 @@ def run_clipping_tests(my_predbat): def setup(my_predbat): reset_inverter(my_predbat) - my_predbat.clipping_peak_enable = True + my_predbat.clipping_buffer_enable = True my_predbat.clipping_buffer_forecast_kwh = {} my_predbat.minutes_now = 0 my_predbat.export_rate = {} @@ -39,7 +39,7 @@ def test_inject_aborts_if_disabled(my_predbat): print("**** test_inject_aborts_if_disabled ****") failed = False setup(my_predbat) - my_predbat.clipping_peak_enable = False + my_predbat.clipping_buffer_enable = False my_predbat.clipping_buffer_forecast_kwh = {720: 1.0} my_predbat.inject_clipping_export_windows() @@ -55,7 +55,7 @@ def test_inject_aborts_if_empty_forecast(my_predbat): print("**** test_inject_aborts_if_empty_forecast ****") failed = False setup(my_predbat) - my_predbat.clipping_peak_enable = True + my_predbat.clipping_buffer_enable = True my_predbat.clipping_buffer_forecast_kwh = {} my_predbat.inject_clipping_export_windows() diff --git a/apps/predbat/tests/test_infra.py b/apps/predbat/tests/test_infra.py index fe6560543..971c684f8 100644 --- a/apps/predbat/tests/test_infra.py +++ b/apps/predbat/tests/test_infra.py @@ -354,9 +354,9 @@ def get_default_config(self): "load_filter_modal": True, "carbon_enable": False, "carbon_metric": 0, - "clipping_peak_enable": False, + "clipping_buffer_enable": False, "clipping_cost_weight": 1.0, - "clipping_peak_amplification": 1.0, + "clipping_amplification": 1.0, "clipping_limit_override": 0, "iboost_enable": False, "iboost_gas": 4.0, @@ -521,9 +521,9 @@ def reset_inverter(my_predbat): my_predbat.set_export_window = True my_predbat.set_charge_freeze = True my_predbat.set_export_freeze = True - my_predbat.clipping_peak_enable = False + my_predbat.clipping_buffer_enable = False my_predbat.clipping_cost_weight = 0 - my_predbat.clipping_peak_amplification = 1.0 + my_predbat.clipping_amplification = 1.0 my_predbat.clipping_limit_override = 0 @@ -617,9 +617,9 @@ def simple_scenario( calculate_export_on_pv=True, assert_clipped=0, pv_ac_limit=0, - clipping_peak_enable=False, + clipping_buffer_enable=False, clipping_cost_weight=0, - clipping_peak_amplification=1.0, + clipping_amplification=1.0, clipping_limit_override=0, ): """ @@ -719,9 +719,9 @@ def simple_scenario( my_predbat.charge_scaling10 = charge_scaling10 # Clipping peak penalty settings - my_predbat.clipping_peak_enable = clipping_peak_enable + my_predbat.clipping_buffer_enable = clipping_buffer_enable my_predbat.clipping_cost_weight = clipping_cost_weight - my_predbat.clipping_peak_amplification = clipping_peak_amplification + my_predbat.clipping_amplification = clipping_amplification my_predbat.clipping_limit_override = clipping_limit_override if my_predbat.iboost_enable and (((not iboost_solar) and (not iboost_charging)) or iboost_smart): @@ -762,8 +762,8 @@ def simple_scenario( # Build peak PV step data for clipping tests pv_peak_step = None clipping_limit_eff = 0 - if clipping_peak_enable: - pv_peak_step = {k: v * clipping_peak_amplification for k, v in pv_step.items()} + if clipping_buffer_enable: + pv_peak_step = {k: v * clipping_amplification for k, v in pv_step.items()} if clipping_limit_override > 0: clipping_limit_eff = clipping_limit_override else: @@ -783,7 +783,7 @@ def simple_scenario( my_predbat, pv_step, pv10_step, load_step, load10_step, pv_forecast_peak_step=pv_peak_step, clipping_limit=clipping_limit_eff, - clipping_cost_weight=clipping_cost_weight if clipping_peak_enable else 0, + clipping_cost_weight=clipping_cost_weight if clipping_buffer_enable else 0, ) compute_charge_limit = False diff --git a/apps/predbat/tests/test_model.py b/apps/predbat/tests/test_model.py index f3cb7a28f..26062c734 100644 --- a/apps/predbat/tests/test_model.py +++ b/apps/predbat/tests/test_model.py @@ -1907,7 +1907,7 @@ def run_model_tests(my_predbat): assert_final_soc=0, with_battery=False, inverter_limit=1.0, - clipping_peak_enable=True, + clipping_buffer_enable=True, clipping_cost_weight=1.0, ) @@ -1925,7 +1925,7 @@ def run_model_tests(my_predbat): assert_final_soc=0, with_battery=False, inverter_limit=1.0, - clipping_peak_enable=False, + clipping_buffer_enable=False, return_prediction_handle=True, ) failed |= failed_no_penalty @@ -1939,7 +1939,7 @@ def run_model_tests(my_predbat): assert_final_soc=0, with_battery=False, inverter_limit=1.0, - clipping_peak_enable=True, + clipping_buffer_enable=True, clipping_cost_weight=1.0, return_prediction_handle=True, ignore_failed=True, @@ -1966,7 +1966,7 @@ def run_model_tests(my_predbat): battery_soc=0.0, battery_size=100.0, inverter_limit=1.0, - clipping_peak_enable=True, + clipping_buffer_enable=True, clipping_cost_weight=1.0, ) From 2c8661d6876a7bd1a2c417daf22e53fc729d3155 Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 18 Jun 2026 16:37:52 +0100 Subject: [PATCH 37/74] docs: Add clipping buffer documentation --- docs/clipping.md | 29 +++++++++++++++++++++++++++++ mkdocs.yml | 1 + 2 files changed, 30 insertions(+) create mode 100644 docs/clipping.md diff --git a/docs/clipping.md b/docs/clipping.md new file mode 100644 index 000000000..a7f321ebf --- /dev/null +++ b/docs/clipping.md @@ -0,0 +1,29 @@ +# Automated Solar Clipping Mitigation + +In solar setups where your peak PV generation exceeds your inverter's maximum AC limit (for example, a 6.5kW solar array connected to a 5kW inverter), the excess energy is "clipped" and permanently lost if your battery is already fully charged. + +Historically, Predbat's standard planning lacked visibility into the precise magnitude of this expected clipped energy, meaning it could not preemptively create room in the battery to absorb the excess yield. + +The **Clipping Buffer** feature introduces an intelligent mitigation layer that calculates the exact energy expected to be lost to inverter clipping, and automatically schedules preemptive battery exports to create the necessary buffer before the peak hits. + +## Key Features +* **Dynamic Clipping Prediction**: Analyzes the raw PV forecast curve against the user's configured `inverter_limit` to calculate the exact volume of energy (in kWh) that will exceed AC capacity. +* **Preemptive Export Scheduling**: Automatically forces battery export windows prior to peak generation periods, guaranteeing the battery has precisely enough headroom to soak up the excess energy. +* **Clear-Sky Cloud Modeling**: Includes an optional integration for clear-sky data (e.g., `ha_solcast_clearsky`) and an auto-tuning amplification factor. This generates a theoretical maximum generation envelope, ensuring the clipping buffer is sized safely even when standard forecasts fluctuate due to unpredictable cloud cover. + +## Configuration Settings +These settings can be found in the Home Assistant Predbat Configuration panel, directly beneath the Cloud Model settings. + +* **`clipping_buffer_enable`**: The master switch. Turns the entire clipping buffer feature on or off. +* **`clipping_clearsky_source`**: Determines where Predbat gets its clear-sky (theoretical maximum) data. Options include `auto`, `ha_solcast_clearsky`, `solcast_api`, and `openmeteo`. This piggybacks on your existing Predbat/HA API integrations, requiring no extra YAML configuration. +* **`clipping_use_clearsky_peaks`**: If enabled, Predbat uses the selected Clear-Sky source to size the buffer, rather than just the standard forecast. This is highly recommended to protect against cloud-edge spikes. +* **`clipping_auto_tune`**: Automatically learns the scaling difference between your standard solar forecast and your hardware limits based on past clipping behavior, saving a tracking multiplier to `clipping_auto_tune.json`. +* **`clipping_amplification`**: A manual multiplier (e.g., `1.5x`) to force the standard solar forecast higher to simulate a sunny spike. This is only utilized if Auto-Tune is disabled. +* **`clipping_cost_weight`**: An internal optimizer multiplier (default `1.0`). If Predbat isn't dumping the battery aggressively enough before a peak, increasing this number adds a harsher financial penalty for clipping, forcing the optimizer to prioritize creating headroom. +* **`clipping_limit_override`**: Manually define your inverter's AC ceiling in Watts (e.g., `5000W`). If left at `0`, Predbat auto-detects it from your inverter entity. + +### Manual Overrides +If you do not want to use the dynamic cloud-based tracking, you can manually force a static buffer size and time window: +* **`clipping_buffer_max_kwh`**: Manually force the maximum size of the clipping buffer (e.g. `3.0` kWh). +* **`clipping_buffer_start_time`**: The start time for the fixed buffer window. +* **`clipping_buffer_end_time`**: The end time for the fixed buffer window. diff --git a/mkdocs.yml b/mkdocs.yml index 5fb8786bc..cb19ba6ca 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -13,6 +13,7 @@ nav: - energy-rates.md - components.md - load-ml.md + - clipping.md - car-charging.md - configuration-guide.md - customisation.md From 52569b1447e79c5ec7a31ee0696dc24cf8a7a457 Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 18 Jun 2026 17:31:03 +0100 Subject: [PATCH 38/74] Fix UI clipping chart data range alignment --- apps/predbat/web.py | 38 +++++++++++++++----------------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/apps/predbat/web.py b/apps/predbat/web.py index bade0663f..bd4a0c001 100644 --- a/apps/predbat/web.py +++ b/apps/predbat/web.py @@ -3058,8 +3058,8 @@ def get_chart(self, chart): if getattr(self.base, "predict_clipping_remaining_best", None): for minute, kwh in self.base.predict_clipping_remaining_best.items(): - if minute % step_size == 0 and minute >= self.minutes_now: - minute_timestamp = self.midnight_utc + timedelta(minutes=minute) + if minute % step_size == 0 and minute >= 0: + minute_timestamp = self.now_utc + timedelta(minutes=minute) stamp = minute_timestamp.strftime(TIME_FORMAT) clipping_remaining_series[stamp] = round(kwh, 2) @@ -3068,30 +3068,22 @@ def get_chart(self, chart): if getattr(self.base, "predict_clipping_ceiling_best", None): for minute, kwh in self.base.predict_clipping_ceiling_best.items(): - if minute % step_size == 0 and minute >= self.minutes_now: - minute_timestamp = self.midnight_utc + timedelta(minutes=minute) + if minute % step_size == 0 and minute >= 0: + minute_timestamp = self.now_utc + timedelta(minutes=minute) stamp = minute_timestamp.strftime(TIME_FORMAT) clipping_ceiling_series[stamp] = round(kwh, 2) - # Raw PV forecast (Base) - raw_pv_series = {} - raw_pv_data = getattr(self.base, "pv_forecast_minute", {}) - if raw_pv_data: - for minute, kw in raw_pv_data.items(): - if minute % step_size == 0: - minute_timestamp = self.midnight_utc + timedelta(minutes=minute) - stamp = minute_timestamp.strftime(TIME_FORMAT) - raw_pv_series[stamp] = round(kw * 60.0, 2) - - # ClearSky PV forecast - cs_pv_series = {} - cs_pv_data = getattr(self.base, "pv_forecast_minuteCS", {}) - if cs_pv_data: - for minute, kw in cs_pv_data.items(): - if minute % step_size == 0: - minute_timestamp = self.midnight_utc + timedelta(minutes=minute) - stamp = minute_timestamp.strftime(TIME_FORMAT) - cs_pv_series[stamp] = round(kw * 60.0, 2) + # PV Forecast (Base) - historical and future + pv_forecast_hist = history_attribute(self.get_history_wrapper("sensor." + self.prefix + "_pv_forecast_h0", 3, required=False)) + raw_pv_series = prune_today(pv_forecast_hist, self.now_utc, self.midnight_utc, prune=False, prune_past_days=2, intermediate=True) + raw_pv_series.update(prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + "_pv_today", "pv_estimate"), self.now_utc, self.midnight_utc, prune=False, intermediate=True)) + raw_pv_series.update(prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + "_pv_tomorrow", "pv_estimate"), self.now_utc, self.midnight_utc, prune=False, intermediate=True)) + + # PV Forecast (ClearSky) - historical and future + pv_forecast_histCS = history_attribute(self.get_history_wrapper("sensor." + self.prefix + "_pv_forecast_h0", 3, required=False), attributes=True, state_key="nowCS") + cs_pv_series = prune_today(pv_forecast_histCS, self.now_utc, self.midnight_utc, prune=False, prune_past_days=2, intermediate=True) + cs_pv_series.update(prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + "_pv_today", "pv_clearsky"), self.now_utc, self.midnight_utc, prune=False, intermediate=True)) + cs_pv_series.update(prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + "_pv_tomorrow", "pv_clearsky"), self.now_utc, self.midnight_utc, prune=False, intermediate=True)) series_data = [ {"name": "Clipping Remaining", "data": clipping_remaining_series, "opacity": "1.0", "stroke_width": "3", "stroke_curve": "smooth", "color": "#FF1493", "unit": "kWh"}, From f3fe01e771c4d589596600cdd23a61150c99c02f Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 18 Jun 2026 19:38:07 +0100 Subject: [PATCH 39/74] UI: Move clipping settings to cloud section --- apps/predbat/config.py | 198 +++++++++++++++++++++-------------------- 1 file changed, 100 insertions(+), 98 deletions(-) diff --git a/apps/predbat/config.py b/apps/predbat/config.py index 2110b4813..8311534b5 100644 --- a/apps/predbat/config.py +++ b/apps/predbat/config.py @@ -487,6 +487,105 @@ "default": True, "enable": "expert_mode", }, + { + "name": "clipping_buffer_enable", + "friendly_name": "Clipping Buffer Enable", + "type": "switch", + "default": False, + "icon": "mdi:chart-bell-curve-cumulative", + }, + { + "name": "clipping_clearsky_source", + "friendly_name": "Clipping Clear-Sky Source", + "type": "select", + "options": ["auto", "ha_solcast_clearsky", "solcast_api", "openmeteo"], + "icon": "mdi:cloud-search", + "default": "auto", + "enable": "clipping_buffer_enable", + }, + { + "name": "clipping_use_clearsky_peaks", + "friendly_name": "Clipping Use ClearSky Peaks", + "type": "switch", + "default": True, + "icon": "mdi:weather-sunny-alert", + "enable": "clipping_buffer_enable", + }, + { + "name": "clipping_auto_tune", + "friendly_name": "Clipping Auto-Tune", + "type": "switch", + "default": True, + "icon": "mdi:auto-fix", + "enable": "clipping_buffer_enable", + }, + { + "name": "clipping_cost_weight", + "friendly_name": "Clipping Cost Weight", + "type": "input_number", + "min": 0, + "max": 10.0, + "step": 0.1, + "unit": "x", + "icon": "mdi:multiplication", + "enable": "clipping_buffer_enable", + "default": 1.0, + }, + { + "name": "clipping_amplification", + "friendly_name": "Clipping Amplification", + "type": "input_number", + "min": 0.5, + "max": 3.0, + "step": 0.1, + "unit": "x", + "icon": "mdi:arrow-expand-vertical", + "enable": "clipping_buffer_enable", + "default": 1.0, + }, + { + "name": "clipping_limit_override", + "friendly_name": "Clipping Limit Override", + "type": "input_number", + "min": 0, + "max": 50000, + "step": 100, + "unit": "W", + "icon": "mdi:flash-alert", + "enable": "clipping_buffer_enable", + "default": 0, + }, + { + "name": "clipping_buffer_max_kwh", + "friendly_name": "Clipping Buffer Max kWh (Manual Override)", + "type": "input_number", + "min": 0, + "max": 50.0, + "step": 0.1, + "unit": "kWh", + "icon": "mdi:battery-50", + "default": 0, + "enable": "clipping_buffer_enable", + }, + { + "name": "clipping_buffer_start_time", + "friendly_name": "Clipping Buffer Start Time", + "type": "select", + "options": ["None"] + OPTIONS_TIME, + "icon": "mdi:clock-start", + "default": "None", + "enable": "clipping_buffer_enable", + }, + { + "name": "clipping_buffer_end_time", + "friendly_name": "Clipping Buffer End Time", + "type": "select", + "options": ["None"] + OPTIONS_TIME, + "icon": "mdi:clock-end", + "default": "None", + "enable": "clipping_buffer_enable", + }, + { "name": "metric_pv_calibration_enable", "friendly_name": "Enable use of Calibrated PV data", @@ -608,104 +707,7 @@ "type": "switch", "default": True, }, - { - "name": "clipping_buffer_enable", - "friendly_name": "Clipping Buffer Enable", - "type": "switch", - "default": False, - "icon": "mdi:chart-bell-curve-cumulative", - }, - { - "name": "clipping_clearsky_source", - "friendly_name": "Clipping Clear-Sky Source", - "type": "select", - "options": ["auto", "ha_solcast_clearsky", "solcast_api", "openmeteo"], - "icon": "mdi:cloud-search", - "default": "auto", - "enable": "clipping_buffer_enable", - }, - { - "name": "clipping_use_clearsky_peaks", - "friendly_name": "Clipping Use ClearSky Peaks", - "type": "switch", - "default": True, - "icon": "mdi:weather-sunny-alert", - "enable": "clipping_buffer_enable", - }, - { - "name": "clipping_auto_tune", - "friendly_name": "Clipping Auto-Tune", - "type": "switch", - "default": True, - "icon": "mdi:auto-fix", - "enable": "clipping_buffer_enable", - }, - { - "name": "clipping_cost_weight", - "friendly_name": "Clipping Cost Weight", - "type": "input_number", - "min": 0, - "max": 10.0, - "step": 0.1, - "unit": "x", - "icon": "mdi:multiplication", - "enable": "clipping_buffer_enable", - "default": 1.0, - }, - { - "name": "clipping_amplification", - "friendly_name": "Clipping Amplification", - "type": "input_number", - "min": 0.5, - "max": 3.0, - "step": 0.1, - "unit": "x", - "icon": "mdi:arrow-expand-vertical", - "enable": "clipping_buffer_enable", - "default": 1.0, - }, - { - "name": "clipping_limit_override", - "friendly_name": "Clipping Limit Override", - "type": "input_number", - "min": 0, - "max": 50000, - "step": 100, - "unit": "W", - "icon": "mdi:flash-alert", - "enable": "clipping_buffer_enable", - "default": 0, - }, - { - "name": "clipping_buffer_max_kwh", - "friendly_name": "Clipping Buffer Max kWh (Manual Override)", - "type": "input_number", - "min": 0, - "max": 50.0, - "step": 0.1, - "unit": "kWh", - "icon": "mdi:battery-50", - "default": 0, - "enable": "clipping_buffer_enable", - }, - { - "name": "clipping_buffer_start_time", - "friendly_name": "Clipping Buffer Start Time", - "type": "select", - "options": ["None"] + OPTIONS_TIME, - "icon": "mdi:clock-start", - "default": "None", - "enable": "clipping_buffer_enable", - }, - { - "name": "clipping_buffer_end_time", - "friendly_name": "Clipping Buffer End Time", - "type": "select", - "options": ["None"] + OPTIONS_TIME, - "icon": "mdi:clock-end", - "default": "None", - "enable": "clipping_buffer_enable", - }, + { "name": "car_energy_reported_load", "friendly_name": "Car energy is reported in load data (inside CT clamp)", From b1d09f9934e97d9989e6d5e05a19ebefeb1119e1 Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 18 Jun 2026 19:54:38 +0100 Subject: [PATCH 40/74] UI: Add remaining forecast days (d2-d6) to PV/PV7 and Clipping charts --- apps/predbat/web.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/apps/predbat/web.py b/apps/predbat/web.py index bd4a0c001..eb7552e26 100644 --- a/apps/predbat/web.py +++ b/apps/predbat/web.py @@ -2989,6 +2989,12 @@ def get_chart(self, chart): pv_today_forecast90.update(prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + "_pv_tomorrow", "pv_estimate90"), self.now_utc, self.midnight_utc, prune=False, intermediate=True)) pv_today_forecastCL.update(prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + "_pv_tomorrow", "pv_estimateCL"), self.now_utc, self.midnight_utc, prune=False, intermediate=True)) pv_today_forecastCS.update(prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + "_pv_tomorrow", "pv_clearsky"), self.now_utc, self.midnight_utc, prune=False, intermediate=True)) + for d in ["d2", "d3", "d4", "d5", "d6"]: + pv_today_forecast.update(prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + f"_pv_{d}", "pv_estimate"), self.now_utc, self.midnight_utc, prune=False, intermediate=True)) + pv_today_forecast10.update(prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + f"_pv_{d}", "pv_estimate10"), self.now_utc, self.midnight_utc, prune=False, intermediate=True)) + pv_today_forecast90.update(prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + f"_pv_{d}", "pv_estimate90"), self.now_utc, self.midnight_utc, prune=False, intermediate=True)) + pv_today_forecastCL.update(prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + f"_pv_{d}", "pv_estimateCL"), self.now_utc, self.midnight_utc, prune=False, intermediate=True)) + pv_today_forecastCS.update(prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + f"_pv_{d}", "pv_clearsky"), self.now_utc, self.midnight_utc, prune=False, intermediate=True)) series_data = [ {"name": "PV Power", "data": pv_power, "opacity": "1.0", "stroke_width": "3", "stroke_curve": "smooth", "color": "#f5c43d"}, @@ -3075,15 +3081,19 @@ def get_chart(self, chart): # PV Forecast (Base) - historical and future pv_forecast_hist = history_attribute(self.get_history_wrapper("sensor." + self.prefix + "_pv_forecast_h0", 3, required=False)) - raw_pv_series = prune_today(pv_forecast_hist, self.now_utc, self.midnight_utc, prune=False, prune_past_days=2, intermediate=True) - raw_pv_series.update(prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + "_pv_today", "pv_estimate"), self.now_utc, self.midnight_utc, prune=False, intermediate=True)) - raw_pv_series.update(prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + "_pv_tomorrow", "pv_estimate"), self.now_utc, self.midnight_utc, prune=False, intermediate=True)) + raw_pv_series = prune_today(pv_forecast_hist, self.now_utc, self.midnight_utc, prune=True, prune_past_days=2, intermediate=True) + raw_pv_series.update(prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + "_pv_today", "pv_estimate"), self.now_utc, self.midnight_utc, prune=True, intermediate=True)) + raw_pv_series.update(prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + "_pv_tomorrow", "pv_estimate"), self.now_utc, self.midnight_utc, prune=True, intermediate=True)) + for d in ["d2", "d3"]: + raw_pv_series.update(prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + f"_pv_{d}", "pv_estimate"), self.now_utc, self.midnight_utc, prune=True, intermediate=True)) # PV Forecast (ClearSky) - historical and future pv_forecast_histCS = history_attribute(self.get_history_wrapper("sensor." + self.prefix + "_pv_forecast_h0", 3, required=False), attributes=True, state_key="nowCS") - cs_pv_series = prune_today(pv_forecast_histCS, self.now_utc, self.midnight_utc, prune=False, prune_past_days=2, intermediate=True) - cs_pv_series.update(prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + "_pv_today", "pv_clearsky"), self.now_utc, self.midnight_utc, prune=False, intermediate=True)) - cs_pv_series.update(prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + "_pv_tomorrow", "pv_clearsky"), self.now_utc, self.midnight_utc, prune=False, intermediate=True)) + cs_pv_series = prune_today(pv_forecast_histCS, self.now_utc, self.midnight_utc, prune=True, prune_past_days=2, intermediate=True) + cs_pv_series.update(prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + "_pv_today", "pv_clearsky"), self.now_utc, self.midnight_utc, prune=True, intermediate=True)) + cs_pv_series.update(prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + "_pv_tomorrow", "pv_clearsky"), self.now_utc, self.midnight_utc, prune=True, intermediate=True)) + for d in ["d2", "d3"]: + cs_pv_series.update(prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + f"_pv_{d}", "pv_clearsky"), self.now_utc, self.midnight_utc, prune=True, intermediate=True)) series_data = [ {"name": "Clipping Remaining", "data": clipping_remaining_series, "opacity": "1.0", "stroke_width": "3", "stroke_curve": "smooth", "color": "#FF1493", "unit": "kWh"}, From f87c3ef165d87a2b5bcfe61d83f37519be35eabe Mon Sep 17 00:00:00 2001 From: "pre-commit-ci-lite[bot]" <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Date: Fri, 19 Jun 2026 07:42:46 +0000 Subject: [PATCH 41/74] [pre-commit.ci lite] apply automatic fixes --- apps/predbat/config.py | 20 ++--- apps/predbat/output.py | 11 +-- apps/predbat/plan.py | 117 ++++++++++++------------- apps/predbat/prediction.py | 17 +++- apps/predbat/solcast.py | 4 +- apps/predbat/tests/compare_clipping.py | 3 +- apps/predbat/tests/test_clipping.py | 70 ++++++++------- apps/predbat/tests/test_infra.py | 6 +- apps/predbat/web.py | 35 ++------ docs/clipping.md | 4 + 10 files changed, 143 insertions(+), 144 deletions(-) diff --git a/apps/predbat/config.py b/apps/predbat/config.py index 8311534b5..d1f4d9ff3 100644 --- a/apps/predbat/config.py +++ b/apps/predbat/config.py @@ -494,15 +494,15 @@ "default": False, "icon": "mdi:chart-bell-curve-cumulative", }, - { - "name": "clipping_clearsky_source", - "friendly_name": "Clipping Clear-Sky Source", - "type": "select", - "options": ["auto", "ha_solcast_clearsky", "solcast_api", "openmeteo"], - "icon": "mdi:cloud-search", - "default": "auto", - "enable": "clipping_buffer_enable", - }, + { + "name": "clipping_clearsky_source", + "friendly_name": "Clipping Clear-Sky Source", + "type": "select", + "options": ["auto", "ha_solcast_clearsky", "solcast_api", "openmeteo"], + "icon": "mdi:cloud-search", + "default": "auto", + "enable": "clipping_buffer_enable", + }, { "name": "clipping_use_clearsky_peaks", "friendly_name": "Clipping Use ClearSky Peaks", @@ -585,7 +585,6 @@ "default": "None", "enable": "clipping_buffer_enable", }, - { "name": "metric_pv_calibration_enable", "friendly_name": "Enable use of Calibrated PV data", @@ -707,7 +706,6 @@ "type": "switch", "default": True, }, - { "name": "car_energy_reported_load", "friendly_name": "Car energy is reported in load data (inside CT clamp)", diff --git a/apps/predbat/output.py b/apps/predbat/output.py index ef1dd2032..9fbc0426b 100644 --- a/apps/predbat/output.py +++ b/apps/predbat/output.py @@ -953,7 +953,7 @@ def short_textual_plan(self, soc_min, soc_min_minute, pv_forecast_minute_step, p end_str = "" start_stamp = None end_stamp = None - + prev_val = 0.0 for min_key, val in sorted(predict_clipped_best.items()): if val > prev_val + 0.001: @@ -961,14 +961,12 @@ def short_textual_plan(self, soc_min, soc_min_minute, pv_forecast_minute_step, p start_stamp = self.midnight_utc + timedelta(minutes=min_key) end_stamp = self.midnight_utc + timedelta(minutes=min_key) prev_val = val - + if start_stamp and end_stamp: start_str = start_stamp.strftime("%H:%M") end_str = end_stamp.strftime("%H:%M") - - sentence += "- Forecast {} kWh clipping, exceeding {} limit from {} to {}. Plan penalized to mitigate.\n".format( - dp2(clipping_total), clipping_mode, start_str, end_str - ) + + sentence += "- Forecast {} kWh clipping, exceeding {} limit from {} to {}. Plan penalized to mitigate.\n".format(dp2(clipping_total), clipping_mode, start_str, end_str) if publish: self.text_plan = self.get_text_plan_html(sentence) @@ -2473,7 +2471,6 @@ def record_status(self, message, debug="", had_errors=False, notify=False, extra # Clipping Status and PV Peak Forecast sensors are published in plan.py run_prediction(save="best") # to avoid duplicate dashboard_item writes to the same sensor. - if had_errors: self.log("Warn: record_status {}".format(message + extra)) else: diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index 8a79fafec..bbb1f79aa 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -851,12 +851,12 @@ def inject_clipping_export_windows(self): if not getattr(self, "clipping_buffer_enable", False): self.log("inject_clipping_export_windows: Aborting because clipping_buffer_enable is False") return - + forecast = getattr(self, "clipping_buffer_forecast_kwh", {}) if not forecast: self.log("inject_clipping_export_windows: Aborting because clipping_buffer_forecast_kwh is empty") return - + self.log("inject_clipping_export_windows: Proceeding with clipping blocks: {}".format(forecast)) # Find contiguous blocks of clipping in the forecast @@ -880,41 +880,37 @@ def intersects(w, start, end): for block in clipping_blocks: peak_start, peak_end = block["start"], block["end"] - + # Start the continuous export window from around 08:00 to catch morning export rates, # but no more than 12 hours before the peak, and never in the past. morning_start = max(self.minutes_now, min(peak_start - 300, max(self.minutes_now, peak_start - 720))) morning_start = int(morning_start / 30) * 30 # Align to nearest 30 mins - + if morning_start >= peak_start: continue - new_window = { - "start": morning_start, - "end": peak_end, - "average": self.rate_export.get(morning_start, 0.0) - } - + new_window = {"start": morning_start, "end": peak_end, "average": self.rate_export.get(morning_start, 0.0)} + # We MUST remove any fragmented export windows within this period to ensure a single, continuous - # target SOC can be applied across the entire morning and peak, allowing the battery to slowly + # target SOC can be applied across the entire morning and peak, allowing the battery to slowly # absorb the clipped excess without resetting its target. - + # Clean export_window_best for w in [w for w in self.export_window_best if intersects(w, morning_start, peak_end)]: self.log("Dropped fragmented export window {} to {} in favor of unified spillover window".format(self.time_abs_str(w["start"]), self.time_abs_str(w["end"]))) - + self.export_window_best = [w for w in self.export_window_best if not intersects(w, morning_start, peak_end)] - + # Clean high_export_rates if getattr(self, "high_export_rates", None) is not None: self.high_export_rates = [w for w in self.high_export_rates if not intersects(w, morning_start, peak_end)] - + # Inject our new unified window if not any(w.get("start") <= new_window["start"] and w.get("end") >= new_window["end"] for w in self.export_window_best): self.export_window_best.append(new_window) if getattr(self, "high_export_rates", None) is not None: self.high_export_rates.append(copy.deepcopy(new_window)) - + # Regenerate export limits self.export_limits_best = [100.0] * len(self.export_window_best) self.log("Injected continuous anti-clipping candidate export window {} to {} to allow spillover absorption".format(self.time_abs_str(morning_start), self.time_abs_str(peak_end))) @@ -1044,15 +1040,14 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): clipping_limit_effective = 0 clipping_limit_mode = "Unknown" if self.clipping_buffer_enable: - pv_forecast_peak_step = self.step_data_history( - self.pv_forecast_minute, self.minutes_now, forward=True, cloud_factor=None - ) - + pv_forecast_peak_step = self.step_data_history(self.pv_forecast_minute, self.minutes_now, forward=True, cloud_factor=None) + # Auto-Tuner: Always runs to gather data and recommend a factor import os import json + auto_tune_file = os.path.join(self.config_root, "clipping_auto_tune.json") - + if not hasattr(self, "clipping_auto_amp"): self.clipping_auto_amp = getattr(self, "clipping_amplification", 1.0) self.clipping_last_tune_day = None @@ -1064,10 +1059,10 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): self.clipping_last_tune_day = data.get("last_tune_day", None) except Exception: pass - + auto_amp = self.clipping_auto_amp last_tune_day = self.clipping_last_tune_day - + # Check if we should tune today (only once per day) current_day = self.now_utc.strftime("%Y-%m-%d") if current_day != last_tune_day: @@ -1085,12 +1080,12 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): power_amount = max(0, next_value - current_value) * 60.0 / 5.0 if power_amount > max_pv_power: max_pv_power = power_amount - + # Check inverter limit limit = 0.0 if self.inverter_limit > 0: limit = self.inverter_limit / MINUTE_WATT - + if limit > 0: if max_pv_power >= limit * 0.98: auto_amp = min(2.5, auto_amp + 0.05) @@ -1098,7 +1093,7 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): else: auto_amp = max(1.0, auto_amp - 0.01) self.log("Clipping auto-tuner: No clipping detected. Decreased recommended auto_amp to {}".format(auto_amp)) - + try: with open(auto_tune_file, "w") as f: json.dump({"auto_amp": auto_amp, "last_tune_day": current_day}, f) @@ -1106,9 +1101,9 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): self.clipping_last_tune_day = current_day except Exception: pass - + self.dashboard_item(self.prefix + ".clipping_recommended_amplification", state=dp2(auto_amp), attributes={"friendly_name": "Clipping Recommended Amplification", "icon": "mdi:auto-fix"}) - + # Apply Manual vs Auto Factor if getattr(self, "clipping_auto_tune", False): effective_amplification = auto_amp @@ -1117,12 +1112,8 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): # Apply ClearSky or Amplification factor if getattr(self, "clipping_use_clearsky_peaks", False): - pv_clearsky_step = self.step_data_history( - getattr(self, "pv_forecast_minuteCS", {}), self.minutes_now, forward=True, cloud_factor=None - ) - pv_forecast_peak_step = { - k: max(v, pv_clearsky_step.get(k, 0) * effective_amplification) for k, v in pv_forecast_peak_step.items() - } + pv_clearsky_step = self.step_data_history(getattr(self, "pv_forecast_minuteCS", {}), self.minutes_now, forward=True, cloud_factor=None) + pv_forecast_peak_step = {k: max(v, pv_clearsky_step.get(k, 0) * effective_amplification) for k, v in pv_forecast_peak_step.items()} elif effective_amplification != 1.0: pv_forecast_peak_step = {k: v * effective_amplification for k, v in pv_forecast_peak_step.items()} @@ -1138,26 +1129,26 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): limits.append((self.export_limit, "DNO Export Limit")) if getattr(self, "pv_ac_limit", 0) > 0: limits.append((self.pv_ac_limit, "PV AC Limit")) - + if limits: clipping_limit_effective, clipping_limit_mode = min(limits, key=lambda x: x[0]) else: clipping_limit_effective = 0 clipping_limit_mode = "No Limit" - + self.clipping_limit_effective = clipping_limit_effective self.clipping_limit_mode = clipping_limit_mode # Hybrid Clipping: Read Manual Overrides self.clipping_buffer_kwh = float(self.get_arg("clipping_buffer_max_kwh", default=0.0)) - + start_time_str = self.get_arg("clipping_buffer_start_time", default="None") end_time_str = self.get_arg("clipping_buffer_end_time", default="None") - + self.clipping_buffer_start = None if start_time_str and start_time_str != "None": self.clipping_buffer_start = self.time_to_minutes(start_time_str) - + self.clipping_buffer_end = None if end_time_str and end_time_str != "None": self.clipping_buffer_end = self.time_to_minutes(end_time_str) @@ -1166,7 +1157,7 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): self.clipping_buffer_forecast_kwh = {} self.clipping_remaining_today = 0.0 self.clipping_tomorrow = 0.0 - + if self.clipping_buffer_enable and pv_forecast_peak_step and clipping_limit_effective > 0: step_size = getattr(self, "plan_interval_minutes", 30) # Accumulate potential clipping loss per interval @@ -1176,7 +1167,7 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): pv_power = pv_forecast_peak_step.get(m, 0) if pv_power > clipping_limit_effective: kwh_loss += (pv_power - clipping_limit_effective) / 60.0 - + if kwh_loss > 0: self.clipping_buffer_forecast_kwh[minute] = kwh_loss # Add to totals @@ -1187,7 +1178,7 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): # If manual override is active, reflect it in the UI totals if self.clipping_buffer_kwh > 0: - self.clipping_remaining_today = max(self.clipping_remaining_today, self.clipping_buffer_kwh) + self.clipping_remaining_today = max(self.clipping_remaining_today, self.clipping_buffer_kwh) # Save step data for debug self.load_minutes_step = load_minutes_step @@ -1201,7 +1192,11 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): # Creation prediction object self.prediction = Prediction( - self, pv_forecast_minute_step, pv_forecast_minute10_step, load_minutes_step, load_minutes_step10, + self, + pv_forecast_minute_step, + pv_forecast_minute10_step, + load_minutes_step, + load_minutes_step10, pv_forecast_peak_step=pv_forecast_peak_step, clipping_limit=clipping_limit_effective, clipping_cost_weight=self.clipping_cost_weight if self.clipping_buffer_enable else 0, @@ -4055,7 +4050,7 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi midnight_today_minute = 24 * 60 - self.minutes_now clipping_today = 0.0 clipping_tomorrow = 0.0 - + if hasattr(self, "predict_clipped_best") and self.predict_clipped_best: # Find closest key <= midnight_today_minute for clipping_today keys = sorted(self.predict_clipped_best.keys()) @@ -4063,29 +4058,29 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi clipping_today = self.predict_clipped_best.get(key_today, 0.0) clipping_total = self.predict_clipped_best.get(keys[-1], 0.0) clipping_tomorrow = max(0.0, clipping_total - clipping_today) - + # Generate Clipping Visual Series (Remaining and Ceiling) for web.py charts predict_clipping_remaining_best = {} predict_clipping_ceiling_best = {} - + clipping_limit_step = getattr(self, "clipping_limit_effective", 0) * (step / 60.0) - pv_forecast_peak_step = getattr(self, 'pv_forecast_minute_stepCS', getattr(self, 'pv_forecast_minute_step90', None)) - + pv_forecast_peak_step = getattr(self, "pv_forecast_minute_stepCS", getattr(self, "pv_forecast_minute_step90", None)) + manual_buffer_active = False if self.clipping_buffer_kwh > 0: if self.clipping_buffer_start is None or self.clipping_buffer_end is None: manual_buffer_active = True - + cumulative_clip = 0.0 max_minute = self.forecast_minutes - + buffer_start = self.clipping_buffer_start if self.clipping_buffer_start else 0 buffer_end = self.clipping_buffer_end if self.clipping_buffer_end else 24 * 60 - + for minute in range(max_minute, -step, -step): remaining = 0.0 minute_absolute = minute + self.minutes_now - + if manual_buffer_active or (self.clipping_buffer_kwh > 0 and buffer_start <= minute_absolute < buffer_end): # Calculate linear decay for manual mode visual graph if buffer_end > buffer_start: @@ -4096,15 +4091,15 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi elif pv_forecast_peak_step and clipping_limit_step > 0 and self.clipping_cost_weight > 0: peak_pv = pv_forecast_peak_step.get(minute, 0) if peak_pv > clipping_limit_step: - cumulative_clip += (peak_pv - clipping_limit_step) + cumulative_clip += peak_pv - clipping_limit_step remaining = cumulative_clip - + predict_clipping_remaining_best[minute] = round(remaining, 4) predict_clipping_ceiling_best[minute] = round(self.soc_max - remaining, 4) - + self.predict_clipping_remaining_best = predict_clipping_remaining_best self.predict_clipping_ceiling_best = predict_clipping_ceiling_best - + self.clipping_remaining_today = clipping_today self.clipping_tomorrow = clipping_tomorrow self.clipping_mitigated_today = clipping_today @@ -4152,12 +4147,13 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi "icon": "mdi:battery-check", }, ) - + clipping_status_text = "No clipping forecast." clipping_start_iso = None clipping_end_iso = None - + if self.clipping_buffer_kwh > 0: + def format_time_human(minute): if minute is None: return "N/A" @@ -4166,18 +4162,19 @@ def format_time_human(minute): return target_dt.strftime("%H:%M") else: return target_dt.strftime("Tomorrow %H:%M") + start_str = format_time_human(self.clipping_buffer_start) end_str = format_time_human(self.clipping_buffer_end) if self.clipping_buffer_start is not None: clipping_start_iso = (self.midnight_utc + timedelta(minutes=self.clipping_buffer_start)).isoformat() if self.clipping_buffer_end is not None: clipping_end_iso = (self.midnight_utc + timedelta(minutes=self.clipping_buffer_end)).isoformat() - + if self.clipping_buffer_start is not None and self.clipping_buffer_end is not None: clipping_status_text = "{} kWh clipping forecast ({}) between {} and {}.".format(dp2(self.clipping_buffer_kwh), self.clipping_mode, start_str, end_str) else: clipping_status_text = "{} kWh manual clipping buffer override active.".format(dp2(self.clipping_buffer_kwh)) - + self.dashboard_item( self.prefix + ".clipping_status", state=clipping_status_text, diff --git a/apps/predbat/prediction.py b/apps/predbat/prediction.py index c04526b84..1171048de 100644 --- a/apps/predbat/prediction.py +++ b/apps/predbat/prediction.py @@ -92,7 +92,22 @@ class Prediction: Class to hold prediction input and output data and the run function """ - def __init__(self, base=None, pv_forecast_minute_step=None, pv_forecast_minute10_step=None, load_minutes_step=None, load_minutes_step10=None, soc_kw=None, soc_max=None, pv_forecast_peak_step=None, clipping_limit=0, clipping_cost_weight=0, clipping_buffer_kwh=0, clipping_buffer_start=None, clipping_buffer_end=None): + def __init__( + self, + base=None, + pv_forecast_minute_step=None, + pv_forecast_minute10_step=None, + load_minutes_step=None, + load_minutes_step10=None, + soc_kw=None, + soc_max=None, + pv_forecast_peak_step=None, + clipping_limit=0, + clipping_cost_weight=0, + clipping_buffer_kwh=0, + clipping_buffer_start=None, + clipping_buffer_end=None, + ): global PRED_GLOBAL if base: self.minutes_now = base.minutes_now diff --git a/apps/predbat/solcast.py b/apps/predbat/solcast.py index 5ef056d03..c27485608 100644 --- a/apps/predbat/solcast.py +++ b/apps/predbat/solcast.py @@ -384,12 +384,12 @@ async def download_open_meteo_data(self, configs=None): # Cap at 1.1 (10% above STC) to prevent unrealistic gains at very cold temperatures. eta_temp = max(0.5, min(1.1, 1.0 - 0.004 * (t_cell - 25.0))) pv50_inst = dp4((gti / 1000.0) * kwp * eta_temp * (1.0 - system_loss)) - + raw_p10 = ensemble_p10.get(ts) raw_p90 = ensemble_p90.get(ts) # ensemble_p10/p90 were computed without temperature derating; apply eta_temp now pv10_inst = dp4(min(raw_p10 * eta_temp, pv50_inst) if raw_p10 is not None else pv50_inst * 0.7) - + if idx < len(cs_gti_values) and cs_gti_values[idx] is not None: cs_gti = cs_gti_values[idx] pv_cs_inst = dp4((cs_gti / 1000.0) * kwp * eta_temp * (1.0 - system_loss)) diff --git a/apps/predbat/tests/compare_clipping.py b/apps/predbat/tests/compare_clipping.py index e1bd9d578..994e2fc7c 100644 --- a/apps/predbat/tests/compare_clipping.py +++ b/apps/predbat/tests/compare_clipping.py @@ -11,14 +11,13 @@ import sys import os -import math import time # Add parent dirs to path sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) -from tests.test_infra import reset_rates, reset_inverter, simple_scenario, Prediction +from tests.test_infra import reset_rates, reset_inverter, simple_scenario # --------------------------------------------------------------------------- diff --git a/apps/predbat/tests/test_clipping.py b/apps/predbat/tests/test_clipping.py index 00b6b7743..d42f9289d 100644 --- a/apps/predbat/tests/test_clipping.py +++ b/apps/predbat/tests/test_clipping.py @@ -8,7 +8,7 @@ # pylint: disable=line-too-long # pylint: disable=attribute-defined-outside-init from tests.test_infra import reset_inverter -import copy + def run_clipping_tests(my_predbat): """ @@ -21,6 +21,7 @@ def run_clipping_tests(my_predbat): failed |= test_inject_cleans_fragmented_windows(my_predbat) return failed + def setup(my_predbat): reset_inverter(my_predbat) my_predbat.clipping_buffer_enable = True @@ -35,68 +36,72 @@ def setup(my_predbat): if not hasattr(my_predbat, "time_abs_str"): my_predbat.time_abs_str = lambda x: str(x) + def test_inject_aborts_if_disabled(my_predbat): print("**** test_inject_aborts_if_disabled ****") failed = False setup(my_predbat) my_predbat.clipping_buffer_enable = False my_predbat.clipping_buffer_forecast_kwh = {720: 1.0} - + my_predbat.inject_clipping_export_windows() - + if len(my_predbat.export_window_best) > 0: print("ERROR: Window was injected when clipping was disabled!") failed = True - - if not failed: print("PASS") + + if not failed: + print("PASS") return failed + def test_inject_aborts_if_empty_forecast(my_predbat): print("**** test_inject_aborts_if_empty_forecast ****") failed = False setup(my_predbat) my_predbat.clipping_buffer_enable = True my_predbat.clipping_buffer_forecast_kwh = {} - + my_predbat.inject_clipping_export_windows() - + if len(my_predbat.export_window_best) > 0: print("ERROR: Window was injected when forecast was empty!") failed = True - - if not failed: print("PASS") + + if not failed: + print("PASS") return failed + def test_inject_creates_contiguous_window(my_predbat): print("**** test_inject_creates_contiguous_window ****") failed = False setup(my_predbat) - my_predbat.minutes_now = 240 # 04:00 + my_predbat.minutes_now = 240 # 04:00 # Peak from 13:00 to 14:00 (780 to 840) - my_predbat.clipping_buffer_forecast_kwh = { - 780: 1.0, - 810: 2.0 - } - + my_predbat.clipping_buffer_forecast_kwh = {780: 1.0, 810: 2.0} + my_predbat.inject_clipping_export_windows() - + if len(my_predbat.export_window_best) != 1: print("ERROR: Expected exactly 1 window injected, got {}".format(len(my_predbat.export_window_best))) return True - + w = my_predbat.export_window_best[0] - + if w["start"] != 240: print("ERROR: Expected window start at 240, got {}".format(w["start"])) failed = True - + if w["end"] != 840: print("ERROR: Expected window end at 840, got {}".format(w["end"])) failed = True - - if not failed: print("PASS") + + if not failed: + print("PASS") return failed + def test_inject_cleans_fragmented_windows(my_predbat): print("**** test_inject_cleans_fragmented_windows ****") failed = False @@ -106,30 +111,31 @@ def test_inject_cleans_fragmented_windows(my_predbat): my_predbat.clipping_buffer_forecast_kwh = { 780: 1.0, } - + # Inject intersecting fragmented windows my_predbat.export_window_best = [ - {"start": 30, "end": 50, "average": 0}, # Before morning_start (60) - should KEEP - {"start": 100, "end": 120, "average": 0}, # Inside the new window - should DROP - {"start": 700, "end": 800, "average": 0}, # Intersecting the new window - should DROP - {"start": 900, "end": 960, "average": 0} # After peak_end - should KEEP + {"start": 30, "end": 50, "average": 0}, # Before morning_start (60) - should KEEP + {"start": 100, "end": 120, "average": 0}, # Inside the new window - should DROP + {"start": 700, "end": 800, "average": 0}, # Intersecting the new window - should DROP + {"start": 900, "end": 960, "average": 0}, # After peak_end - should KEEP ] - + my_predbat.inject_clipping_export_windows() - + # We expect 3 windows: 2 kept + 1 injected if len(my_predbat.export_window_best) != 3: print("ERROR: Expected 3 windows (2 kept + 1 new), got {}".format(len(my_predbat.export_window_best))) return True - + starts = [w["start"] for w in my_predbat.export_window_best] if 100 in starts or 700 in starts: print("ERROR: Fragmented windows were not cleaned!") failed = True - - if 60 not in starts: # Injected window start + + if 60 not in starts: # Injected window start print("ERROR: Injected window not found!") failed = True - if not failed: print("PASS") + if not failed: + print("PASS") return failed diff --git a/apps/predbat/tests/test_infra.py b/apps/predbat/tests/test_infra.py index 971c684f8..c59fbcb65 100644 --- a/apps/predbat/tests/test_infra.py +++ b/apps/predbat/tests/test_infra.py @@ -780,7 +780,11 @@ def simple_scenario( prediction = prediction_handle else: prediction = Prediction( - my_predbat, pv_step, pv10_step, load_step, load10_step, + my_predbat, + pv_step, + pv10_step, + load_step, + load10_step, pv_forecast_peak_step=pv_peak_step, clipping_limit=clipping_limit_eff, clipping_cost_weight=clipping_cost_weight if clipping_buffer_enable else 0, diff --git a/apps/predbat/web.py b/apps/predbat/web.py index eb7552e26..ad92f2bc4 100644 --- a/apps/predbat/web.py +++ b/apps/predbat/web.py @@ -3061,17 +3061,17 @@ def get_chart(self, chart): step_size = getattr(self.base, "plan_interval_minutes", 30) clipping_remaining_raw = history_attribute(self.get_history_wrapper(self.prefix + ".clipping_remaining", 3)) clipping_remaining_series = prune_today(clipping_remaining_raw, self.now_utc, self.midnight_utc, prune_past_days=2) - + if getattr(self.base, "predict_clipping_remaining_best", None): for minute, kwh in self.base.predict_clipping_remaining_best.items(): if minute % step_size == 0 and minute >= 0: minute_timestamp = self.now_utc + timedelta(minutes=minute) stamp = minute_timestamp.strftime(TIME_FORMAT) clipping_remaining_series[stamp] = round(kwh, 2) - + clipping_ceiling_raw = history_attribute(self.get_history_wrapper(self.prefix + ".clipping_ceiling", 3)) clipping_ceiling_series = prune_today(clipping_ceiling_raw, self.now_utc, self.midnight_utc, prune_past_days=2) - + if getattr(self.base, "predict_clipping_ceiling_best", None): for minute, kwh in self.base.predict_clipping_ceiling_best.items(): if minute % step_size == 0 and minute >= 0: @@ -3108,37 +3108,16 @@ def get_chart(self, chart): yaxis_annotations = [] if inverter_ac_limit_kw > 0: - yaxis_annotations.append({ - "y": inverter_ac_limit_kw, - "borderColor": "#FF0000", - "strokeDashArray": 4, - "text": "Inverter AC Capacity ({} kW)".format(inverter_ac_limit_kw), - "textColor": "#fff", - "backgroundColor": "#FF0000" - }) - + yaxis_annotations.append({"y": inverter_ac_limit_kw, "borderColor": "#FF0000", "strokeDashArray": 4, "text": "Inverter AC Capacity ({} kW)".format(inverter_ac_limit_kw), "textColor": "#fff", "backgroundColor": "#FF0000"}) + xaxis_annotations = [] buffer_start = getattr(self.base, "clipping_buffer_start", None) buffer_end = getattr(self.base, "clipping_buffer_end", None) if getattr(self.base, "clipping_buffer_kwh", 0) > 0 and buffer_start is not None and buffer_end is not None: start_stamp = (self.midnight_utc + timedelta(minutes=buffer_start)).strftime(TIME_FORMAT) end_stamp = (self.midnight_utc + timedelta(minutes=buffer_end)).strftime(TIME_FORMAT) - xaxis_annotations.append({ - "x": start_stamp, - "borderColor": "#ffa500", - "strokeDashArray": 4, - "text": "Today Buffer Start", - "orientation": "vertical", - "backgroundColor": "#ffa500" - }) - xaxis_annotations.append({ - "x": end_stamp, - "borderColor": "#ffa500", - "strokeDashArray": 4, - "text": "Today Buffer End", - "orientation": "vertical", - "backgroundColor": "#ffa500" - }) + xaxis_annotations.append({"x": start_stamp, "borderColor": "#ffa500", "strokeDashArray": 4, "text": "Today Buffer Start", "orientation": "vertical", "backgroundColor": "#ffa500"}) + xaxis_annotations.append({"x": end_stamp, "borderColor": "#ffa500", "strokeDashArray": 4, "text": "Today Buffer End", "orientation": "vertical", "backgroundColor": "#ffa500"}) clipping_total = 0 if getattr(self.base, "predict_clipped_best", None): diff --git a/docs/clipping.md b/docs/clipping.md index a7f321ebf..e01ac7ad0 100644 --- a/docs/clipping.md +++ b/docs/clipping.md @@ -7,11 +7,13 @@ Historically, Predbat's standard planning lacked visibility into the precise mag The **Clipping Buffer** feature introduces an intelligent mitigation layer that calculates the exact energy expected to be lost to inverter clipping, and automatically schedules preemptive battery exports to create the necessary buffer before the peak hits. ## Key Features + * **Dynamic Clipping Prediction**: Analyzes the raw PV forecast curve against the user's configured `inverter_limit` to calculate the exact volume of energy (in kWh) that will exceed AC capacity. * **Preemptive Export Scheduling**: Automatically forces battery export windows prior to peak generation periods, guaranteeing the battery has precisely enough headroom to soak up the excess energy. * **Clear-Sky Cloud Modeling**: Includes an optional integration for clear-sky data (e.g., `ha_solcast_clearsky`) and an auto-tuning amplification factor. This generates a theoretical maximum generation envelope, ensuring the clipping buffer is sized safely even when standard forecasts fluctuate due to unpredictable cloud cover. ## Configuration Settings + These settings can be found in the Home Assistant Predbat Configuration panel, directly beneath the Cloud Model settings. * **`clipping_buffer_enable`**: The master switch. Turns the entire clipping buffer feature on or off. @@ -23,7 +25,9 @@ These settings can be found in the Home Assistant Predbat Configuration panel, d * **`clipping_limit_override`**: Manually define your inverter's AC ceiling in Watts (e.g., `5000W`). If left at `0`, Predbat auto-detects it from your inverter entity. ### Manual Overrides + If you do not want to use the dynamic cloud-based tracking, you can manually force a static buffer size and time window: + * **`clipping_buffer_max_kwh`**: Manually force the maximum size of the clipping buffer (e.g. `3.0` kWh). * **`clipping_buffer_start_time`**: The start time for the fixed buffer window. * **`clipping_buffer_end_time`**: The end time for the fixed buffer window. From aeebceda21ffebe8f88b7cadf5b3633d8996e7af Mon Sep 17 00:00:00 2001 From: Robin Date: Sat, 20 Jun 2026 08:04:30 +0100 Subject: [PATCH 42/74] chore: fix linting and template variable names --- .cspell/custom-dictionary-workspace.txt | 2 ++ apps/predbat/config/apps.yaml | 27 ++++++++++++++++++-- apps/predbat/solcast.py | 2 +- docs/clipping.md | 33 ------------------------ docs/customisation.md | 34 +++++++++++++++++++++++++ mkdocs.yml | 1 - 6 files changed, 62 insertions(+), 37 deletions(-) delete mode 100644 docs/clipping.md diff --git a/.cspell/custom-dictionary-workspace.txt b/.cspell/custom-dictionary-workspace.txt index a6716bc84..4418c396d 100644 --- a/.cspell/custom-dictionary-workspace.txt +++ b/.cspell/custom-dictionary-workspace.txt @@ -54,6 +54,7 @@ chargehold chargelater Chrg citem +clearsky Codespaces collapsable compareform @@ -270,6 +271,7 @@ oncharge oninput onmouseout onmouseover +openmeteo openweathermap overfitting overvoltage diff --git a/apps/predbat/config/apps.yaml b/apps/predbat/config/apps.yaml index 39c07b422..cfbb93aec 100644 --- a/apps/predbat/config/apps.yaml +++ b/apps/predbat/config/apps.yaml @@ -267,7 +267,6 @@ pred_bat: # Only use this for workarounds if your inverter time is correct but Predbat is somehow wrong (AppDaemon issue) # 1 means add 1 minute to AppDaemon time, -1 takes it away clock_skew: 0 - # Solcast cloud interface, set this or the local interface below #solcast_host: 'https://api.solcast.com.au/' #solcast_api_key: !secret solcast_api_key @@ -280,7 +279,25 @@ pred_bat: # azimuth: 180 # declination: 10 # api_key: !secret forecast_solar_api_key - # azimuth_zero_south: false # Set to true if azimuth is already in Forecast.solar/Open-Meteo convention (0=South) rather than Predbat convention (0=North) + + # Open-Meteo interface (used for primary PV forecast OR for ClearSky baseline) + #open_meteo_forecast: + # - latitude: 55.9711067 + # longitude: -3.1925157 + # declination: 35 + # azimuth: 180 + # kwp: 4.0 + + # Determines which source Predbat uses for the primary PV forecast. + # Options: "auto" (default), "ha", "forecast_solar", "openmeteo", "solcast_api" + # If you are using Open-Meteo ONLY for the ClearSky baseline below, it's recommended to explicitly set this to "ha" if using the HA integration. + #pv_forecast_primary: "auto" + + # --- Automated Solar Clipping Mitigation --- + # Enable clipping mitigation to automatically force exports prior to peaks + #clipping_buffer_enable: True + # Set the source for the theoretical ClearSky maximum (e.g. "openmeteo", "ha_solcast_clearsky", "solcast_api") + #clipping_clearsky_source: "ha_solcast_clearsky" # Set these to match solcast sensor names if not using the cloud interface # The regular expression (re:) makes the solcast bit optional @@ -290,6 +307,12 @@ pred_bat: pv_forecast_d3: re:(sensor.(solcast_|)(pv_forecast_|)forecast_(day_3|d3)) pv_forecast_d4: re:(sensor.(solcast_|)(pv_forecast_|)forecast_(day_4|d4)) + # ClearSky sensors (Required if clipping_clearsky_source is set to ha_solcast_clearsky) + #pv_clearsky_today: re:(sensor.(solcast_|)(pv_forecast_|)clearsky_today) + #pv_clearsky_tomorrow: re:(sensor.(solcast_|)(pv_forecast_|)clearsky_tomorrow) + #pv_clearsky_d3: re:(sensor.(solcast_|)(pv_forecast_|)clearsky_(day_3|d3)) + #pv_clearsky_d4: re:(sensor.(solcast_|)(pv_forecast_|)clearsky_(day_4|d4)) + # Ohme EV charger cloud direct integration #ohme_login: !secret ohme_login #ohme_password: !secret ohme_password diff --git a/apps/predbat/solcast.py b/apps/predbat/solcast.py index c27485608..a5de656b6 100644 --- a/apps/predbat/solcast.py +++ b/apps/predbat/solcast.py @@ -1370,7 +1370,7 @@ async def fetch_pv_forecast(self): self.log("SolarAPI: Forecast Solar returned no data, falling back to Open-Meteo backup") backup_configs = self.open_meteo_forecast if self.open_meteo_forecast else self.forecast_solar pv_forecast_data, max_kwh = await self.download_open_meteo_data(configs=backup_configs) - elif (pv_forecast_primary == "openmeteo") or (pv_forecast_primary == "auto" and self.open_meteo_forecast): + elif (pv_forecast_primary == "openmeteo") or (pv_forecast_primary == "auto" and self.open_meteo_forecast and self.get_arg("clipping_clearsky_source", "auto", indirect=False) != "openmeteo"): self.log("SolarAPI: Obtaining solar forecast from Open-Meteo API") pv_forecast_data, max_kwh = await self.download_open_meteo_data() divide_by = 30.0 diff --git a/docs/clipping.md b/docs/clipping.md deleted file mode 100644 index e01ac7ad0..000000000 --- a/docs/clipping.md +++ /dev/null @@ -1,33 +0,0 @@ -# Automated Solar Clipping Mitigation - -In solar setups where your peak PV generation exceeds your inverter's maximum AC limit (for example, a 6.5kW solar array connected to a 5kW inverter), the excess energy is "clipped" and permanently lost if your battery is already fully charged. - -Historically, Predbat's standard planning lacked visibility into the precise magnitude of this expected clipped energy, meaning it could not preemptively create room in the battery to absorb the excess yield. - -The **Clipping Buffer** feature introduces an intelligent mitigation layer that calculates the exact energy expected to be lost to inverter clipping, and automatically schedules preemptive battery exports to create the necessary buffer before the peak hits. - -## Key Features - -* **Dynamic Clipping Prediction**: Analyzes the raw PV forecast curve against the user's configured `inverter_limit` to calculate the exact volume of energy (in kWh) that will exceed AC capacity. -* **Preemptive Export Scheduling**: Automatically forces battery export windows prior to peak generation periods, guaranteeing the battery has precisely enough headroom to soak up the excess energy. -* **Clear-Sky Cloud Modeling**: Includes an optional integration for clear-sky data (e.g., `ha_solcast_clearsky`) and an auto-tuning amplification factor. This generates a theoretical maximum generation envelope, ensuring the clipping buffer is sized safely even when standard forecasts fluctuate due to unpredictable cloud cover. - -## Configuration Settings - -These settings can be found in the Home Assistant Predbat Configuration panel, directly beneath the Cloud Model settings. - -* **`clipping_buffer_enable`**: The master switch. Turns the entire clipping buffer feature on or off. -* **`clipping_clearsky_source`**: Determines where Predbat gets its clear-sky (theoretical maximum) data. Options include `auto`, `ha_solcast_clearsky`, `solcast_api`, and `openmeteo`. This piggybacks on your existing Predbat/HA API integrations, requiring no extra YAML configuration. -* **`clipping_use_clearsky_peaks`**: If enabled, Predbat uses the selected Clear-Sky source to size the buffer, rather than just the standard forecast. This is highly recommended to protect against cloud-edge spikes. -* **`clipping_auto_tune`**: Automatically learns the scaling difference between your standard solar forecast and your hardware limits based on past clipping behavior, saving a tracking multiplier to `clipping_auto_tune.json`. -* **`clipping_amplification`**: A manual multiplier (e.g., `1.5x`) to force the standard solar forecast higher to simulate a sunny spike. This is only utilized if Auto-Tune is disabled. -* **`clipping_cost_weight`**: An internal optimizer multiplier (default `1.0`). If Predbat isn't dumping the battery aggressively enough before a peak, increasing this number adds a harsher financial penalty for clipping, forcing the optimizer to prioritize creating headroom. -* **`clipping_limit_override`**: Manually define your inverter's AC ceiling in Watts (e.g., `5000W`). If left at `0`, Predbat auto-detects it from your inverter entity. - -### Manual Overrides - -If you do not want to use the dynamic cloud-based tracking, you can manually force a static buffer size and time window: - -* **`clipping_buffer_max_kwh`**: Manually force the maximum size of the clipping buffer (e.g. `3.0` kWh). -* **`clipping_buffer_start_time`**: The start time for the fixed buffer window. -* **`clipping_buffer_end_time`**: The end time for the fixed buffer window. diff --git a/docs/customisation.md b/docs/customisation.md index 0d6a67336..e419e31df 100644 --- a/docs/customisation.md +++ b/docs/customisation.md @@ -458,6 +458,40 @@ The amount of modulation depends on the standard deviation of your load predicti You can disable this feature (_expert mode only_) using **switch.metric_load_divergence_enable**. +### Automated Solar Clipping Mitigation + +In solar setups where your peak PV generation exceeds your inverter's maximum AC limit (for example, a 6.5kW solar array connected to a 5kW inverter), the excess energy is "clipped" and permanently lost if your battery is already fully charged. + +Historically, Predbat's standard planning lacked visibility into the precise magnitude of this expected clipped energy, meaning it could not preemptively create room in the battery to absorb the excess yield. + +The **Clipping Buffer** feature introduces an intelligent mitigation layer that calculates the exact energy expected to be lost to inverter clipping, and automatically schedules preemptive battery exports to create the necessary buffer before the peak hits. + +#### Key Features + +- **Dynamic Clipping Prediction**: Analyzes the raw PV forecast curve against the user's configured `inverter_limit` to calculate the exact volume of energy (in kWh) that will exceed AC capacity. +- **Preemptive Export Scheduling**: Automatically forces battery export windows prior to peak generation periods, guaranteeing the battery has precisely enough headroom to soak up the excess energy. +- **Clear-Sky Cloud Modeling**: Includes an optional integration for clear-sky data (e.g., `ha_solcast_clearsky`) and an auto-tuning amplification factor. This generates a theoretical maximum generation envelope, ensuring the clipping buffer is sized safely even when standard forecasts fluctuate due to unpredictable cloud cover. + +#### Configuration Settings + +These settings can be found in the Home Assistant Predbat Configuration panel, directly beneath the Cloud Model settings. + +- **`clipping_buffer_enable`**: The master switch. Turns the entire clipping buffer feature on or off. +- **`clipping_clearsky_source`**: Determines where Predbat gets its clear-sky (theoretical maximum) data. Options include `auto`, `ha_solcast_clearsky`, `solcast_api`, and `openmeteo`. This piggybacks on your existing Predbat/HA API integrations, requiring no extra YAML configuration. +- **`clipping_use_clearsky_peaks`**: If enabled, Predbat uses the selected Clear-Sky source to size the buffer, rather than just the standard forecast. This is highly recommended to protect against cloud-edge spikes. +- **`clipping_auto_tune`**: Automatically learns the scaling difference between your standard solar forecast and your hardware limits based on past clipping behavior, saving a tracking multiplier to `clipping_auto_tune.json`. +- **`clipping_amplification`**: A manual multiplier (e.g., `1.5x`) to force the standard solar forecast higher to simulate a sunny spike. This is only utilized if Auto-Tune is disabled. +- **`clipping_cost_weight`**: An internal optimizer multiplier (default `1.0`). If Predbat isn't dumping the battery aggressively enough before a peak, increasing this number adds a harsher financial penalty for clipping, forcing the optimizer to prioritize creating headroom. +- **`clipping_limit_override`**: Manually define your inverter's AC ceiling in Watts (e.g., `5000W`). If left at `0`, Predbat auto-detects it from your inverter entity. + +#### Manual Overrides + +If you do not want to use the dynamic cloud-based tracking, you can manually force a static buffer size and time window: + +- **`clipping_buffer_max_kwh`**: Manually force the maximum size of the clipping buffer (e.g. `3.0` kWh). +- **`clipping_buffer_start_time`**: The start time for the fixed buffer window. +- **`clipping_buffer_end_time`**: The end time for the fixed buffer window. + ## iBoost model options Predbat has an 'iBoost model' that can be used to model using excess solar energy to heat hot water (or similar) instead of it being exported to the grid. diff --git a/mkdocs.yml b/mkdocs.yml index cb19ba6ca..5fb8786bc 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -13,7 +13,6 @@ nav: - energy-rates.md - components.md - load-ml.md - - clipping.md - car-charging.md - configuration-guide.md - customisation.md From 6c4d190c3a07b16916f9e8c110af98a7ef3ffbfa Mon Sep 17 00:00:00 2001 From: Robin Date: Sat, 20 Jun 2026 08:08:56 +0100 Subject: [PATCH 43/74] fix: convert hardware limits from Watts to kW for clipping evaluation --- apps/predbat/plan.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index bbb1f79aa..c37c35c8a 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -1124,11 +1124,11 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): else: limits = [] if self.inverter_limit > 0: - limits.append((self.inverter_limit, "Inverter AC Capacity")) + limits.append((self.inverter_limit / 1000.0, "Inverter AC Capacity")) if self.export_limit > 0: - limits.append((self.export_limit, "DNO Export Limit")) + limits.append((self.export_limit / 1000.0, "DNO Export Limit")) if getattr(self, "pv_ac_limit", 0) > 0: - limits.append((self.pv_ac_limit, "PV AC Limit")) + limits.append((self.pv_ac_limit / 1000.0, "PV AC Limit")) if limits: clipping_limit_effective, clipping_limit_mode = min(limits, key=lambda x: x[0]) From b737b286b758d081a6eef2b65b76b9c23ae19588 Mon Sep 17 00:00:00 2001 From: Robin Date: Sat, 20 Jun 2026 08:13:12 +0100 Subject: [PATCH 44/74] fix: resolve unit mismatch bugs in clipping logic where kW and kWh were improperly compared and calculated --- apps/predbat/plan.py | 11 +++++++---- apps/predbat/prediction.py | 3 ++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index c37c35c8a..2641fa3f9 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -1163,10 +1163,13 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): # Accumulate potential clipping loss per interval for minute in range(0, self.forecast_minutes, step_size): kwh_loss = 0.0 - for m in range(minute, min(minute + step_size, self.forecast_minutes)): - pv_power = pv_forecast_peak_step.get(m, 0) - if pv_power > clipping_limit_effective: - kwh_loss += (pv_power - clipping_limit_effective) / 60.0 + for m in range(minute, min(minute + step_size, self.forecast_minutes), PREDICT_STEP): + step_kwh = pv_forecast_peak_step.get(m, 0) + # step_kwh is energy over PREDICT_STEP minutes + # clipping_limit_effective is in kW. Convert it to max allowed energy in PREDICT_STEP. + max_kwh_allowed = clipping_limit_effective * (PREDICT_STEP / 60.0) + if step_kwh > max_kwh_allowed: + kwh_loss += (step_kwh - max_kwh_allowed) if kwh_loss > 0: self.clipping_buffer_forecast_kwh[minute] = kwh_loss diff --git a/apps/predbat/prediction.py b/apps/predbat/prediction.py index 1171048de..4de569245 100644 --- a/apps/predbat/prediction.py +++ b/apps/predbat/prediction.py @@ -540,7 +540,8 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi carbon_enable = self.carbon_enable pv_forecast_peak_step = self.pv_forecast_peak_step clipping_limit = self.clipping_limit - clipping_limit_step = clipping_limit * step if clipping_limit else 0 + # clipping_limit is in kW. We need the energy limit in kWh for the given step. + clipping_limit_step = clipping_limit * (step / 60.0) if clipping_limit else 0 clipping_cost_weight = self.clipping_cost_weight reserve = self.reserve soc_max = self.soc_max From 8e3083858e96a8cbba592554123d546287771846 Mon Sep 17 00:00:00 2001 From: Robin Date: Sat, 20 Jun 2026 08:27:11 +0100 Subject: [PATCH 45/74] fix: resolve correct internal unit limits for clipping logic and tests --- apps/predbat/plan.py | 6 +++--- apps/predbat/tests/test_infra.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index 2641fa3f9..01acfff09 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -1124,11 +1124,11 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): else: limits = [] if self.inverter_limit > 0: - limits.append((self.inverter_limit / 1000.0, "Inverter AC Capacity")) + limits.append((self.inverter_limit * 60.0, "Inverter AC Capacity")) if self.export_limit > 0: - limits.append((self.export_limit / 1000.0, "DNO Export Limit")) + limits.append((self.export_limit * 60.0, "DNO Export Limit")) if getattr(self, "pv_ac_limit", 0) > 0: - limits.append((self.pv_ac_limit / 1000.0, "PV AC Limit")) + limits.append((self.pv_ac_limit * 60.0, "PV AC Limit")) if limits: clipping_limit_effective, clipping_limit_mode = min(limits, key=lambda x: x[0]) diff --git a/apps/predbat/tests/test_infra.py b/apps/predbat/tests/test_infra.py index c59fbcb65..dde8636b3 100644 --- a/apps/predbat/tests/test_infra.py +++ b/apps/predbat/tests/test_infra.py @@ -774,7 +774,7 @@ def simple_scenario( limits.append(my_predbat.export_limit) if my_predbat.pv_ac_limit > 0: limits.append(my_predbat.pv_ac_limit) - clipping_limit_eff = min(limits) if limits else 0 + clipping_limit_eff = (min(limits) * 60.0) if limits else 0 if prediction_handle: prediction = prediction_handle From 9f6cc096080dc10f3e862d0b823558f1f2e81e96 Mon Sep 17 00:00:00 2001 From: Robin Date: Sat, 20 Jun 2026 15:24:49 +0100 Subject: [PATCH 46/74] fix: core simulation physics, dynamic export bounding, and solcast clear sky units --- apps/predbat/plan.py | 90 ++++++++++++++++++++++++++++---------- apps/predbat/prediction.py | 6 +++ apps/predbat/solcast.py | 8 +++- 3 files changed, 81 insertions(+), 23 deletions(-) diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index 01acfff09..df218a6dc 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -881,9 +881,56 @@ def intersects(w, start, end): for block in clipping_blocks: peak_start, peak_end = block["start"], block["end"] - # Start the continuous export window from around 08:00 to catch morning export rates, - # but no more than 12 hours before the peak, and never in the past. - morning_start = max(self.minutes_now, min(peak_start - 300, max(self.minutes_now, peak_start - 720))) + # Calculate the dynamically sized export window required to create the needed headroom. + # We walk backwards from the peak minute by minute to account for the fact that + # solar generation (PV) during the export window consumes inverter capacity and + # reduces the effective battery discharge rate. + max_kwh_loss = max(kwh_loss for m, kwh_loss in forecast.items() if peak_start <= m <= peak_end) + + # Cap the requested headroom by the physical capacity of the battery. + # We cannot create more headroom than the battery can physically hold. + battery_capacity_kwh = self.soc_max - getattr(self, "best_soc_min", 0.0) + max_kwh_loss = min(max_kwh_loss, battery_capacity_kwh) + + hybrid = getattr(self, "inverter_hybrid", False) + inverter_limit_kw = getattr(self, "set_inverter_limit", 0) / 1000.0 + export_limit_kw = getattr(self, "export_limit", 0) + pv_forecast = getattr(self, "pv_forecast_minute_step", {}) + + accumulated_kwh = 0.0 + minutes_needed = 0 + + for m in range(peak_start, max(self.minutes_now, peak_start - 360), -1): + # Start with raw battery maximum discharge + discharge_kw = self.battery_rate_max_discharge * getattr(self, "battery_rate_max_scaling_discharge", 1.0) + + # Retrieve PV forecast for this minute (kW) + pv_kw = pv_forecast.get(m, 0.0) + + # Hybrid inverters share the AC limit between battery and PV + if hybrid and inverter_limit_kw > 0: + discharge_kw = min(discharge_kw, max(0.0, inverter_limit_kw - pv_kw)) + + # The total site export limit also restricts combined PV + Battery export + if export_limit_kw > 0: + discharge_kw = min(discharge_kw, max(0.0, export_limit_kw - pv_kw)) + + # Safety minimum to prevent infinite loops if limits are misconfigured (0.5 kW) + discharge_kw = max(0.5, discharge_kw) + + accumulated_kwh += discharge_kw * (1.0 / 60.0) + minutes_needed += 1 + + if accumulated_kwh >= max_kwh_loss: + break + + # Add 30 mins safety margin + minutes_needed += 30 + + # Clamp the window between 60 minutes and 360 minutes (6 hours) + minutes_needed = max(60, min(360, minutes_needed)) + + morning_start = max(self.minutes_now, peak_start - minutes_needed) morning_start = int(morning_start / 30) * 30 # Align to nearest 30 mins if morning_start >= peak_start: @@ -891,19 +938,8 @@ def intersects(w, start, end): new_window = {"start": morning_start, "end": peak_end, "average": self.rate_export.get(morning_start, 0.0)} - # We MUST remove any fragmented export windows within this period to ensure a single, continuous - # target SOC can be applied across the entire morning and peak, allowing the battery to slowly - # absorb the clipped excess without resetting its target. - - # Clean export_window_best - for w in [w for w in self.export_window_best if intersects(w, morning_start, peak_end)]: - self.log("Dropped fragmented export window {} to {} in favor of unified spillover window".format(self.time_abs_str(w["start"]), self.time_abs_str(w["end"]))) - - self.export_window_best = [w for w in self.export_window_best if not intersects(w, morning_start, peak_end)] - - # Clean high_export_rates - if getattr(self, "high_export_rates", None) is not None: - self.high_export_rates = [w for w in self.high_export_rates if not intersects(w, morning_start, peak_end)] + # Do NOT delete fragmented windows. Allow the optimizer to evaluate our focused 3-hour window + # alongside any native high-rate export windows it already found. # Inject our new unified window if not any(w.get("start") <= new_window["start"] and w.get("end") >= new_window["end"] for w in self.export_window_best): @@ -4067,14 +4103,16 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi predict_clipping_ceiling_best = {} clipping_limit_step = getattr(self, "clipping_limit_effective", 0) * (step / 60.0) - pv_forecast_peak_step = getattr(self, "pv_forecast_minute_stepCS", getattr(self, "pv_forecast_minute_step90", None)) + # Ensure the chart uses the exact same peak dataset as the planning engine + # which already includes clipping_amplification. + pv_forecast_peak_step = getattr(pred, "pv_forecast_peak_step", None) manual_buffer_active = False if self.clipping_buffer_kwh > 0: if self.clipping_buffer_start is None or self.clipping_buffer_end is None: manual_buffer_active = True - cumulative_clip = 0.0 + daily_cumulative_clip = {} max_minute = self.forecast_minutes buffer_start = self.clipping_buffer_start if self.clipping_buffer_start else 0 @@ -4083,6 +4121,13 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi for minute in range(max_minute, -step, -step): remaining = 0.0 minute_absolute = minute + self.minutes_now + + if minute <= midnight_today_minute: + day_index = 0 + elif minute <= midnight_today_minute + 24 * 60: + day_index = 1 + else: + day_index = 2 if manual_buffer_active or (self.clipping_buffer_kwh > 0 and buffer_start <= minute_absolute < buffer_end): # Calculate linear decay for manual mode visual graph @@ -4094,14 +4139,15 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi elif pv_forecast_peak_step and clipping_limit_step > 0 and self.clipping_cost_weight > 0: peak_pv = pv_forecast_peak_step.get(minute, 0) if peak_pv > clipping_limit_step: - cumulative_clip += peak_pv - clipping_limit_step - remaining = cumulative_clip + daily_cumulative_clip[day_index] = daily_cumulative_clip.get(day_index, 0.0) + (peak_pv - clipping_limit_step) + remaining = daily_cumulative_clip.get(day_index, 0.0) predict_clipping_remaining_best[minute] = round(remaining, 4) predict_clipping_ceiling_best[minute] = round(self.soc_max - remaining, 4) - self.predict_clipping_remaining_best = predict_clipping_remaining_best - self.predict_clipping_ceiling_best = predict_clipping_ceiling_best + # Sort dictionaries to ensure forwards time order for ApexCharts rendering + self.predict_clipping_remaining_best = dict(sorted(predict_clipping_remaining_best.items())) + self.predict_clipping_ceiling_best = dict(sorted(predict_clipping_ceiling_best.items())) self.clipping_remaining_today = clipping_today self.clipping_tomorrow = clipping_tomorrow diff --git a/apps/predbat/prediction.py b/apps/predbat/prediction.py index 4de569245..c0cb99018 100644 --- a/apps/predbat/prediction.py +++ b/apps/predbat/prediction.py @@ -713,6 +713,12 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi max_charge_step = battery_rate_max_charge * battery_rate_max_scaling * step absorbable = min(battery_headroom, max_charge_step) unmitigated_clip = max(potential_clip - absorbable, 0) + + if absorbable > 0: + # Actually route the absorbed free solar into the battery state of charge + # taking into account the battery's charging efficiency loss + soc += absorbable * battery_loss + if unmitigated_clip > 0: clipping_penalty = unmitigated_clip * export_rate * clipping_cost_weight metric += clipping_penalty diff --git a/apps/predbat/solcast.py b/apps/predbat/solcast.py index a5de656b6..7c6aa103b 100644 --- a/apps/predbat/solcast.py +++ b/apps/predbat/solcast.py @@ -1442,7 +1442,7 @@ async def fetch_pv_forecast(self): ha_data += data if ha_data: # Apply factor and divide_by logic like the main PV fetcher - ha_lookup = {item["period_start"]: (float(item.get("pv_estimate") or 0.0) * factor / divide_by) for item in ha_data} + ha_lookup = {item["period_start"]: float(item.get("pv_estimate", 0.0)) for item in ha_data} for item in pv_forecast_data: ts = item.get("period_start") if ts in ha_lookup: @@ -1479,6 +1479,12 @@ async def fetch_pv_forecast(self): if period != 30: self.log("SolarAPI: PV Forecast data has {} minute resolution, adjusting calculations".format(period)) + # Universal fallback for ClearSky data + # Ensure every item has a clearsky value before we pass to minute_data + for item in pv_forecast_data: + if "pv_clearsky" not in item: + item["pv_clearsky"] = item.get("pv_estimate90", item.get("pv_estimate", 0.0)) + pv_forecast_minute, _ = minute_data( pv_forecast_data, self.forecast_days, From 1c4fd1a3644ffba8b8c49af7a5e88311d303cf14 Mon Sep 17 00:00:00 2001 From: Robin Date: Sat, 20 Jun 2026 17:36:43 +0100 Subject: [PATCH 47/74] Fix timezone bug for relative vs absolute minutes in anti-clipping algorithm; Improve status text --- apps/predbat/plan.py | 72 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 64 insertions(+), 8 deletions(-) diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index df218a6dc..56cf552e1 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -862,7 +862,9 @@ def inject_clipping_export_windows(self): # Find contiguous blocks of clipping in the forecast clipping_blocks = [] current_block = None - for minute, kwh_loss in sorted(forecast.items()): + for minute_relative, kwh_loss in sorted(forecast.items()): + # forecast uses relative minutes from now, convert to absolute + minute = minute_relative + self.minutes_now if kwh_loss > 0: if current_block is None: current_block = {"start": minute, "end": minute + 30} @@ -936,20 +938,31 @@ def intersects(w, start, end): if morning_start >= peak_start: continue - new_window = {"start": morning_start, "end": peak_end, "average": self.rate_export.get(morning_start, 0.0)} + # Calculate the precise export limit (Target SOC percentage) required to create this headroom + target_soc_kwh = max(0.0, self.soc_max - max_kwh_loss) + target_soc_pct = max(0.0, min(100.0, target_soc_kwh / self.soc_max * 100.0)) + target_soc_pct = float(int(target_soc_pct)) - # Do NOT delete fragmented windows. Allow the optimizer to evaluate our focused 3-hour window - # alongside any native high-rate export windows it already found. + new_window = { + "start": morning_start, + "end": peak_end, + "average": self.rate_export.get(morning_start, 0.0), + "clipping_ceiling_pct": target_soc_pct + } + + # Tag any overlapping native export windows with the ceiling so they don't dump below it + for w in self.export_window_best: + if (w.get("start") >= morning_start and w.get("start") < peak_end) or (w.get("end") > morning_start and w.get("end") <= peak_end): + w["clipping_ceiling_pct"] = target_soc_pct # Inject our new unified window if not any(w.get("start") <= new_window["start"] and w.get("end") >= new_window["end"] for w in self.export_window_best): self.export_window_best.append(new_window) + self.export_limits_best.append(target_soc_pct) if getattr(self, "high_export_rates", None) is not None: self.high_export_rates.append(copy.deepcopy(new_window)) - # Regenerate export limits - self.export_limits_best = [100.0] * len(self.export_window_best) - self.log("Injected continuous anti-clipping candidate export window {} to {} to allow spillover absorption".format(self.time_abs_str(morning_start), self.time_abs_str(peak_end))) + self.log("Injected continuous anti-clipping candidate export window {} to {} to allow spillover absorption (Target SOC: {}%)".format(self.time_abs_str(morning_start), self.time_abs_str(peak_end), target_soc_pct)) def calculate_plan(self, recompute=True, debug_mode=False, publish=True): """ @@ -1210,7 +1223,9 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): if kwh_loss > 0: self.clipping_buffer_forecast_kwh[minute] = kwh_loss # Add to totals - if minute < 1440: + minute_absolute = minute + self.minutes_now + midnight_today_minute = int(self.minutes_now / 1440) * 1440 + if minute_absolute < midnight_today_minute + 1440: self.clipping_remaining_today += kwh_loss else: self.clipping_tomorrow += kwh_loss @@ -1908,6 +1923,10 @@ def optimise_export(self, window_n, record_charge_windows, try_charge_limit, cha if self.set_export_low_power: loop_options.extend([0.3, 0.5, 0.7]) + # Ensure any pre-assigned fractional limits (like clipping thresholds) are evaluated + if try_export[window_n] not in loop_options and try_export[window_n] > 0.0 and try_export[window_n] < 99.0: + loop_options.append(try_export[window_n]) + # Collect all options results = [] results10 = [] @@ -4223,6 +4242,43 @@ def format_time_human(minute): clipping_status_text = "{} kWh clipping forecast ({}) between {} and {}.".format(dp2(self.clipping_buffer_kwh), self.clipping_mode, start_str, end_str) else: clipping_status_text = "{} kWh manual clipping buffer override active.".format(dp2(self.clipping_buffer_kwh)) + + elif getattr(self, "clipping_buffer_enable", False) and getattr(self, "clipping_buffer_forecast_kwh", {}): + def format_time_human_abs(minute_absolute): + target_dt = self.midnight + timedelta(minutes=minute_absolute) + return target_dt.strftime("%H:%M") + + today_start = None + today_end = None + tomorrow_start = None + tomorrow_end = None + + midnight_today_minute = int(self.minutes_now / 1440) * 1440 + for minute_relative, kwh_loss in sorted(self.clipping_buffer_forecast_kwh.items()): + minute_absolute = minute_relative + self.minutes_now + if minute_absolute < midnight_today_minute + 1440: + if today_start is None: today_start = minute_absolute + today_end = minute_absolute + 30 + else: + if tomorrow_start is None: tomorrow_start = minute_absolute + tomorrow_end = minute_absolute + 30 + + status_parts = [] + if self.clipping_remaining_today > 0 and today_start is not None: + status_parts.append("{} kWh clipping buffer remaining between {} and {} today".format( + dp2(self.clipping_remaining_today), + format_time_human_abs(today_start), + format_time_human_abs(today_end) + )) + if self.clipping_tomorrow > 0 and tomorrow_start is not None: + status_parts.append("{} kWh clipping buffer forecast between {} and {} tomorrow".format( + dp2(self.clipping_tomorrow), + format_time_human_abs(tomorrow_start), + format_time_human_abs(tomorrow_end) + )) + + if status_parts: + clipping_status_text = ". ".join(status_parts) + "." self.dashboard_item( self.prefix + ".clipping_status", From 381bb347bb7a331a797561a3324ad694838fcd5e Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 22 Jun 2026 07:54:03 +0100 Subject: [PATCH 48/74] clipping: Auto-detect ClearSky peaks and clean up clipping config --- apps/predbat/config.py | 8 -- apps/predbat/fetch.py | 1 - apps/predbat/plan.py | 52 +++++++---- apps/predbat/prediction.py | 5 +- apps/predbat/solcast.py | 130 ++++++++++++++++++++-------- apps/predbat/tests/test_clipping.py | 50 +++++++---- apps/predbat/tests/test_inverter.py | 1 + apps/predbat/tests/test_solcast.py | 77 ++++++++++++++++ apps/predbat/web.py | 4 +- 9 files changed, 250 insertions(+), 78 deletions(-) diff --git a/apps/predbat/config.py b/apps/predbat/config.py index c6361d0dd..314b85d02 100644 --- a/apps/predbat/config.py +++ b/apps/predbat/config.py @@ -503,14 +503,6 @@ "default": "auto", "enable": "clipping_buffer_enable", }, - { - "name": "clipping_use_clearsky_peaks", - "friendly_name": "Clipping Use ClearSky Peaks", - "type": "switch", - "default": True, - "icon": "mdi:weather-sunny-alert", - "enable": "clipping_buffer_enable", - }, { "name": "clipping_auto_tune", "friendly_name": "Clipping Auto-Tune", diff --git a/apps/predbat/fetch.py b/apps/predbat/fetch.py index 79064d137..94c134b3b 100644 --- a/apps/predbat/fetch.py +++ b/apps/predbat/fetch.py @@ -2365,7 +2365,6 @@ def fetch_config_options(self): # Clipping peak cost penalty model self.clipping_buffer_enable = self.get_arg("clipping_buffer_enable") - self.clipping_use_clearsky_peaks = self.get_arg("clipping_use_clearsky_peaks") self.clipping_auto_tune = self.get_arg("clipping_auto_tune") self.clipping_cost_weight = self.get_arg("clipping_cost_weight") self.clipping_amplification = self.get_arg("clipping_amplification") diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index 56cf552e1..348e7122a 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -887,7 +887,7 @@ def intersects(w, start, end): # We walk backwards from the peak minute by minute to account for the fact that # solar generation (PV) during the export window consumes inverter capacity and # reduces the effective battery discharge rate. - max_kwh_loss = max(kwh_loss for m, kwh_loss in forecast.items() if peak_start <= m <= peak_end) + max_kwh_loss = max(kwh_loss for m_rel, kwh_loss in forecast.items() if peak_start <= (m_rel + self.minutes_now) <= peak_end) # Cap the requested headroom by the physical capacity of the battery. # We cannot create more headroom than the battery can physically hold. @@ -895,8 +895,11 @@ def intersects(w, start, end): max_kwh_loss = min(max_kwh_loss, battery_capacity_kwh) hybrid = getattr(self, "inverter_hybrid", False) - inverter_limit_kw = getattr(self, "set_inverter_limit", 0) / 1000.0 - export_limit_kw = getattr(self, "export_limit", 0) + inverter_limit_kw = getattr(self, "inverter_limit", 0.0) * 60.0 + user_inverter_limit = getattr(self, "set_inverter_limit", 0) + if user_inverter_limit > 0: + inverter_limit_kw = user_inverter_limit / 1000.0 + export_limit_kw = getattr(self, "export_limit", 0.0) * 60.0 pv_forecast = getattr(self, "pv_forecast_minute_step", {}) accumulated_kwh = 0.0 @@ -904,10 +907,10 @@ def intersects(w, start, end): for m in range(peak_start, max(self.minutes_now, peak_start - 360), -1): # Start with raw battery maximum discharge - discharge_kw = self.battery_rate_max_discharge * getattr(self, "battery_rate_max_scaling_discharge", 1.0) + discharge_kw = self.battery_rate_max_discharge * getattr(self, "battery_rate_max_scaling_discharge", 1.0) * 60.0 - # Retrieve PV forecast for this minute (kW) - pv_kw = pv_forecast.get(m, 0.0) + # Retrieve PV forecast for this minute (kW). Note pv_forecast uses relative minutes. + pv_kw = pv_forecast.get(m - self.minutes_now, 0.0) # Hybrid inverters share the AC limit between battery and PV if hybrid and inverter_limit_kw > 0: @@ -923,7 +926,7 @@ def intersects(w, start, end): accumulated_kwh += discharge_kw * (1.0 / 60.0) minutes_needed += 1 - if accumulated_kwh >= max_kwh_loss: + if accumulated_kwh >= max_kwh_loss - 1e-9: break # Add 30 mins safety margin @@ -955,6 +958,16 @@ def intersects(w, start, end): if (w.get("start") >= morning_start and w.get("start") < peak_end) or (w.get("end") > morning_start and w.get("end") <= peak_end): w["clipping_ceiling_pct"] = target_soc_pct + # Remove any existing export windows that intersect our new window and keep export_limits_best aligned + new_export_windows = [] + new_export_limits = [] + for w, limit in zip(self.export_window_best, self.export_limits_best): + if not intersects(w, morning_start, peak_end): + new_export_windows.append(w) + new_export_limits.append(limit) + self.export_window_best = new_export_windows + self.export_limits_best = new_export_limits + # Inject our new unified window if not any(w.get("start") <= new_window["start"] and w.get("end") >= new_window["end"] for w in self.export_window_best): self.export_window_best.append(new_window) @@ -1160,9 +1173,12 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): effective_amplification = self.clipping_amplification # Apply ClearSky or Amplification factor - if getattr(self, "clipping_use_clearsky_peaks", False): - pv_clearsky_step = self.step_data_history(getattr(self, "pv_forecast_minuteCS", {}), self.minutes_now, forward=True, cloud_factor=None) - pv_forecast_peak_step = {k: max(v, pv_clearsky_step.get(k, 0) * effective_amplification) for k, v in pv_forecast_peak_step.items()} + # Check if ClearSky data is present and has non-zero values; otherwise fall back to amplified base peaks. + pv_clearsky_step = self.step_data_history(getattr(self, "pv_forecast_minuteCS", {}), self.minutes_now, forward=True, cloud_factor=None) + has_clearsky = any(val > 0.001 for val in pv_clearsky_step.values()) if pv_clearsky_step else False + + if has_clearsky: + pv_forecast_peak_step = {k: max(v, pv_clearsky_step.get(k, 0)) for k, v in pv_forecast_peak_step.items()} elif effective_amplification != 1.0: pv_forecast_peak_step = {k: v * effective_amplification for k, v in pv_forecast_peak_step.items()} @@ -1227,7 +1243,7 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): midnight_today_minute = int(self.minutes_now / 1440) * 1440 if minute_absolute < midnight_today_minute + 1440: self.clipping_remaining_today += kwh_loss - else: + elif minute_absolute < midnight_today_minute + 2880: self.clipping_tomorrow += kwh_loss # If manual override is active, reflect it in the UI totals @@ -4161,16 +4177,22 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi daily_cumulative_clip[day_index] = daily_cumulative_clip.get(day_index, 0.0) + (peak_pv - clipping_limit_step) remaining = daily_cumulative_clip.get(day_index, 0.0) + best_soc_min = getattr(self, "best_soc_min", 0.0) + max_headroom = self.soc_max - best_soc_min + capped_remaining = min(remaining, max_headroom) predict_clipping_remaining_best[minute] = round(remaining, 4) - predict_clipping_ceiling_best[minute] = round(self.soc_max - remaining, 4) + predict_clipping_ceiling_best[minute] = round(self.soc_max - capped_remaining, 4) # Sort dictionaries to ensure forwards time order for ApexCharts rendering self.predict_clipping_remaining_best = dict(sorted(predict_clipping_remaining_best.items())) self.predict_clipping_ceiling_best = dict(sorted(predict_clipping_ceiling_best.items())) - self.clipping_remaining_today = clipping_today - self.clipping_tomorrow = clipping_tomorrow - self.clipping_mitigated_today = clipping_today + if not self.clipping_buffer_enable: + self.clipping_remaining_today = clipping_today + self.clipping_tomorrow = clipping_tomorrow + self.clipping_mitigated_today = clipping_today + else: + self.clipping_mitigated_today = self.clipping_remaining_today # Add Clipping Summary Dashboard Items self.dashboard_item( diff --git a/apps/predbat/prediction.py b/apps/predbat/prediction.py index c0cb99018..cfcd01b69 100644 --- a/apps/predbat/prediction.py +++ b/apps/predbat/prediction.py @@ -714,10 +714,7 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi absorbable = min(battery_headroom, max_charge_step) unmitigated_clip = max(potential_clip - absorbable, 0) - if absorbable > 0: - # Actually route the absorbed free solar into the battery state of charge - # taking into account the battery's charging efficiency loss - soc += absorbable * battery_loss + if unmitigated_clip > 0: clipping_penalty = unmitigated_clip * export_rate * clipping_cost_weight diff --git a/apps/predbat/solcast.py b/apps/predbat/solcast.py index 7c6aa103b..69ba0ca32 100644 --- a/apps/predbat/solcast.py +++ b/apps/predbat/solcast.py @@ -753,7 +753,7 @@ def fetch_pv_datapoints(self, argname, entity_id): # check the total vs the sum of 30 minute slots and work out scale factor if data: for entry in data: - total_data += entry["pv_estimate"] + total_data += entry.get("pv_estimate", entry.get("pv_clearsky", 0.0)) total_data = dp2(total_data) total_sensor = self.get_state_wrapper(entity_id=entity_id, default=1.0) try: @@ -763,6 +763,57 @@ def fetch_pv_datapoints(self, argname, entity_id): return data, total_data, total_sensor + def overlay_clearsky_data(self, pv_forecast_data, source_data, value_field, source_period, source_divide_by, target_divide_by, target_period): + """ + Overlay clearsky data from source_data into pv_forecast_data, + correctly normalising units to kW and mapping between different time alignments/resolutions. + """ + timeline = [] + for item in source_data: + ts_str = item.get("period_start") + val = float(item.get(value_field, 0.0)) + try: + start_dt = datetime.strptime(ts_str, TIME_FORMAT) + start_dt = start_dt.replace(tzinfo=pytz.utc) + except (ValueError, TypeError): + continue + + # Calculate source power in kW + power_kw = (val / source_divide_by) * 60.0 + end_dt = start_dt + timedelta(minutes=source_period) + timeline.append((start_dt, end_dt, power_kw)) + + if not timeline: + return + + timeline.sort(key=lambda x: x[0]) + + for target_item in pv_forecast_data: + ts_str = target_item.get("period_start") + try: + target_start = datetime.strptime(ts_str, TIME_FORMAT) + target_start = target_start.replace(tzinfo=pytz.utc) + except (ValueError, TypeError): + continue + target_end = target_start + timedelta(minutes=target_period) + + # Find all overlapping source intervals + weighted_power_sum = 0.0 + total_overlap_mins = 0.0 + for src_start, src_end, src_power in timeline: + # Check for overlap + overlap_start = max(target_start, src_start) + overlap_end = min(target_end, src_end) + if overlap_start < overlap_end: + overlap_duration = (overlap_end - overlap_start).total_seconds() / 60.0 + weighted_power_sum += src_power * overlap_duration + total_overlap_mins += overlap_duration + + if total_overlap_mins > 0: + clearsky_kw = weighted_power_sum / total_overlap_mins + # Convert clearsky_kw to target's raw unit + target_item["pv_clearsky"] = (clearsky_kw / 60.0) * target_divide_by + def publish_pv_stats(self, pv_forecast_data, divide_by, period): """ Publish some PV stats @@ -1411,26 +1462,51 @@ async def fetch_pv_forecast(self): self.log("SolarAPI: PV Forecast today adds up to {} kWh, and total sensors add up to {} kWh, factor is {}".format(pv_forecast_total_data, pv_forecast_total_sensor, factor)) if pv_forecast_data: + # Detect the actual period of the forecast data (e.g. 15 or 30 minutes) + # by examining the time difference between consecutive entries. + # This ensures 15-minute resolution data is handled correctly. + period = 30 # Default period in minutes + if len(pv_forecast_data) >= 2: + try: + t0 = datetime.strptime(pv_forecast_data[0]["period_start"], TIME_FORMAT) + t1 = datetime.strptime(pv_forecast_data[1]["period_start"], TIME_FORMAT) + detected_period = int(abs((t1 - t0).total_seconds() / 60)) + # Sanity-check: only accept periods in the plausible range for forecast data. + # Values outside 5–60 minutes (e.g. 1440 if the first two entries span a day + # boundary when multiple sensor days are concatenated) are treated as invalid. + if 5 <= detected_period <= 60: + period = detected_period + except (ValueError, TypeError, KeyError): + pass + # Optional overlay of ClearSky data from a secondary source clipping_clearsky_source = self.get_arg("clipping_clearsky_source", "auto", indirect=False) if clipping_clearsky_source == "openmeteo" and self.open_meteo_forecast: self.log("SolarAPI: Overlaying ClearSky data from Open-Meteo API") om_data, _ = await self.download_open_meteo_data() if om_data: - om_lookup = {item["period_start"]: item.get("pv_clearsky", item.get("pv_estimate90", 0)) for item in om_data} - for item in pv_forecast_data: - ts = item.get("period_start") - if ts in om_lookup: - item["pv_clearsky"] = om_lookup[ts] + self.overlay_clearsky_data( + pv_forecast_data, + om_data, + value_field="pv_clearsky", + source_period=60, + source_divide_by=60.0, + target_divide_by=divide_by, + target_period=period + ) elif clipping_clearsky_source == "solcast_api" and self.solcast_api_key: self.log("SolarAPI: Overlaying ClearSky data from Solcast API") sol_data = await self.download_solcast_data() if sol_data: - sol_lookup = {item["period_start"]: item.get("pv_clearsky", item.get("pv_estimate90", 0)) for item in sol_data} - for item in pv_forecast_data: - ts = item.get("period_start") - if ts in sol_lookup: - item["pv_clearsky"] = sol_lookup[ts] + self.overlay_clearsky_data( + pv_forecast_data, + sol_data, + value_field="pv_clearsky", + source_period=30, + source_divide_by=30.0, + target_divide_by=divide_by, + target_period=period + ) elif clipping_clearsky_source == "ha_solcast_clearsky": self.log("SolarAPI: Overlaying ClearSky data from Home Assistant integration") ha_data = [] @@ -1441,34 +1517,20 @@ async def fetch_pv_forecast(self): if data: ha_data += data if ha_data: - # Apply factor and divide_by logic like the main PV fetcher - ha_lookup = {item["period_start"]: float(item.get("pv_estimate", 0.0)) for item in ha_data} - for item in pv_forecast_data: - ts = item.get("period_start") - if ts in ha_lookup: - item["pv_clearsky"] = ha_lookup[ts] + self.overlay_clearsky_data( + pv_forecast_data, + ha_data, + value_field="pv_clearsky", + source_period=period, + source_divide_by=divide_by, + target_divide_by=divide_by, + target_period=period + ) elif clipping_clearsky_source == "pv90_scaled": self.log("SolarAPI: Overlaying ClearSky data explicitly disabled, PV90 will be scaled and used as base for clipping mitigation") elif clipping_clearsky_source not in ["auto", "base", ""]: self.log("Warn: SolarAPI: clipping_clearsky_source '{}' not configured properly, using base data".format(clipping_clearsky_source)) - # Detect the actual period of the forecast data (e.g. 15 or 30 minutes) - # by examining the time difference between consecutive entries. - # This ensures 15-minute resolution data is handled correctly. - period = 30 # Default period in minutes - if len(pv_forecast_data) >= 2: - try: - t0 = datetime.strptime(pv_forecast_data[0]["period_start"], TIME_FORMAT) - t1 = datetime.strptime(pv_forecast_data[1]["period_start"], TIME_FORMAT) - detected_period = int(abs((t1 - t0).total_seconds() / 60)) - # Sanity-check: only accept periods in the plausible range for forecast data. - # Values outside 5–60 minutes (e.g. 1440 if the first two entries span a day - # boundary when multiple sensor days are concatenated) are treated as invalid. - if 5 <= detected_period <= 60: - period = detected_period - except (ValueError, TypeError, KeyError): - pass - # For the HA sensor path the divide_by was computed assuming 30-minute periods; # recalculate it using the actual detected period so that the per-minute kWh # values are correctly scaled regardless of the forecast resolution. diff --git a/apps/predbat/tests/test_clipping.py b/apps/predbat/tests/test_clipping.py index d42f9289d..d92b401c2 100644 --- a/apps/predbat/tests/test_clipping.py +++ b/apps/predbat/tests/test_clipping.py @@ -78,8 +78,8 @@ def test_inject_creates_contiguous_window(my_predbat): failed = False setup(my_predbat) my_predbat.minutes_now = 240 # 04:00 - # Peak from 13:00 to 14:00 (780 to 840) - my_predbat.clipping_buffer_forecast_kwh = {780: 1.0, 810: 2.0} + # Peak from 13:00 to 14:00 (780 to 840). Keys must be relative: 780-240=540, 810-240=570. + my_predbat.clipping_buffer_forecast_kwh = {540: 1.0, 570: 2.0} my_predbat.inject_clipping_export_windows() @@ -89,8 +89,12 @@ def test_inject_creates_contiguous_window(my_predbat): w = my_predbat.export_window_best[0] - if w["start"] != 240: - print("ERROR: Expected window start at 240, got {}".format(w["start"])) + # Correct calculation: peak at 780 to 840. + # To create 2.0 kWh headroom with 1.0 kW effective discharge rate requires 120 minutes of discharge. + # Plus 30 mins safety margin = 150 minutes. + # Start = max(240, 780 - 150) = 630. + if w["start"] != 630: + print("ERROR: Expected window start at 630, got {}".format(w["start"])) failed = True if w["end"] != 840: @@ -107,35 +111,51 @@ def test_inject_cleans_fragmented_windows(my_predbat): failed = False setup(my_predbat) my_predbat.minutes_now = 0 - # Peak from 780 to 810 + # Peak from 780 to 810 (absolute and relative are same since minutes_now=0) my_predbat.clipping_buffer_forecast_kwh = { 780: 1.0, } # Inject intersecting fragmented windows my_predbat.export_window_best = [ - {"start": 30, "end": 50, "average": 0}, # Before morning_start (60) - should KEEP - {"start": 100, "end": 120, "average": 0}, # Inside the new window - should DROP - {"start": 700, "end": 800, "average": 0}, # Intersecting the new window - should DROP - {"start": 900, "end": 960, "average": 0}, # After peak_end - should KEEP + {"start": 30, "end": 50, "average": 0}, # Before morning_start (690) - should KEEP + {"start": 100, "end": 120, "average": 0}, # Before morning_start (690) - should KEEP + {"start": 700, "end": 800, "average": 0}, # Intersecting the new window [690, 810] - should DROP + {"start": 900, "end": 960, "average": 0}, # After peak_end (810) - should KEEP ] + my_predbat.export_limits_best = [10.0, 20.0, 30.0, 40.0] my_predbat.inject_clipping_export_windows() - # We expect 3 windows: 2 kept + 1 injected - if len(my_predbat.export_window_best) != 3: - print("ERROR: Expected 3 windows (2 kept + 1 new), got {}".format(len(my_predbat.export_window_best))) + # We expect 4 windows: 3 kept + 1 newly injected + if len(my_predbat.export_window_best) != 4: + print("ERROR: Expected 4 windows (3 kept + 1 new), got {}".format(len(my_predbat.export_window_best))) return True + if len(my_predbat.export_limits_best) != len(my_predbat.export_window_best): + print("ERROR: Length mismatch! export_window_best is {}, export_limits_best is {}".format(len(my_predbat.export_window_best), len(my_predbat.export_limits_best))) + failed = True + starts = [w["start"] for w in my_predbat.export_window_best] - if 100 in starts or 700 in starts: + if 700 in starts: print("ERROR: Fragmented windows were not cleaned!") failed = True - if 60 not in starts: # Injected window start - print("ERROR: Injected window not found!") + if 100 not in starts or 30 not in starts: + print("ERROR: Non-intersecting windows were incorrectly dropped!") failed = True + if 690 not in starts: # Injected window start + print("ERROR: Injected window start 690 not found, got starts: {}".format(starts)) + failed = True + + # Check that limits are aligned: W0 (limit 10.0), W1 (limit 20.0), W3 (limit 40.0), and new window (target_soc_pct, e.g. 80.0) + expected_limits = [10.0, 20.0, 40.0] + for limit in expected_limits: + if limit not in my_predbat.export_limits_best: + print("ERROR: Expected limit {} not found in export_limits_best: {}".format(limit, my_predbat.export_limits_best)) + failed = True + if not failed: print("PASS") return failed diff --git a/apps/predbat/tests/test_inverter.py b/apps/predbat/tests/test_inverter.py index f250abc20..739e4c03c 100644 --- a/apps/predbat/tests/test_inverter.py +++ b/apps/predbat/tests/test_inverter.py @@ -681,6 +681,7 @@ def test_inverter_self_test(test_name, my_predbat): # Call self test - doesn't really check much as such except the code isn't dead dummy_rest = DummyRestAPI() my_predbat.args["givtcp_rest"] = "dummy" + my_predbat.args["inverter_reserve_max"] = 100 dummy_rest.rest_data = {} dummy_rest.rest_data["Control"] = {} diff --git a/apps/predbat/tests/test_solcast.py b/apps/predbat/tests/test_solcast.py index a05902e0f..1512c665b 100644 --- a/apps/predbat/tests/test_solcast.py +++ b/apps/predbat/tests/test_solcast.py @@ -1889,6 +1889,82 @@ def create_mock_session(*args, **kwargs): return failed +def test_fetch_pv_forecast_ha_clearsky(my_predbat): + """ + Integration test: fetch_pv_forecast using HA sensors for clearsky forecast. + Ensures that: + - Regex pattern resolves sensor names (e.g. sensor.solcast_clear_sky_today). + - fetch_pv_datapoints does not raise KeyError for pv_clearsky key. + - overlay_clearsky_data correctly overlays pv_clearsky values. + """ + print(" - test_fetch_pv_forecast_ha_clearsky") + failed = False + + test_api = create_test_solar_api() + try: + # Configure matching options + test_api.solar.solcast_host = None + test_api.solar.solcast_api_key = None + test_api.solar.forecast_solar = None + test_api.solar.pv_forecast_today = "sensor.solcast_pv_forecast_today" + test_api.solar.pv_forecast_tomorrow = None + + # Enable clearsky source as HA + test_api.mock_base.set_arg("clipping_clearsky_source", "ha_solcast_clearsky") + test_api.solar.pv_clearsky_today = "sensor.solcast_clear_sky_today" + test_api.solar.pv_clearsky_tomorrow = None + test_api.solar.pv_clearsky_d3 = None + test_api.solar.pv_clearsky_d4 = None + + # Setup mock forecast sensor (base) + test_api.set_mock_ha_state( + "sensor.solcast_pv_forecast_today", + { + "state": "2.0", + "detailedForecast": [ + {"period_start": "2025-06-15T12:00:00+0000", "pv_estimate": 1.0}, + {"period_start": "2025-06-15T12:30:00+0000", "pv_estimate": 1.0}, + ], + }, + ) + + # Setup mock clearsky sensor + test_api.set_mock_ha_state( + "sensor.solcast_clear_sky_today", + { + "state": "3.0", + "detailedForecast": [ + {"period_start": "2025-06-15T12:00:00+0000", "pv_clearsky": 1.5}, + {"period_start": "2025-06-15T12:30:00+0000", "pv_clearsky": 1.5}, + ], + }, + ) + + def create_mock_session(*args, **kwargs): + return test_api.mock_aiohttp_session() + + with patch("solcast.aiohttp.ClientSession", side_effect=create_mock_session): + run_async(test_api.solar.fetch_pv_forecast()) + + # Verify that clearsky was overlayed correctly and published + today_entity = f"sensor.{test_api.mock_base.prefix}_pv_today" + if today_entity not in test_api.dashboard_items: + print("ERROR: Expected pv_today entity to be published") + failed = True + else: + today_item = test_api.dashboard_items[today_entity] + total_cs = today_item["attributes"].get("totalCS", 0) + expected_total_cs = 3.0 + if abs(total_cs - expected_total_cs) > 0.05: + print(f"ERROR: Expected totalCS ~{expected_total_cs} kWh, got {total_cs} kWh") + failed = True + + finally: + test_api.cleanup() + + return failed + + # ============================================================================ # 15-minute resolution tests # ============================================================================ @@ -3454,6 +3530,7 @@ def run_solcast_tests(my_predbat): failed |= test_fetch_pv_forecast_forecast_solar_open_meteo_backup_on_failure(my_predbat) failed |= test_fetch_pv_forecast_forecast_solar_open_meteo_backup_not_used_on_success(my_predbat) failed |= test_fetch_pv_forecast_ha_sensors(my_predbat) + failed |= test_fetch_pv_forecast_ha_clearsky(my_predbat) # 15-minute resolution tests failed |= test_fetch_pv_forecast_ha_sensors_15min_kwh(my_predbat) diff --git a/apps/predbat/web.py b/apps/predbat/web.py index 911eaa9bb..48efefd8c 100644 --- a/apps/predbat/web.py +++ b/apps/predbat/web.py @@ -3120,7 +3120,9 @@ def get_chart(self, chart): xaxis_annotations.append({"x": end_stamp, "borderColor": "#ffa500", "strokeDashArray": 4, "text": "Today Buffer End", "orientation": "vertical", "backgroundColor": "#ffa500"}) clipping_total = 0 - if getattr(self.base, "predict_clipped_best", None): + if getattr(self.base, "clipping_buffer_enable", False): + clipping_total = getattr(self.base, "clipping_remaining_today", 0.0) + getattr(self.base, "clipping_tomorrow", 0.0) + elif getattr(self.base, "predict_clipped_best", None): clipping_total = self.base.predict_clipped_best.get(max(self.base.predict_clipped_best.keys()), 0.0) chart_title = "Clipping Analysis (Expected Total Clipping: {:.2f} kWh)".format(clipping_total) From ada1d5de1ac5c99e4f724fac7d98c007379a029b Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 22 Jun 2026 10:43:01 +0100 Subject: [PATCH 49/74] feat(clipping): simplify configuration flow, add offsets auto-tuning, and direct dashboard sync --- apps/predbat/config.py | 26 +++++ apps/predbat/fetch.py | 2 + apps/predbat/plan.py | 78 +++++++++++++-- apps/predbat/tests/test_clipping.py | 150 ++++++++++++++++++++++++++++ docs/customisation.md | 11 +- 5 files changed, 254 insertions(+), 13 deletions(-) diff --git a/apps/predbat/config.py b/apps/predbat/config.py index 314b85d02..21bafd085 100644 --- a/apps/predbat/config.py +++ b/apps/predbat/config.py @@ -535,6 +535,30 @@ "enable": "clipping_buffer_enable", "default": 1.0, }, + { + "name": "clipping_buffer_start_offset", + "friendly_name": "Clipping Buffer Start Offset", + "type": "input_number", + "min": 0, + "max": 120, + "step": 5, + "unit": "minutes", + "icon": "mdi:clock-start", + "default": 0, + "enable": "clipping_buffer_enable", + }, + { + "name": "clipping_buffer_end_offset", + "friendly_name": "Clipping Buffer End Offset", + "type": "input_number", + "min": 0, + "max": 120, + "step": 5, + "unit": "minutes", + "icon": "mdi:clock-end", + "default": 0, + "enable": "clipping_buffer_enable", + }, { "name": "clipping_limit_override", "friendly_name": "Clipping Limit Override", @@ -2328,6 +2352,8 @@ "clipping_use_clearsky_peaks": {"type": "boolean"}, "clipping_cost_weight": {"type": "float"}, "clipping_amplification": {"type": "float"}, + "clipping_buffer_start_offset": {"type": "integer"}, + "clipping_buffer_end_offset": {"type": "integer"}, "clipping_limit_override": {"type": "integer"}, "clipping_buffer_max_kwh": {"type": "float"}, "clipping_buffer_start_time": {"type": "string"}, diff --git a/apps/predbat/fetch.py b/apps/predbat/fetch.py index 94c134b3b..7654c01c0 100644 --- a/apps/predbat/fetch.py +++ b/apps/predbat/fetch.py @@ -2368,6 +2368,8 @@ def fetch_config_options(self): self.clipping_auto_tune = self.get_arg("clipping_auto_tune") self.clipping_cost_weight = self.get_arg("clipping_cost_weight") self.clipping_amplification = self.get_arg("clipping_amplification") + self.clipping_buffer_start_offset = self.get_arg("clipping_buffer_start_offset", default=0) + self.clipping_buffer_end_offset = self.get_arg("clipping_buffer_end_offset", default=0) self.clipping_limit_override = self.get_arg("clipping_limit_override") / MINUTE_WATT if self.get_arg("clipping_limit_override") else 0 # iBoost solar diverter model diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index 348e7122a..07d7ceb85 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -1112,17 +1112,23 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): if not hasattr(self, "clipping_auto_amp"): self.clipping_auto_amp = getattr(self, "clipping_amplification", 1.0) + self.clipping_auto_start_offset = getattr(self, "clipping_buffer_start_offset", 0) + self.clipping_auto_end_offset = getattr(self, "clipping_buffer_end_offset", 0) self.clipping_last_tune_day = None if os.path.exists(auto_tune_file): try: with open(auto_tune_file, "r") as f: data = json.load(f) self.clipping_auto_amp = data.get("auto_amp", self.clipping_auto_amp) + self.clipping_auto_start_offset = data.get("auto_start_offset", self.clipping_auto_start_offset) + self.clipping_auto_end_offset = data.get("auto_end_offset", self.clipping_auto_end_offset) self.clipping_last_tune_day = data.get("last_tune_day", None) except Exception: pass auto_amp = self.clipping_auto_amp + auto_start_offset = self.clipping_auto_start_offset + auto_end_offset = self.clipping_auto_end_offset last_tune_day = self.clipping_last_tune_day # Check if we should tune today (only once per day) @@ -1151,26 +1157,62 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): if limit > 0: if max_pv_power >= limit * 0.98: auto_amp = min(2.5, auto_amp + 0.05) - self.log("Clipping auto-tuner: Real clipping detected (max PV {} kW >= limit {} kW). Increased recommended auto_amp to {}".format(dp2(max_pv_power), dp2(limit), auto_amp)) + auto_start_offset = min(60, auto_start_offset + 5) + auto_end_offset = min(60, auto_end_offset + 5) + self.log("Clipping auto-tuner: Real clipping detected (max PV {} kW >= limit {} kW). Increased safety margins - amp: {}, start_offset: {}m, end_offset: {}m".format(dp2(max_pv_power), dp2(limit), auto_amp, auto_start_offset, auto_end_offset)) else: auto_amp = max(1.0, auto_amp - 0.01) - self.log("Clipping auto-tuner: No clipping detected. Decreased recommended auto_amp to {}".format(auto_amp)) + auto_start_offset = max(0, auto_start_offset - 5) + auto_end_offset = max(0, auto_end_offset - 5) + self.log("Clipping auto-tuner: No clipping detected. Decreased safety margins - amp: {}, start_offset: {}m, end_offset: {}m".format(auto_amp, auto_start_offset, auto_end_offset)) try: with open(auto_tune_file, "w") as f: - json.dump({"auto_amp": auto_amp, "last_tune_day": current_day}, f) + json.dump({ + "auto_amp": auto_amp, + "auto_start_offset": auto_start_offset, + "auto_end_offset": auto_end_offset, + "last_tune_day": current_day + }, f) self.clipping_auto_amp = auto_amp + self.clipping_auto_start_offset = auto_start_offset + self.clipping_auto_end_offset = auto_end_offset self.clipping_last_tune_day = current_day except Exception: pass - self.dashboard_item(self.prefix + ".clipping_recommended_amplification", state=dp2(auto_amp), attributes={"friendly_name": "Clipping Recommended Amplification", "icon": "mdi:auto-fix"}) - - # Apply Manual vs Auto Factor + # If auto-tune is enabled, sync clipping configuration to the recommended values if getattr(self, "clipping_auto_tune", False): - effective_amplification = auto_amp - else: - effective_amplification = self.clipping_amplification + config_changed = False + + current_amp = getattr(self, "clipping_amplification", 1.0) + if current_amp is None or abs(current_amp - auto_amp) > 1e-4: + self.clipping_amplification = auto_amp + if publish: + self.expose_config("clipping_amplification", auto_amp) + config_changed = True + + current_start = getattr(self, "clipping_buffer_start_offset", 0) + if current_start is None or current_start != auto_start_offset: + self.clipping_buffer_start_offset = auto_start_offset + if publish: + self.expose_config("clipping_buffer_start_offset", auto_start_offset) + config_changed = True + + current_end = getattr(self, "clipping_buffer_end_offset", 0) + if current_end is None or current_end != auto_end_offset: + self.clipping_buffer_end_offset = auto_end_offset + if publish: + self.expose_config("clipping_buffer_end_offset", auto_end_offset) + config_changed = True + + if config_changed and publish: + self.save_current_config() + self.log("Clipping auto-tuner: Synced entities to HA and predbat_config.json") + + effective_amplification = getattr(self, "clipping_amplification", 1.0) + if effective_amplification is None: + effective_amplification = 1.0 # Apply ClearSky or Amplification factor # Check if ClearSky data is present and has non-zero values; otherwise fall back to amplified base peaks. @@ -1182,6 +1224,24 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): elif effective_amplification != 1.0: pv_forecast_peak_step = {k: v * effective_amplification for k, v in pv_forecast_peak_step.items()} + # Apply start/end offsets to widen the clipping risk window + start_offset = int(getattr(self, "clipping_buffer_start_offset", 0)) + end_offset = int(getattr(self, "clipping_buffer_end_offset", 0)) + if (start_offset > 0 or end_offset > 0) and pv_forecast_peak_step: + widened_peak_step = {} + for k, v in pv_forecast_peak_step.items(): + m_min = k - end_offset + m_max = k + start_offset + max_val = v + m_start = 5 * (m_min // 5) + m_end = 5 * ((m_max + 4) // 5) + for m in range(m_start, m_end + 1, 5): + val = pv_forecast_peak_step.get(m, 0.0) + if val > max_val: + max_val = val + widened_peak_step[k] = max_val + pv_forecast_peak_step = widened_peak_step + # Calculate effective clipping limit: most restrictive hardware constraint if self.clipping_limit_override > 0: clipping_limit_effective = self.clipping_limit_override diff --git a/apps/predbat/tests/test_clipping.py b/apps/predbat/tests/test_clipping.py index d92b401c2..9e0a8a3d1 100644 --- a/apps/predbat/tests/test_clipping.py +++ b/apps/predbat/tests/test_clipping.py @@ -19,6 +19,8 @@ def run_clipping_tests(my_predbat): failed |= test_inject_aborts_if_empty_forecast(my_predbat) failed |= test_inject_creates_contiguous_window(my_predbat) failed |= test_inject_cleans_fragmented_windows(my_predbat) + failed |= test_clipping_buffer_offsets(my_predbat) + failed |= test_clipping_auto_tune_sync(my_predbat) return failed @@ -26,6 +28,9 @@ def setup(my_predbat): reset_inverter(my_predbat) my_predbat.clipping_buffer_enable = True my_predbat.clipping_buffer_forecast_kwh = {} + my_predbat.clipping_buffer_start_offset = 0 + my_predbat.clipping_buffer_end_offset = 0 + my_predbat.clipping_amplification = 1.0 my_predbat.minutes_now = 0 my_predbat.export_rate = {} my_predbat.export_window_best = [] @@ -159,3 +164,148 @@ def test_inject_cleans_fragmented_windows(my_predbat): if not failed: print("PASS") return failed + + +def test_clipping_buffer_offsets(my_predbat): + print("**** test_clipping_buffer_offsets ****") + failed = False + setup(my_predbat) + + # We want to test the peak PV forecast widening logic directly. + my_predbat.pv_forecast_peak_step = {m: 0.0 for m in range(0, 125, 5)} # pre-fill all 5-minute step keys + my_predbat.pv_forecast_peak_step[60] = 10.0 # peak at minute 60 + + # Set offsets + my_predbat.clipping_buffer_start_offset = 15 + my_predbat.clipping_buffer_end_offset = 15 + + # Run the widening logic: + pv_forecast_peak_step = my_predbat.pv_forecast_peak_step + start_offset = int(getattr(my_predbat, "clipping_buffer_start_offset", 0)) + end_offset = int(getattr(my_predbat, "clipping_buffer_end_offset", 0)) + + if (start_offset > 0 or end_offset > 0) and pv_forecast_peak_step: + widened_peak_step = {} + for k, v in pv_forecast_peak_step.items(): + m_min = k - end_offset + m_max = k + start_offset + max_val = v + m_start = 5 * (m_min // 5) + m_end = 5 * ((m_max + 4) // 5) + for m in range(m_start, m_end + 1, 5): + val = pv_forecast_peak_step.get(m, 0.0) + if val > max_val: + max_val = val + widened_peak_step[k] = max_val + pv_forecast_peak_step = widened_peak_step + + # Verify that minutes 45, 50, 55, 60, 65, 70, 75 all have the peak value 10.0 + expected_minutes = [45, 50, 55, 60, 65, 70, 75] + for m in expected_minutes: + val = pv_forecast_peak_step.get(m, 0.0) + if val != 10.0: + print("ERROR: Expected widened peak value 10.0 at minute {}, got {}".format(m, val)) + failed = True + + # Verify that minutes outside range do not have the peak value (e.g. 40, 80) + for m in [40, 80]: + val = pv_forecast_peak_step.get(m, 0.0) + if val == 10.0: + print("ERROR: Widened peak overflowed to minute {}, got {}".format(m, val)) + failed = True + + if not failed: + print("PASS") + return failed + + +def test_clipping_auto_tune_sync(my_predbat): + print("**** test_clipping_auto_tune_sync ****") + failed = False + setup(my_predbat) + + # Save original functions to restore them later + original_expose_config = my_predbat.expose_config + original_save_current_config = my_predbat.save_current_config + + # Mock expose_config and save_current_config + exposed_calls = [] + def mock_expose_config(name, value, *args, **kwargs): + exposed_calls.append((name, value)) + + saved_calls = 0 + def mock_save_current_config(*args, **kwargs): + nonlocal saved_calls + saved_calls += 1 + + my_predbat.expose_config = mock_expose_config + my_predbat.save_current_config = mock_save_current_config + + try: + # Configure auto_tune to True + my_predbat.clipping_auto_tune = True + + # Sync logic + auto_amp = 1.35 + auto_start_offset = 15 + auto_end_offset = 15 + + # Perform sync + config_changed = False + + current_amp = getattr(my_predbat, "clipping_amplification", 1.0) + if current_amp is None or abs(current_amp - auto_amp) > 1e-4: + my_predbat.clipping_amplification = auto_amp + my_predbat.expose_config("clipping_amplification", auto_amp) + config_changed = True + + current_start = getattr(my_predbat, "clipping_buffer_start_offset", 0) + if current_start is None or current_start != auto_start_offset: + my_predbat.clipping_buffer_start_offset = auto_start_offset + my_predbat.expose_config("clipping_buffer_start_offset", auto_start_offset) + config_changed = True + + current_end = getattr(my_predbat, "clipping_buffer_end_offset", 0) + if current_end is None or current_end != auto_end_offset: + my_predbat.clipping_buffer_end_offset = auto_end_offset + my_predbat.expose_config("clipping_buffer_end_offset", auto_end_offset) + config_changed = True + + if config_changed: + my_predbat.save_current_config() + + if my_predbat.clipping_amplification != 1.35: + print("ERROR: clipping_amplification was not updated to 1.35, got {}".format(my_predbat.clipping_amplification)) + failed = True + + if my_predbat.clipping_buffer_start_offset != 15: + print("ERROR: clipping_buffer_start_offset was not updated to 15, got {}".format(my_predbat.clipping_buffer_start_offset)) + failed = True + + if my_predbat.clipping_buffer_end_offset != 15: + print("ERROR: clipping_buffer_end_offset was not updated to 15, got {}".format(my_predbat.clipping_buffer_end_offset)) + failed = True + + if ("clipping_amplification", 1.35) not in exposed_calls: + print("ERROR: expose_config was not called for clipping_amplification") + failed = True + + if ("clipping_buffer_start_offset", 15) not in exposed_calls: + print("ERROR: expose_config was not called for clipping_buffer_start_offset") + failed = True + + if ("clipping_buffer_end_offset", 15) not in exposed_calls: + print("ERROR: expose_config was not called for clipping_buffer_end_offset") + failed = True + + if saved_calls != 1: + print("ERROR: save_current_config was not called once, got: {}".format(saved_calls)) + failed = True + finally: + # RESTORE the original functions to avoid contaminating subsequent tests + my_predbat.expose_config = original_expose_config + my_predbat.save_current_config = original_save_current_config + + if not failed: + print("PASS") + return failed diff --git a/docs/customisation.md b/docs/customisation.md index e419e31df..33ee0ec1f 100644 --- a/docs/customisation.md +++ b/docs/customisation.md @@ -477,10 +477,13 @@ The **Clipping Buffer** feature introduces an intelligent mitigation layer that These settings can be found in the Home Assistant Predbat Configuration panel, directly beneath the Cloud Model settings. - **`clipping_buffer_enable`**: The master switch. Turns the entire clipping buffer feature on or off. -- **`clipping_clearsky_source`**: Determines where Predbat gets its clear-sky (theoretical maximum) data. Options include `auto`, `ha_solcast_clearsky`, `solcast_api`, and `openmeteo`. This piggybacks on your existing Predbat/HA API integrations, requiring no extra YAML configuration. -- **`clipping_use_clearsky_peaks`**: If enabled, Predbat uses the selected Clear-Sky source to size the buffer, rather than just the standard forecast. This is highly recommended to protect against cloud-edge spikes. -- **`clipping_auto_tune`**: Automatically learns the scaling difference between your standard solar forecast and your hardware limits based on past clipping behavior, saving a tracking multiplier to `clipping_auto_tune.json`. -- **`clipping_amplification`**: A manual multiplier (e.g., `1.5x`) to force the standard solar forecast higher to simulate a sunny spike. This is only utilized if Auto-Tune is disabled. +- **`clipping_clearsky_source`**: Determines where Predbat gets its clear-sky (theoretical maximum) data. Options include: + * `auto`: Automatically select the best available source from the options below. + * `ha_solcast_clearsky`: Integrates with the HACS [Solcast Solar integration](https://github.com/BJReplay/ha-solcast-solar) (requires separate installation, but uses existing sensors). + * `solcast_api`: Direct native Solcast API integration (only works if your Solcast API key has clear-sky access). + * `openmeteo`: Uses the free Open-Meteo API as a fallback source. +- **`clipping_auto_tune`**: Automatically learns the safety buffer margins from past clipping behavior. When enabled, it dynamically tunes and auto-populates `clipping_amplification`, `clipping_buffer_start_offset`, and `clipping_buffer_end_offset` directly on your dashboard entities. +- **`clipping_amplification`**: A manual multiplier (e.g. `1.5x`) to scale the modeled solar forecast peak. Tuned dynamically if Auto-Tune is ON. - **`clipping_cost_weight`**: An internal optimizer multiplier (default `1.0`). If Predbat isn't dumping the battery aggressively enough before a peak, increasing this number adds a harsher financial penalty for clipping, forcing the optimizer to prioritize creating headroom. - **`clipping_limit_override`**: Manually define your inverter's AC ceiling in Watts (e.g., `5000W`). If left at `0`, Predbat auto-detects it from your inverter entity. From 6bb6caa037c39ed95601de08fbc0cba61cc99eb5 Mon Sep 17 00:00:00 2001 From: Robin Date: Sat, 27 Jun 2026 14:32:29 +0100 Subject: [PATCH 50/74] Optimize early morning export window dropping and apply code quality formatting --- apps/predbat/plan.py | 128 ++++++++++++++------------ apps/predbat/prediction.py | 3 +- apps/predbat/solcast.py | 36 ++------ apps/predbat/tests/test_clipping.py | 133 ++++++++++++++++++++++++++++ apps/predbat/tests/test_solcast.py | 6 +- apps/predbat/web.py | 26 ++++-- docs/customisation.md | 8 +- 7 files changed, 238 insertions(+), 102 deletions(-) diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index 07d7ceb85..092f922ea 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -888,12 +888,12 @@ def intersects(w, start, end): # solar generation (PV) during the export window consumes inverter capacity and # reduces the effective battery discharge rate. max_kwh_loss = max(kwh_loss for m_rel, kwh_loss in forecast.items() if peak_start <= (m_rel + self.minutes_now) <= peak_end) - + # Cap the requested headroom by the physical capacity of the battery. # We cannot create more headroom than the battery can physically hold. battery_capacity_kwh = self.soc_max - getattr(self, "best_soc_min", 0.0) max_kwh_loss = min(max_kwh_loss, battery_capacity_kwh) - + hybrid = getattr(self, "inverter_hybrid", False) inverter_limit_kw = getattr(self, "inverter_limit", 0.0) * 60.0 user_inverter_limit = getattr(self, "set_inverter_limit", 0) @@ -901,40 +901,40 @@ def intersects(w, start, end): inverter_limit_kw = user_inverter_limit / 1000.0 export_limit_kw = getattr(self, "export_limit", 0.0) * 60.0 pv_forecast = getattr(self, "pv_forecast_minute_step", {}) - + accumulated_kwh = 0.0 minutes_needed = 0 - + for m in range(peak_start, max(self.minutes_now, peak_start - 360), -1): # Start with raw battery maximum discharge discharge_kw = self.battery_rate_max_discharge * getattr(self, "battery_rate_max_scaling_discharge", 1.0) * 60.0 - + # Retrieve PV forecast for this minute (kW). Note pv_forecast uses relative minutes. pv_kw = pv_forecast.get(m - self.minutes_now, 0.0) - + # Hybrid inverters share the AC limit between battery and PV if hybrid and inverter_limit_kw > 0: discharge_kw = min(discharge_kw, max(0.0, inverter_limit_kw - pv_kw)) - + # The total site export limit also restricts combined PV + Battery export if export_limit_kw > 0: discharge_kw = min(discharge_kw, max(0.0, export_limit_kw - pv_kw)) - + # Safety minimum to prevent infinite loops if limits are misconfigured (0.5 kW) discharge_kw = max(0.5, discharge_kw) - + accumulated_kwh += discharge_kw * (1.0 / 60.0) minutes_needed += 1 - + if accumulated_kwh >= max_kwh_loss - 1e-9: break # Add 30 mins safety margin minutes_needed += 30 - + # Clamp the window between 60 minutes and 360 minutes (6 hours) minutes_needed = max(60, min(360, minutes_needed)) - + morning_start = max(self.minutes_now, peak_start - minutes_needed) morning_start = int(morning_start / 30) * 30 # Align to nearest 30 mins @@ -946,17 +946,12 @@ def intersects(w, start, end): target_soc_pct = max(0.0, min(100.0, target_soc_kwh / self.soc_max * 100.0)) target_soc_pct = float(int(target_soc_pct)) - new_window = { - "start": morning_start, - "end": peak_end, - "average": self.rate_export.get(morning_start, 0.0), - "clipping_ceiling_pct": target_soc_pct - } + new_window = {"start": morning_start, "end": peak_end, "average": self.rate_export.get(morning_start, 0.0), "clipping_target_soc_pct": target_soc_pct} - # Tag any overlapping native export windows with the ceiling so they don't dump below it + # Tag any overlapping native export windows with the target SOC so they don't dump below it for w in self.export_window_best: if (w.get("start") >= morning_start and w.get("start") < peak_end) or (w.get("end") > morning_start and w.get("end") <= peak_end): - w["clipping_ceiling_pct"] = target_soc_pct + w["clipping_target_soc_pct"] = target_soc_pct # Remove any existing export windows that intersect our new window and keep export_limits_best aligned new_export_windows = [] @@ -1159,7 +1154,11 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): auto_amp = min(2.5, auto_amp + 0.05) auto_start_offset = min(60, auto_start_offset + 5) auto_end_offset = min(60, auto_end_offset + 5) - self.log("Clipping auto-tuner: Real clipping detected (max PV {} kW >= limit {} kW). Increased safety margins - amp: {}, start_offset: {}m, end_offset: {}m".format(dp2(max_pv_power), dp2(limit), auto_amp, auto_start_offset, auto_end_offset)) + self.log( + "Clipping auto-tuner: Real clipping detected (max PV {} kW >= limit {} kW). Increased safety margins - amp: {}, start_offset: {}m, end_offset: {}m".format( + dp2(max_pv_power), dp2(limit), auto_amp, auto_start_offset, auto_end_offset + ) + ) else: auto_amp = max(1.0, auto_amp - 0.01) auto_start_offset = max(0, auto_start_offset - 5) @@ -1168,12 +1167,7 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): try: with open(auto_tune_file, "w") as f: - json.dump({ - "auto_amp": auto_amp, - "auto_start_offset": auto_start_offset, - "auto_end_offset": auto_end_offset, - "last_tune_day": current_day - }, f) + json.dump({"auto_amp": auto_amp, "auto_start_offset": auto_start_offset, "auto_end_offset": auto_end_offset, "last_tune_day": current_day}, f) self.clipping_auto_amp = auto_amp self.clipping_auto_start_offset = auto_start_offset self.clipping_auto_end_offset = auto_end_offset @@ -1294,7 +1288,7 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): # clipping_limit_effective is in kW. Convert it to max allowed energy in PREDICT_STEP. max_kwh_allowed = clipping_limit_effective * (PREDICT_STEP / 60.0) if step_kwh > max_kwh_allowed: - kwh_loss += (step_kwh - max_kwh_allowed) + kwh_loss += step_kwh - max_kwh_allowed if kwh_loss > 0: self.clipping_buffer_forecast_kwh[minute] = kwh_loss @@ -2781,7 +2775,7 @@ def optimise_swap_export(self, record_charge_windows, record_export_windows, dro swapped_target = {} curr = self.currency_symbols[1] - if self.calculate_best_export and record_export_windows >= 2: + if self.calculate_best_export and record_export_windows >= 1: swapped = True while swapped: selected_metric, selected_battery_value, selected_cost, selected_keep, selected_cycle, selected_carbon, selected_import, select_export = self.run_prediction_metric( @@ -2792,7 +2786,7 @@ def optimise_swap_export(self, record_charge_windows, record_export_windows, dro ) swapped = False - for window_n_target in range(record_export_windows - 1, 0, -1): + for window_n_target in range(record_export_windows - 1, -1, -1): previous_end_target = 0 if window_n_target > 0: previous_end_target = self.export_window_best[window_n_target - 1]["end"] @@ -3595,7 +3589,7 @@ def optimise_all_windows(self, best_metric, metric_keep, debug_mode=False): debug_mode=debug_mode, ) # Swaps - self.optimise_swap_export(record_charge_windows, record_export_windows, debug_mode=debug_mode) + self.optimise_swap_export(record_charge_windows, record_export_windows, drop=True, debug_mode=debug_mode) self.plan_write_debug(debug_mode, "plan_swap_final.html", self.pv_forecast_minute_step, self.pv_forecast_minute10_step, self.load_minutes_step, self.load_minutes_step10, self.end_record) # Second pass optimisation @@ -4193,9 +4187,9 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi clipping_total = self.predict_clipped_best.get(keys[-1], 0.0) clipping_tomorrow = max(0.0, clipping_total - clipping_today) - # Generate Clipping Visual Series (Remaining and Ceiling) for web.py charts + # Generate Clipping Visual Series (Remaining and Target SOC) for web.py charts predict_clipping_remaining_best = {} - predict_clipping_ceiling_best = {} + predict_clipping_target_soc_best = {} clipping_limit_step = getattr(self, "clipping_limit_effective", 0) * (step / 60.0) # Ensure the chart uses the exact same peak dataset as the planning engine @@ -4216,7 +4210,7 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi for minute in range(max_minute, -step, -step): remaining = 0.0 minute_absolute = minute + self.minutes_now - + if minute <= midnight_today_minute: day_index = 0 elif minute <= midnight_today_minute + 24 * 60: @@ -4241,11 +4235,11 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi max_headroom = self.soc_max - best_soc_min capped_remaining = min(remaining, max_headroom) predict_clipping_remaining_best[minute] = round(remaining, 4) - predict_clipping_ceiling_best[minute] = round(self.soc_max - capped_remaining, 4) + predict_clipping_target_soc_best[minute] = round(self.soc_max - capped_remaining, 4) # Sort dictionaries to ensure forwards time order for ApexCharts rendering self.predict_clipping_remaining_best = dict(sorted(predict_clipping_remaining_best.items())) - self.predict_clipping_ceiling_best = dict(sorted(predict_clipping_ceiling_best.items())) + self.predict_clipping_target_soc_best = dict(sorted(predict_clipping_target_soc_best.items())) if not self.clipping_buffer_enable: self.clipping_remaining_today = clipping_today @@ -4267,11 +4261,11 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi }, ) self.dashboard_item( - self.prefix + ".clipping_ceiling", - state=dp2(self.predict_clipping_ceiling_best.get(0, self.soc_max)), + self.prefix + ".clipping_target_soc", + state=dp2(self.predict_clipping_target_soc_best.get(0, self.soc_max)), attributes={ - "results": self.filtered_times(self.predict_clipping_ceiling_best), - "friendly_name": "Clipping Ceiling", + "results": self.filtered_times(self.predict_clipping_target_soc_best), + "friendly_name": "Clipping Target SOC", "unit_of_measurement": "kWh", "device_class": "energy", "icon": "mdi:arrow-collapse-up", @@ -4324,8 +4318,9 @@ def format_time_human(minute): clipping_status_text = "{} kWh clipping forecast ({}) between {} and {}.".format(dp2(self.clipping_buffer_kwh), self.clipping_mode, start_str, end_str) else: clipping_status_text = "{} kWh manual clipping buffer override active.".format(dp2(self.clipping_buffer_kwh)) - + elif getattr(self, "clipping_buffer_enable", False) and getattr(self, "clipping_buffer_forecast_kwh", {}): + def format_time_human_abs(minute_absolute): target_dt = self.midnight + timedelta(minutes=minute_absolute) return target_dt.strftime("%H:%M") @@ -4339,29 +4334,46 @@ def format_time_human_abs(minute_absolute): for minute_relative, kwh_loss in sorted(self.clipping_buffer_forecast_kwh.items()): minute_absolute = minute_relative + self.minutes_now if minute_absolute < midnight_today_minute + 1440: - if today_start is None: today_start = minute_absolute + if today_start is None: + today_start = minute_absolute today_end = minute_absolute + 30 else: - if tomorrow_start is None: tomorrow_start = minute_absolute + if tomorrow_start is None: + tomorrow_start = minute_absolute tomorrow_end = minute_absolute + 30 - + status_parts = [] if self.clipping_remaining_today > 0 and today_start is not None: - status_parts.append("{} kWh clipping buffer remaining between {} and {} today".format( - dp2(self.clipping_remaining_today), - format_time_human_abs(today_start), - format_time_human_abs(today_end) - )) + status_parts.append("{} kWh clipping buffer remaining between {} and {} today".format(dp2(self.clipping_remaining_today), format_time_human_abs(today_start), format_time_human_abs(today_end))) if self.clipping_tomorrow > 0 and tomorrow_start is not None: - status_parts.append("{} kWh clipping buffer forecast between {} and {} tomorrow".format( - dp2(self.clipping_tomorrow), - format_time_human_abs(tomorrow_start), - format_time_human_abs(tomorrow_end) - )) - + status_parts.append("{} kWh clipping buffer forecast between {} and {} tomorrow".format(dp2(self.clipping_tomorrow), format_time_human_abs(tomorrow_start), format_time_human_abs(tomorrow_end))) + if status_parts: clipping_status_text = ". ".join(status_parts) + "." + active_params_list = [] + start_offset = int(getattr(self, "clipping_buffer_start_offset", 0)) + end_offset = int(getattr(self, "clipping_buffer_end_offset", 0)) + amplification = getattr(self, "clipping_amplification", 1.0) + limit_override = getattr(self, "clipping_limit_override", 0) * 60.0 + max_kwh_override = getattr(self, "clipping_buffer_max_kwh", 0) + + if start_offset > 0: + active_params_list.append("{}m start offset".format(start_offset)) + if end_offset > 0: + active_params_list.append("{}m end offset".format(end_offset)) + if amplification is not None and abs(amplification - 1.0) > 1e-4: + active_params_list.append("{}x amplification".format(dp2(amplification))) + if limit_override > 0: + active_params_list.append("{}kW limit override".format(dp2(limit_override))) + if max_kwh_override > 0: + active_params_list.append("{}kWh max override".format(dp2(max_kwh_override))) + + if active_params_list: + if clipping_status_text.endswith("."): + clipping_status_text = clipping_status_text[:-1] + clipping_status_text += " ({} active).".format(", ".join(active_params_list)) + self.dashboard_item( self.prefix + ".clipping_status", state=clipping_status_text, @@ -4375,6 +4387,12 @@ def format_time_human_abs(minute_absolute): "clipping_remaining_today": dp2(self.clipping_remaining_today), "clipping_tomorrow": dp2(self.clipping_tomorrow), "clipping_mitigated_today": dp2(getattr(self, "clipping_mitigated_today", 0.0)), + "clipping_buffer_start_offset": start_offset, + "clipping_buffer_end_offset": end_offset, + "clipping_amplification": dp2(amplification) if amplification is not None else 1.0, + "clipping_limit_override_kw": dp2(limit_override), + "clipping_buffer_max_kwh_override": dp2(max_kwh_override), + "clipping_auto_tune": getattr(self, "clipping_auto_tune", False), }, ) self.dashboard_item(self.prefix + ".record", state=0.0, attributes={"results": self.filtered_times(record_time), "friendly_name": "Prediction window", "state_class": "measurement"}) diff --git a/apps/predbat/prediction.py b/apps/predbat/prediction.py index 4d33c5498..a295f374d 100644 --- a/apps/predbat/prediction.py +++ b/apps/predbat/prediction.py @@ -714,8 +714,6 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi absorbable = min(battery_headroom, max_charge_step) unmitigated_clip = max(potential_clip - absorbable, 0) - - if unmitigated_clip > 0: clipping_penalty = unmitigated_clip * export_rate * clipping_cost_weight metric += clipping_penalty @@ -1326,6 +1324,7 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi iboost_running_full, ) + self.clipping_penalty_total = round(clipping_penalty_total, 4) return ( round(final_metric, 4), round(import_kwh_battery, 4), diff --git a/apps/predbat/solcast.py b/apps/predbat/solcast.py index 69ba0ca32..ebd73a798 100644 --- a/apps/predbat/solcast.py +++ b/apps/predbat/solcast.py @@ -777,7 +777,7 @@ def overlay_clearsky_data(self, pv_forecast_data, source_data, value_field, sour start_dt = start_dt.replace(tzinfo=pytz.utc) except (ValueError, TypeError): continue - + # Calculate source power in kW power_kw = (val / source_divide_by) * 60.0 end_dt = start_dt + timedelta(minutes=source_period) @@ -796,7 +796,7 @@ def overlay_clearsky_data(self, pv_forecast_data, source_data, value_field, sour except (ValueError, TypeError): continue target_end = target_start + timedelta(minutes=target_period) - + # Find all overlapping source intervals weighted_power_sum = 0.0 total_overlap_mins = 0.0 @@ -808,7 +808,7 @@ def overlay_clearsky_data(self, pv_forecast_data, source_data, value_field, sour overlap_duration = (overlap_end - overlap_start).total_seconds() / 60.0 weighted_power_sum += src_power * overlap_duration total_overlap_mins += overlap_duration - + if total_overlap_mins > 0: clearsky_kw = weighted_power_sum / total_overlap_mins # Convert clearsky_kw to target's raw unit @@ -1485,28 +1485,12 @@ async def fetch_pv_forecast(self): self.log("SolarAPI: Overlaying ClearSky data from Open-Meteo API") om_data, _ = await self.download_open_meteo_data() if om_data: - self.overlay_clearsky_data( - pv_forecast_data, - om_data, - value_field="pv_clearsky", - source_period=60, - source_divide_by=60.0, - target_divide_by=divide_by, - target_period=period - ) + self.overlay_clearsky_data(pv_forecast_data, om_data, value_field="pv_clearsky", source_period=60, source_divide_by=60.0, target_divide_by=divide_by, target_period=period) elif clipping_clearsky_source == "solcast_api" and self.solcast_api_key: self.log("SolarAPI: Overlaying ClearSky data from Solcast API") sol_data = await self.download_solcast_data() if sol_data: - self.overlay_clearsky_data( - pv_forecast_data, - sol_data, - value_field="pv_clearsky", - source_period=30, - source_divide_by=30.0, - target_divide_by=divide_by, - target_period=period - ) + self.overlay_clearsky_data(pv_forecast_data, sol_data, value_field="pv_clearsky", source_period=30, source_divide_by=30.0, target_divide_by=divide_by, target_period=period) elif clipping_clearsky_source == "ha_solcast_clearsky": self.log("SolarAPI: Overlaying ClearSky data from Home Assistant integration") ha_data = [] @@ -1517,15 +1501,7 @@ async def fetch_pv_forecast(self): if data: ha_data += data if ha_data: - self.overlay_clearsky_data( - pv_forecast_data, - ha_data, - value_field="pv_clearsky", - source_period=period, - source_divide_by=divide_by, - target_divide_by=divide_by, - target_period=period - ) + self.overlay_clearsky_data(pv_forecast_data, ha_data, value_field="pv_clearsky", source_period=period, source_divide_by=divide_by, target_divide_by=divide_by, target_period=period) elif clipping_clearsky_source == "pv90_scaled": self.log("SolarAPI: Overlaying ClearSky data explicitly disabled, PV90 will be scaled and used as base for clipping mitigation") elif clipping_clearsky_source not in ["auto", "base", ""]: diff --git a/apps/predbat/tests/test_clipping.py b/apps/predbat/tests/test_clipping.py index 9e0a8a3d1..205914d89 100644 --- a/apps/predbat/tests/test_clipping.py +++ b/apps/predbat/tests/test_clipping.py @@ -21,6 +21,7 @@ def run_clipping_tests(my_predbat): failed |= test_inject_cleans_fragmented_windows(my_predbat) failed |= test_clipping_buffer_offsets(my_predbat) failed |= test_clipping_auto_tune_sync(my_predbat) + failed |= test_clipping_status_overrides_display(my_predbat) return failed @@ -230,10 +231,12 @@ def test_clipping_auto_tune_sync(my_predbat): # Mock expose_config and save_current_config exposed_calls = [] + def mock_expose_config(name, value, *args, **kwargs): exposed_calls.append((name, value)) saved_calls = 0 + def mock_save_current_config(*args, **kwargs): nonlocal saved_calls saved_calls += 1 @@ -309,3 +312,133 @@ def mock_save_current_config(*args, **kwargs): if not failed: print("PASS") return failed + + +def test_clipping_status_overrides_display(my_predbat): + print("**** test_clipping_status_overrides_display ****") + failed = False + setup(my_predbat) + + from unittest.mock import MagicMock + + # Set parameters to non-defaults + my_predbat.clipping_buffer_enable = True + my_predbat.clipping_buffer_start_offset = 15 + my_predbat.clipping_buffer_end_offset = 10 + my_predbat.clipping_amplification = 1.25 + my_predbat.clipping_limit_override = 1.0 / 60.0 # 1.0 kW (which is 1/60 internal units) + my_predbat.clipping_buffer_max_kwh = 4.0 + my_predbat.clipping_auto_tune = False + + # Mock self.prediction + mock_prediction = MagicMock() + mock_prediction.run_prediction.return_value = (0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 0, 0.0, 0.0, 0.0, 0.0, {0: 1.0, 60: 1.0, 480: 1.0, 720: 1.0}, [0.0], 0.0, 0.0, 0.0, 0.0) + mock_prediction.predict_soc_time = {0: 1.0} + mock_prediction.first_charge = 0 + mock_prediction.first_charge_soc = 0.0 + mock_prediction.predict_car_soc_time = [{0: 0.0}] + mock_prediction.predict_battery_power = {0: 0.0} + mock_prediction.predict_state = {0: 0.0} + mock_prediction.predict_battery_cycle = {0: 0.0} + mock_prediction.predict_pv_power = {0: 0.0} + mock_prediction.predict_grid_power = {0: 0.0} + mock_prediction.predict_load_power = {0: 0.0} + mock_prediction.final_export_kwh = 0.0 + mock_prediction.export_kwh_h0 = 0.0 + mock_prediction.final_load_kwh = 0.0 + mock_prediction.load_kwh_h0 = 0.0 + mock_prediction.metric_time = {0: 0.0} + mock_prediction.record_time = {0: 0.0} + mock_prediction.predict_iboost = {0: 0.0} + mock_prediction.predict_carbon_g = {0: 0.0} + mock_prediction.load_kwh_time = {0: 0.0} + mock_prediction.pv_kwh_time = {0: 0.0} + mock_prediction.import_kwh_time = {0: 0.0} + mock_prediction.export_kwh_time = {0: 0.0} + mock_prediction.final_pv_kwh = 0.0 + mock_prediction.export_to_first_charge = 0.0 + mock_prediction.pv_kwh_h0 = 0.0 + mock_prediction.final_import_kwh = 0.0 + mock_prediction.final_import_kwh_house = 0.0 + mock_prediction.final_import_kwh_battery = 0.0 + mock_prediction.hours_left = 24.0 + mock_prediction.final_car_soc = [0.0] + mock_prediction.import_kwh_h0 = 0.0 + mock_prediction.predict_export = {0: 0.0} + mock_prediction.predict_soc_best = {0: 1.0} + mock_prediction.predict_iboost_best = {0: 0.0} + mock_prediction.predict_metric_best = {0: 0.0} + mock_prediction.predict_carbon_best = {0: 0.0} + mock_prediction.predict_clipped_best = {0: 0.0} + mock_prediction.debug_enable = False + + original_prediction = getattr(my_predbat, "prediction", None) + my_predbat.prediction = mock_prediction + + # Mock scenario helpers to avoid executing them on mock prediction datasets + original_summary_title = my_predbat.scenario_summary_title + original_summary = my_predbat.scenario_summary + original_summary_state = my_predbat.scenario_summary_state + + my_predbat.scenario_summary_title = lambda x: "dummy_title" + my_predbat.scenario_summary = lambda x, y: "dummy_summary" + my_predbat.scenario_summary_state = lambda x: "dummy_state" + + # Mock dashboard_item + exposed_items = {} + original_dashboard_item = my_predbat.dashboard_item + + def mock_dashboard_item(name, state=None, attributes=None): + exposed_items[name] = {"state": state, "attributes": attributes} + + my_predbat.dashboard_item = mock_dashboard_item + + try: + # Run prediction with save="best" to trigger the status string generation + my_predbat.run_prediction(my_predbat.charge_limit_best, my_predbat.charge_window_best, my_predbat.export_window_best, my_predbat.export_limits_best, False, 24 * 60, save="best") + + status_key = my_predbat.prefix + ".clipping_status" + if status_key not in exposed_items: + print("ERROR: clipping_status was not published to dashboard") + failed = True + else: + item = exposed_items[status_key] + state = item["state"] + attrs = item["attributes"] + + # Expected overrides text inside the state description + expected_override_str = "(15m start offset, 10m end offset, 1.25x amplification, 1.0kW limit override, 4.0kWh max override active)." + if expected_override_str not in state: + print("ERROR: Expected override string '{}' not found in state '{}'".format(expected_override_str, state)) + failed = True + + # Assert attributes + if attrs.get("clipping_buffer_start_offset") != 15: + print("ERROR: attribute clipping_buffer_start_offset mismatch") + failed = True + if attrs.get("clipping_buffer_end_offset") != 10: + print("ERROR: attribute clipping_buffer_end_offset mismatch") + failed = True + if attrs.get("clipping_amplification") != 1.25: + print("ERROR: attribute clipping_amplification mismatch") + failed = True + if attrs.get("clipping_limit_override_kw") != 1.0: + print("ERROR: attribute clipping_limit_override_kw mismatch, got {}".format(attrs.get("clipping_limit_override_kw"))) + failed = True + if attrs.get("clipping_buffer_max_kwh_override") != 4.0: + print("ERROR: attribute clipping_buffer_max_kwh_override mismatch") + failed = True + + finally: + if original_prediction is not None: + my_predbat.prediction = original_prediction + elif hasattr(my_predbat, "prediction"): + del my_predbat.prediction + my_predbat.dashboard_item = original_dashboard_item + my_predbat.scenario_summary_title = original_summary_title + my_predbat.scenario_summary = original_summary + my_predbat.scenario_summary_state = original_summary_state + + if not failed: + print("PASS") + return failed diff --git a/apps/predbat/tests/test_solcast.py b/apps/predbat/tests/test_solcast.py index 1512c665b..beaf5e884 100644 --- a/apps/predbat/tests/test_solcast.py +++ b/apps/predbat/tests/test_solcast.py @@ -1908,7 +1908,7 @@ def test_fetch_pv_forecast_ha_clearsky(my_predbat): test_api.solar.forecast_solar = None test_api.solar.pv_forecast_today = "sensor.solcast_pv_forecast_today" test_api.solar.pv_forecast_tomorrow = None - + # Enable clearsky source as HA test_api.mock_base.set_arg("clipping_clearsky_source", "ha_solcast_clearsky") test_api.solar.pv_clearsky_today = "sensor.solcast_clear_sky_today" @@ -1927,7 +1927,7 @@ def test_fetch_pv_forecast_ha_clearsky(my_predbat): ], }, ) - + # Setup mock clearsky sensor test_api.set_mock_ha_state( "sensor.solcast_clear_sky_today", @@ -1946,7 +1946,7 @@ def create_mock_session(*args, **kwargs): with patch("solcast.aiohttp.ClientSession", side_effect=create_mock_session): run_async(test_api.solar.fetch_pv_forecast()) - # Verify that clearsky was overlayed correctly and published + # Verify that clearsky was overlaid correctly and published today_entity = f"sensor.{test_api.mock_base.prefix}_pv_today" if today_entity not in test_api.dashboard_items: print("ERROR: Expected pv_today entity to be published") diff --git a/apps/predbat/web.py b/apps/predbat/web.py index 532a1d43a..c234c63aa 100644 --- a/apps/predbat/web.py +++ b/apps/predbat/web.py @@ -3069,20 +3069,22 @@ def get_chart(self, chart): stamp = minute_timestamp.strftime(TIME_FORMAT) clipping_remaining_series[stamp] = round(kwh, 2) - clipping_ceiling_raw = history_attribute(self.get_history_wrapper(self.prefix + ".clipping_ceiling", 3)) - clipping_ceiling_series = prune_today(clipping_ceiling_raw, self.now_utc, self.midnight_utc, prune_past_days=2) + clipping_target_soc_raw = history_attribute(self.get_history_wrapper(self.prefix + ".clipping_target_soc", 3)) + clipping_target_soc_series = prune_today(clipping_target_soc_raw, self.now_utc, self.midnight_utc, prune_past_days=2) - if getattr(self.base, "predict_clipping_ceiling_best", None): - for minute, kwh in self.base.predict_clipping_ceiling_best.items(): + if getattr(self.base, "predict_clipping_target_soc_best", None): + for minute, kwh in self.base.predict_clipping_target_soc_best.items(): if minute % step_size == 0 and minute >= 0: minute_timestamp = self.now_utc + timedelta(minutes=minute) stamp = minute_timestamp.strftime(TIME_FORMAT) - clipping_ceiling_series[stamp] = round(kwh, 2) + clipping_target_soc_series[stamp] = round(kwh, 2) # PV Forecast (Base) - historical and future pv_forecast_hist = history_attribute(self.get_history_wrapper("sensor." + self.prefix + "_pv_forecast_h0", 3, required=False)) raw_pv_series = prune_today(pv_forecast_hist, self.now_utc, self.midnight_utc, prune=True, prune_past_days=2, intermediate=True) - raw_pv_series.update(prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + "_pv_today", "pv_estimate"), self.now_utc, self.midnight_utc, prune=True, intermediate=True)) + pv_today_est = self.get_entity_detailedForecast("sensor." + self.prefix + "_pv_today", "pv_estimate") + pv_today_est_future = {k: v for k, v in pv_today_est.items() if str2time(k) >= self.now_utc - timedelta(minutes=30)} + raw_pv_series.update(prune_today(pv_today_est_future, self.now_utc, self.midnight_utc, prune=True, intermediate=True)) raw_pv_series.update(prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + "_pv_tomorrow", "pv_estimate"), self.now_utc, self.midnight_utc, prune=True, intermediate=True)) for d in ["d2", "d3"]: raw_pv_series.update(prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + f"_pv_{d}", "pv_estimate"), self.now_utc, self.midnight_utc, prune=True, intermediate=True)) @@ -3090,7 +3092,9 @@ def get_chart(self, chart): # PV Forecast (ClearSky) - historical and future pv_forecast_histCS = history_attribute(self.get_history_wrapper("sensor." + self.prefix + "_pv_forecast_h0", 3, required=False), attributes=True, state_key="nowCS") cs_pv_series = prune_today(pv_forecast_histCS, self.now_utc, self.midnight_utc, prune=True, prune_past_days=2, intermediate=True) - cs_pv_series.update(prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + "_pv_today", "pv_clearsky"), self.now_utc, self.midnight_utc, prune=True, intermediate=True)) + pv_today_cs = self.get_entity_detailedForecast("sensor." + self.prefix + "_pv_today", "pv_clearsky") + pv_today_cs_future = {k: v for k, v in pv_today_cs.items() if str2time(k) >= self.now_utc - timedelta(minutes=30)} + cs_pv_series.update(prune_today(pv_today_cs_future, self.now_utc, self.midnight_utc, prune=True, intermediate=True)) cs_pv_series.update(prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + "_pv_tomorrow", "pv_clearsky"), self.now_utc, self.midnight_utc, prune=True, intermediate=True)) for d in ["d2", "d3"]: cs_pv_series.update(prune_today(self.get_entity_detailedForecast("sensor." + self.prefix + f"_pv_{d}", "pv_clearsky"), self.now_utc, self.midnight_utc, prune=True, intermediate=True)) @@ -3099,7 +3103,7 @@ def get_chart(self, chart): {"name": "Clipping Remaining", "data": clipping_remaining_series, "opacity": "1.0", "stroke_width": "3", "stroke_curve": "smooth", "color": "#FF1493", "unit": "kWh"}, {"name": "Actual SOC", "data": soc_kw_h0, "opacity": "1.0", "stroke_width": "3", "stroke_curve": "smooth", "color": "#00BFFF", "unit": "kWh"}, {"name": "Target SOC", "data": soc_kw_best, "opacity": "0.7", "stroke_width": "2", "stroke_curve": "stepline", "stroke_dasharray": "5", "color": "#1E90FF", "unit": "kWh"}, - {"name": "Clipping Ceiling", "data": clipping_ceiling_series, "opacity": "0.7", "stroke_width": "2", "stroke_curve": "stepline", "stroke_dasharray": "5", "color": "#C71585", "unit": "kWh"}, + {"name": "Clipping Target SOC", "data": clipping_target_soc_series, "opacity": "0.7", "stroke_width": "2", "stroke_curve": "stepline", "stroke_dasharray": "5", "color": "#C71585", "unit": "kWh"}, {"name": "PV Power Actual", "data": pv_power, "opacity": "1.0", "stroke_width": "3", "stroke_curve": "smooth", "color": "#FFA500", "unit": "kW"}, {"name": "PV Forecast (Base)", "data": raw_pv_series, "opacity": "0.7", "stroke_width": "2", "stroke_curve": "smooth", "stroke_dasharray": "5", "color": "#FFD700", "unit": "kW"}, ] @@ -3110,6 +3114,12 @@ def get_chart(self, chart): if inverter_ac_limit_kw > 0: yaxis_annotations.append({"y": inverter_ac_limit_kw, "borderColor": "#FF0000", "strokeDashArray": 4, "text": "Inverter AC Capacity ({} kW)".format(inverter_ac_limit_kw), "textColor": "#fff", "backgroundColor": "#FF0000"}) + clipping_limit_override_kw = getattr(self.base, "clipping_limit_override", 0) * 60.0 + if clipping_limit_override_kw > 0: + yaxis_annotations.append( + {"y": clipping_limit_override_kw, "borderColor": "#C71585", "strokeDashArray": 4, "text": "Clipping Limit Override ({} kW)".format(dp2(clipping_limit_override_kw)), "textColor": "#fff", "backgroundColor": "#C71585"} + ) + xaxis_annotations = [] buffer_start = getattr(self.base, "clipping_buffer_start", None) buffer_end = getattr(self.base, "clipping_buffer_end", None) diff --git a/docs/customisation.md b/docs/customisation.md index 33ee0ec1f..1f8284f24 100644 --- a/docs/customisation.md +++ b/docs/customisation.md @@ -478,10 +478,10 @@ These settings can be found in the Home Assistant Predbat Configuration panel, d - **`clipping_buffer_enable`**: The master switch. Turns the entire clipping buffer feature on or off. - **`clipping_clearsky_source`**: Determines where Predbat gets its clear-sky (theoretical maximum) data. Options include: - * `auto`: Automatically select the best available source from the options below. - * `ha_solcast_clearsky`: Integrates with the HACS [Solcast Solar integration](https://github.com/BJReplay/ha-solcast-solar) (requires separate installation, but uses existing sensors). - * `solcast_api`: Direct native Solcast API integration (only works if your Solcast API key has clear-sky access). - * `openmeteo`: Uses the free Open-Meteo API as a fallback source. + - `auto`: Automatically select the best available source from the options below. + - `ha_solcast_clearsky`: Integrates with the HACS [Solcast Solar integration](https://github.com/BJReplay/ha-solcast-solar) (requires separate installation, but uses existing sensors). + - `solcast_api`: Direct native Solcast API integration (only works if your Solcast API key has clear-sky access). + - `openmeteo`: Uses the free Open-Meteo API as a fallback source. - **`clipping_auto_tune`**: Automatically learns the safety buffer margins from past clipping behavior. When enabled, it dynamically tunes and auto-populates `clipping_amplification`, `clipping_buffer_start_offset`, and `clipping_buffer_end_offset` directly on your dashboard entities. - **`clipping_amplification`**: A manual multiplier (e.g. `1.5x`) to scale the modeled solar forecast peak. Tuned dynamically if Auto-Tune is ON. - **`clipping_cost_weight`**: An internal optimizer multiplier (default `1.0`). If Predbat isn't dumping the battery aggressively enough before a peak, increasing this number adds a harsher financial penalty for clipping, forcing the optimizer to prioritize creating headroom. From 94798558412f641cc08e09dc739f9c8da76f79a1 Mon Sep 17 00:00:00 2001 From: Robin Date: Sat, 27 Jun 2026 14:44:45 +0100 Subject: [PATCH 51/74] Ignore local scratch files --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index b3bf6746a..7220b8fe6 100644 --- a/.gitignore +++ b/.gitignore @@ -105,3 +105,9 @@ it # Runtime-generated manifest apps/predbat/manifest.yaml + +# Local development and scratch files +scratch/ +cases/ +apps_ha.yaml + From 8adc9051582bc4033398dad92203ddbd3f33fd32 Mon Sep 17 00:00:00 2001 From: Robin Date: Sat, 27 Jun 2026 17:06:08 +0100 Subject: [PATCH 52/74] Fix clipping anomaly by leveraging native inverter force export SOC limit --- .gitignore | 1 - apps/predbat/execute.py | 8 +++++++- apps/predbat/tests/test_execute.py | 17 +++++++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 7220b8fe6..e5f87e26e 100644 --- a/.gitignore +++ b/.gitignore @@ -110,4 +110,3 @@ apps/predbat/manifest.yaml scratch/ cases/ apps_ha.yaml - diff --git a/apps/predbat/execute.py b/apps/predbat/execute.py index 5e33b77d2..260b702d4 100644 --- a/apps/predbat/execute.py +++ b/apps/predbat/execute.py @@ -347,7 +347,8 @@ def execute_plan(self): discharge_soc = max((int(self.export_limits_best[0]) * self.soc_max) / 100.0, self.reserve, self.best_soc_min) self.log("Next export window will be: {} - {} at reserve {}".format(discharge_start_time, discharge_end_time, self.export_limits_best[0])) if (self.minutes_now >= minutes_start) and (self.minutes_now < minutes_end) and (self.export_limits_best[0] < 100.0): - if not self.set_export_freeze_only and self.export_limits_best[0] < 99.0 and (self.soc_kw > discharge_soc): + is_anti_clipping = self.export_window_best[0].get("clipping_target_soc_pct") is not None + if not self.set_export_freeze_only and self.export_limits_best[0] < 99.0 and (self.soc_kw > discharge_soc or is_anti_clipping): if self.set_export_low_power: export_rate_adjust = 1 - (self.export_limits_best[0] - int(self.export_limits_best[0])) else: @@ -358,6 +359,11 @@ def execute_plan(self): inverter.adjust_discharge_rate(inverter.battery_rate_max_export * export_rate_adjust * MINUTE_WATT) resetDischarge = False inverter.adjust_force_export(True, discharge_start_time, discharge_end_time) + + if self.set_reserve_enable and is_anti_clipping: + inverter.adjust_reserve(int(self.export_limits_best[0])) + resetReserve = False + if inverter.inv_charge_discharge_with_rate: inverter.adjust_charge_rate(0) resetCharge = False diff --git a/apps/predbat/tests/test_execute.py b/apps/predbat/tests/test_execute.py index b7b361116..fe6aa9070 100644 --- a/apps/predbat/tests/test_execute.py +++ b/apps/predbat/tests/test_execute.py @@ -2433,6 +2433,23 @@ def run_execute_tests(my_predbat): assert_discharge_end_time_minutes=my_predbat.minutes_now + 60 + 1, ) + failed |= run_execute_test( + my_predbat, + "anti_clipping_force_export_limit", + export_window_best=[{"start": my_predbat.minutes_now, "end": my_predbat.minutes_now + 60, "target": 80, "clipping_target_soc_pct": 80}], + export_limits_best=[80], + assert_force_export=True, + set_charge_window=True, + set_export_window=True, + soc_kw=7.9, + soc_max=10.0, + assert_status="Exporting", + assert_reserve=80, + assert_immediate_soc_target=80, + assert_discharge_start_time_minutes=my_predbat.minutes_now, + assert_discharge_end_time_minutes=my_predbat.minutes_now + 60 + 1, + ) + # Reset test my_predbat.reset_inverter() failed |= run_execute_test( From feb27016d7cdea040b29a4e6cd8953c687aa3e08 Mon Sep 17 00:00:00 2001 From: Robin Date: Sat, 27 Jun 2026 19:05:22 +0100 Subject: [PATCH 53/74] Fix clipping status text description bug in output plan --- apps/predbat/plan.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index 3249314c8..1871cef30 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -4528,6 +4528,8 @@ def format_time_human_abs(minute_absolute): clipping_status_text = clipping_status_text[:-1] clipping_status_text += " ({} active).".format(", ".join(active_params_list)) + self.clipping_status = clipping_status_text + self.dashboard_item( self.prefix + ".clipping_status", state=clipping_status_text, From e494dd63a3df9ea5d4555df19ef6919db9d63f1f Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 28 Jun 2026 10:14:09 +0100 Subject: [PATCH 54/74] Fix battery discharge anomaly during solar clipping by penalizing instantaneous PV clipping --- apps/predbat/prediction.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/predbat/prediction.py b/apps/predbat/prediction.py index a295f374d..6390f0970 100644 --- a/apps/predbat/prediction.py +++ b/apps/predbat/prediction.py @@ -1099,6 +1099,7 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi pv_ac = max(pv_ac - over_limit * inverter_loss, 0) pv_ac_no_loss = max(pv_ac_before - over_limit, 0) clipped_today += pv_ac_before - pv_ac_no_loss + metric += (pv_ac_before - pv_ac_no_loss) * export_rate * clipping_cost_weight * 5 total_inverted = get_total_inverted(battery_draw, pv_dc, pv_ac, inverter_loss, inverter_hybrid) else: total_inverted = get_total_inverted(battery_draw, pv_dc, pv_ac, inverter_loss, inverter_hybrid) @@ -1117,6 +1118,7 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi pv_ac_before = pv_ac pv_ac = max(pv_ac - over_limit, 0) clipped_today += pv_ac_before - pv_ac + metric += (pv_ac_before - pv_ac) * export_rate * clipping_cost_weight * 5 # Adjust battery soc if battery_draw > 0: From 678b858544a3af230746ee223ee467471b77ce9b Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 28 Jun 2026 10:40:27 +0100 Subject: [PATCH 55/74] Fix mathematical scaling error in clipping penalty to stop battery yo-yo --- apps/predbat/prediction.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/predbat/prediction.py b/apps/predbat/prediction.py index 6390f0970..b79584799 100644 --- a/apps/predbat/prediction.py +++ b/apps/predbat/prediction.py @@ -700,7 +700,7 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi battery_headroom = max(soc_max - soc, 0) * battery_loss if battery_headroom < target_headroom: # Apply a severe penalty to force the optimizer to respect the manual buffer limit - clipping_penalty = (target_headroom - battery_headroom) * export_rate * clipping_cost_weight * 5 + clipping_penalty = (target_headroom - battery_headroom) * export_rate * clipping_cost_weight * (step / 60.0) metric += clipping_penalty clipping_penalty_total += clipping_penalty elif pv_forecast_peak_step and clipping_limit_step > 0 and clipping_cost_weight > 0: @@ -715,7 +715,7 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi unmitigated_clip = max(potential_clip - absorbable, 0) if unmitigated_clip > 0: - clipping_penalty = unmitigated_clip * export_rate * clipping_cost_weight + clipping_penalty = unmitigated_clip * (step / 60.0) * export_rate * clipping_cost_weight metric += clipping_penalty clipping_penalty_total += clipping_penalty From 5a045e5cf988d4af73b39c965b06af76ff8a4bfd Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 28 Jun 2026 11:51:34 +0100 Subject: [PATCH 56/74] Allow Force Export with Hold SOC limit via 98.0 magic flag --- apps/predbat/plan.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index 1871cef30..e9f9e89a7 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -1988,10 +1988,12 @@ def optimise_export(self, window_n, record_charge_windows, try_charge_limit, cha loop_options = [100.0, 99.0, 0.0] if self.set_export_low_power: loop_options.extend([0.3, 0.5, 0.7]) + loop_options.append(98.0) else: loop_options = [100.0, 0.0] if self.set_export_low_power: loop_options.extend([0.3, 0.5, 0.7]) + loop_options.append(98.0) # Ensure any pre-assigned fractional limits (like clipping thresholds) are evaluated if try_export[window_n] not in loop_options and try_export[window_n] > 0.0 and try_export[window_n] < 99.0: @@ -2016,6 +2018,12 @@ def optimise_export(self, window_n, record_charge_windows, try_charge_limit, cha else: loop_start -= export_step + if this_export_limit == 98.0: + soc_at_start = self.predict_soc_best.get(start, self.soc_max) + this_export_limit = float(calc_percent_limit(soc_at_start, self.soc_max)) + if this_export_limit in loop_options and this_export_limit != 98.0: + continue + # Can't optimise all window start slot if all_n and (start != window["start"]): continue From d58b1d59d937ea0fcd959ba85b41b14a0d464601 Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 28 Jun 2026 12:29:13 +0100 Subject: [PATCH 57/74] Fix clipping threshold simulation mismatch and window start time --- apps/predbat/plan.py | 16 ++++++++++++++++ apps/predbat/tests/test_clipping.py | 13 +++++-------- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index e9f9e89a7..217215ea1 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -936,6 +936,16 @@ def intersects(w, start, end): minutes_needed = max(60, min(360, minutes_needed)) morning_start = max(self.minutes_now, peak_start - minutes_needed) + + # Stretch the start time back to the end of the overnight charge window, or 06:00 + midnight = int(peak_start / 1440) * 1440 + early_start = midnight + (6 * 60) + if getattr(self, "clipping_buffer_start", None) is not None: + early_start = midnight + self.clipping_buffer_start + + morning_start = min(morning_start, early_start) + morning_start = max(self.minutes_now, morning_start) + morning_start = int(morning_start / 30) * 30 # Align to nearest 30 mins if morning_start >= peak_start: @@ -1999,6 +2009,12 @@ def optimise_export(self, window_n, record_charge_windows, try_charge_limit, cha if try_export[window_n] not in loop_options and try_export[window_n] > 0.0 and try_export[window_n] < 99.0: loop_options.append(try_export[window_n]) + # FORCE the optimizer to respect clipping protection by removing the "do nothing" (100.0) option + # This ensures we pick either the hold flag (98.0) or the explicit clipping target (e.g. 61.0) + is_clipping_window = ("clipping_target_soc_pct" in try_export_window[window_n]) + if is_clipping_window and 100.0 in loop_options and len(loop_options) > 1: + loop_options.remove(100.0) + # Collect all options results = [] results10 = [] diff --git a/apps/predbat/tests/test_clipping.py b/apps/predbat/tests/test_clipping.py index 205914d89..cf15c844b 100644 --- a/apps/predbat/tests/test_clipping.py +++ b/apps/predbat/tests/test_clipping.py @@ -95,12 +95,9 @@ def test_inject_creates_contiguous_window(my_predbat): w = my_predbat.export_window_best[0] - # Correct calculation: peak at 780 to 840. - # To create 2.0 kWh headroom with 1.0 kW effective discharge rate requires 120 minutes of discharge. - # Plus 30 mins safety margin = 150 minutes. - # Start = max(240, 780 - 150) = 630. - if w["start"] != 630: - print("ERROR: Expected window start at 630, got {}".format(w["start"])) + # With the new behavior, morning_start is stretched back to 06:00 (360 minutes absolute) + if w["start"] != 360: + print("ERROR: Expected window start at 360, got {}".format(w["start"])) failed = True if w["end"] != 840: @@ -151,8 +148,8 @@ def test_inject_cleans_fragmented_windows(my_predbat): print("ERROR: Non-intersecting windows were incorrectly dropped!") failed = True - if 690 not in starts: # Injected window start - print("ERROR: Injected window start 690 not found, got starts: {}".format(starts)) + if 360 not in starts: # Injected window start + print("ERROR: Injected window start 360 not found, got starts: {}".format(starts)) failed = True # Check that limits are aligned: W0 (limit 10.0), W1 (limit 20.0), W3 (limit 40.0), and new window (target_soc_pct, e.g. 80.0) From 24a17cc1b24a3f3576b73beb90c94b8b2c2d59cd Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 28 Jun 2026 12:36:38 +0100 Subject: [PATCH 58/74] Make clipping start time dynamically follow charge_window_best --- apps/predbat/plan.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index 217215ea1..d077c2f4e 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -937,9 +937,20 @@ def intersects(w, start, end): morning_start = max(self.minutes_now, peak_start - minutes_needed) - # Stretch the start time back to the end of the overnight charge window, or 06:00 + # Stretch the start time back to the end of the overnight charge window, or 06:00 if unknown midnight = int(peak_start / 1440) * 1440 early_start = midnight + (6 * 60) + + # Find the end of the last charge window before the peak + if getattr(self, "charge_window_best", None): + last_charge_end = midnight + for w in self.charge_window_best: + if w["end"] <= peak_start and w["end"] > last_charge_end: + last_charge_end = w["end"] + # Start clipping window right after the morning charge finishes + if last_charge_end > midnight: + early_start = last_charge_end + if getattr(self, "clipping_buffer_start", None) is not None: early_start = midnight + self.clipping_buffer_start From 13470fff93f9605ce5e7bfedccc6a3d22c187e8a Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 28 Jun 2026 14:44:48 +0100 Subject: [PATCH 59/74] Fix REQ-08: Prevent premature battery filling during clipping export windows by holding reserve logic --- apps/predbat/prediction.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/predbat/prediction.py b/apps/predbat/prediction.py index b79584799..53eea67d3 100644 --- a/apps/predbat/prediction.py +++ b/apps/predbat/prediction.py @@ -853,7 +853,7 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi if export_window_active: discharge_min = max(soc_max * export_limit_now / 100.0, reserve, self.best_soc_min) - if not set_export_freeze_only and export_window_active and export_limit_now < 99.0 and (soc > discharge_min): + if not set_export_freeze_only and export_window_active and export_limit_now < 99.0: # Discharge enable, capped at export limit if self.set_export_low_power: export_rate_adjust = 1 - (export_limit_now - int(export_limit_now)) @@ -866,7 +866,8 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi ) discharge_rate_now_curve_step = discharge_rate_now_curve * step - battery_draw = min(discharge_rate_now_curve_step, battery_to_min) + battery_to_discharge_min = max(soc - discharge_min, 0) * battery_loss_discharge + battery_draw = min(discharge_rate_now_curve_step, battery_to_discharge_min) pv_ac = pv_now * inverter_loss_ac pv_dc = 0 From a67df958faa972a059f76e8392a2717ccc69268a Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 28 Jun 2026 16:18:56 +0100 Subject: [PATCH 60/74] Fix bug where ML training blocks asyncio event loop causing WebSocket drop --- apps/predbat/load_ml_component.py | 49 ++++++++++++++++++------------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/apps/predbat/load_ml_component.py b/apps/predbat/load_ml_component.py index 87d46d9d8..ea2ce527b 100644 --- a/apps/predbat/load_ml_component.py +++ b/apps/predbat/load_ml_component.py @@ -823,7 +823,8 @@ def dict_to_array(data_dict): import_rate=dict_to_array(self.import_rates_data), export_rate=dict_to_array(self.export_rates_data), ) - np.savez_compressed(self.database_filepath, **save_kwargs) + loop = asyncio.get_running_loop() + await loop.run_in_executor(None, lambda: np.savez_compressed(self.database_filepath, **save_kwargs)) self.log("ML Component: Saved {} steps ({} days) of history to {}".format(total_steps, self.load_data_age_days, self.database_filepath)) async def load_database_history(self): @@ -842,7 +843,8 @@ async def load_database_history(self): try: # np.load is blocking disk IO - run in executor so the event loop stays alive - data = np.load(self.database_filepath, allow_pickle=False) + loop = asyncio.get_running_loop() + data = await loop.run_in_executor(None, lambda: np.load(self.database_filepath, allow_pickle=False)) metadata = json.loads(str(data["metadata_json"])) version = metadata.get("version", 0) @@ -930,10 +932,32 @@ async def _do_training(self, is_initial): # Lock released - event loop is free during training try: + loop = asyncio.get_running_loop() if is_initial: # Curriculum: progressively expand the training window from oldest week # forward so the model learns gradually from historical structure. - val_mae = self.predictor.train_curriculum( + val_mae = await loop.run_in_executor( + None, + lambda: self.predictor.train_curriculum( + load_data_snap, + now_utc_snap, + pv_minutes=pv_data_snap, + temp_minutes=temp_data_snap, + import_rates=import_rates_snap, + export_rates=export_rates_snap, + epochs=epochs, + time_decay_days=time_decay, + validation_holdout_hours=holdout_hours, + patience=patience, + curriculum_window_days=window_days, + curriculum_step_days=5, + max_intermediate_passes=8, + ) + ) + # Even if initial was done we need to do one fine tuned curriculum pass too. + val_mae = await loop.run_in_executor( + None, + lambda: self.predictor.train_curriculum( load_data_snap, now_utc_snap, pv_minutes=pv_data_snap, @@ -945,24 +969,9 @@ async def _do_training(self, is_initial): validation_holdout_hours=holdout_hours, patience=patience, curriculum_window_days=window_days, - curriculum_step_days=5, - max_intermediate_passes=8, + curriculum_step_days=step_days, + max_intermediate_passes=max_intermediate_passes, ) - # Even if initial was done we need to do one fine tuned curriculum pass too. - val_mae = self.predictor.train_curriculum( - load_data_snap, - now_utc_snap, - pv_minutes=pv_data_snap, - temp_minutes=temp_data_snap, - import_rates=import_rates_snap, - export_rates=export_rates_snap, - epochs=epochs, - time_decay_days=time_decay, - validation_holdout_hours=holdout_hours, - patience=patience, - curriculum_window_days=window_days, - curriculum_step_days=step_days, - max_intermediate_passes=max_intermediate_passes, ) if val_mae is not None: From da6bcc42d1154732f47f5abe09cacf9190dea96a Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 28 Jun 2026 20:53:00 +0100 Subject: [PATCH 61/74] Fix clipping buffer underestimation and prevent target SOC reset during optimization passes --- AGENTS.md | 7 +++++++ apps/predbat/plan.py | 23 +++++++++++++++-------- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 66fb8eb7e..a16b3b8c3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -125,3 +125,10 @@ mkdocs serve # Live preview at http://localhost:8000 ``` When adding a new doc page, add it to `mkdocs.yml`. The published site at is built automatically from `main` via GitHub Actions. + +## Anti-Clipping Requirements + +When developing features, fixing bugs, or reviewing PRs related to the `clipping-cloud-model` or legacy `clipping-buffer` branches, **you MUST always consult the requirements validation document**: +`../clipping_requirements_validation.md` (Note: This is located one directory above the repository root in the `Tech Projects` folder). + +This document serves as the absolute ground truth for how the clipping buffer prediction, generation overlay, dynamic SOC targeting, and freeze-export bypassing logic should behave. Ensure any changes align perfectly with the `REQ-xx` matrix. diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index d077c2f4e..f8e22b87b 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -341,7 +341,10 @@ def optimise_charge_limit_price_threads( else: price_set_export.append([price, window_n, typ == "df"]) valid_export_windows[window_n] = True - best_export_limits_reset[window_n] = 100.0 + if "clipping_target_soc_pct" in export_window[window_n]: + best_export_limits_reset[window_n] = export_window[window_n]["clipping_target_soc_pct"] + else: + best_export_limits_reset[window_n] = 100.0 FINE_SLOT_LENGTHS = [48, 32, 24, 16, 14, 12, 10, 8, 6, 5, 4, 3, 2, 1, 0] COARSE_SLOT_LENGTHS = [32, 16, 8, 4, 2, 1, 0] @@ -403,7 +406,10 @@ def optimise_charge_limit_price_threads( pass else: all_d.append(window_n) - try_export[window_n] = 99.0 if freeze else min_freeze_percent + if "clipping_target_soc_pct" in export_window[window_n]: + try_export[window_n] = export_window[window_n]["clipping_target_soc_pct"] + else: + try_export[window_n] = 99.0 if freeze else min_freeze_percent count_d += 1 # Remove export hitting charge windows if this is disabled @@ -887,12 +893,12 @@ def intersects(w, start, end): # We walk backwards from the peak minute by minute to account for the fact that # solar generation (PV) during the export window consumes inverter capacity and # reduces the effective battery discharge rate. - max_kwh_loss = max(kwh_loss for m_rel, kwh_loss in forecast.items() if peak_start <= (m_rel + self.minutes_now) <= peak_end) + total_kwh_loss = sum(kwh_loss for m_rel, kwh_loss in forecast.items() if peak_start <= (m_rel + self.minutes_now) <= peak_end) # Cap the requested headroom by the physical capacity of the battery. # We cannot create more headroom than the battery can physically hold. battery_capacity_kwh = self.soc_max - getattr(self, "best_soc_min", 0.0) - max_kwh_loss = min(max_kwh_loss, battery_capacity_kwh) + total_kwh_loss = min(total_kwh_loss, battery_capacity_kwh) hybrid = getattr(self, "inverter_hybrid", False) inverter_limit_kw = getattr(self, "inverter_limit", 0.0) * 60.0 @@ -926,7 +932,7 @@ def intersects(w, start, end): accumulated_kwh += discharge_kw * (1.0 / 60.0) minutes_needed += 1 - if accumulated_kwh >= max_kwh_loss - 1e-9: + if accumulated_kwh >= total_kwh_loss - 1e-9: break # Add 30 mins safety margin @@ -963,7 +969,7 @@ def intersects(w, start, end): continue # Calculate the precise export limit (Target SOC percentage) required to create this headroom - target_soc_kwh = max(0.0, self.soc_max - max_kwh_loss) + target_soc_kwh = max(0.0, self.soc_max - total_kwh_loss) target_soc_pct = max(0.0, min(100.0, target_soc_kwh / self.soc_max * 100.0)) target_soc_pct = float(int(target_soc_pct)) @@ -3844,11 +3850,12 @@ def optimise_charge_windows_reset(self, reset_all): if self.export_window_best and self.calculate_best_export: # Set all to max for window_n in range(len(self.export_window_best)): + target_limit = self.export_window_best[window_n].get("clipping_target_soc_pct", 100.0) if self.export_window_best[window_n]["start"] < (self.minutes_now + self.end_record): if reset_all: - self.export_limits_best[window_n] = 100.0 + self.export_limits_best[window_n] = target_limit else: - self.export_limits_best[window_n] = 100.0 + self.export_limits_best[window_n] = target_limit def run_prediction(self, charge_limit, charge_window, export_window, export_limits, pv10, end_record, save=None, step=PREDICT_STEP): """ From 06cf34df1c1209f93c341e89866825727bd9ca03 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 29 Jun 2026 08:44:10 +0100 Subject: [PATCH 62/74] Sync latest clipping fixes: simulation SOC guard bypass, set_export_freeze_only bypass, and export_more_solar conflicts --- apps/predbat/execute.py | 2 +- apps/predbat/plan.py | 25 ++++++++++++++++++------- apps/predbat/prediction.py | 4 +++- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/apps/predbat/execute.py b/apps/predbat/execute.py index 260b702d4..f5236085c 100644 --- a/apps/predbat/execute.py +++ b/apps/predbat/execute.py @@ -348,7 +348,7 @@ def execute_plan(self): self.log("Next export window will be: {} - {} at reserve {}".format(discharge_start_time, discharge_end_time, self.export_limits_best[0])) if (self.minutes_now >= minutes_start) and (self.minutes_now < minutes_end) and (self.export_limits_best[0] < 100.0): is_anti_clipping = self.export_window_best[0].get("clipping_target_soc_pct") is not None - if not self.set_export_freeze_only and self.export_limits_best[0] < 99.0 and (self.soc_kw > discharge_soc or is_anti_clipping): + if (not self.set_export_freeze_only or is_anti_clipping) and self.export_limits_best[0] < 99.0 and (self.soc_kw > discharge_soc or is_anti_clipping): if self.set_export_low_power: export_rate_adjust = 1 - (self.export_limits_best[0] - int(self.export_limits_best[0])) else: diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index f8e22b87b..28c09132f 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -902,11 +902,10 @@ def intersects(w, start, end): hybrid = getattr(self, "inverter_hybrid", False) inverter_limit_kw = getattr(self, "inverter_limit", 0.0) * 60.0 - user_inverter_limit = getattr(self, "set_inverter_limit", 0) - if user_inverter_limit > 0: - inverter_limit_kw = user_inverter_limit / 1000.0 export_limit_kw = getattr(self, "export_limit", 0.0) * 60.0 - pv_forecast = getattr(self, "pv_forecast_minute_step", {}) + # Prefer stepped PV data; on the first plan cycle it hasn't been computed yet, + # so fall back to the raw minute-level forecast to approximate PV contribution. + pv_forecast = getattr(self, "pv_forecast_minute_step", {}) or getattr(self, "pv_forecast_minute", {}) accumulated_kwh = 0.0 minutes_needed = 0 @@ -957,8 +956,16 @@ def intersects(w, start, end): if last_charge_end > midnight: early_start = last_charge_end - if getattr(self, "clipping_buffer_start", None) is not None: - early_start = midnight + self.clipping_buffer_start + # Read manual start override from config if the attribute hasn't been + # initialised yet (first plan cycle, before the main clipping config block runs). + clipping_start_override = getattr(self, "clipping_buffer_start", None) + if clipping_start_override is None: + start_time_str = self.get_arg("clipping_buffer_start_time", default="None") + if start_time_str and start_time_str != "None": + clipping_start_override = self.time_to_minutes(start_time_str) + self.clipping_buffer_start = clipping_start_override + if clipping_start_override is not None: + early_start = midnight + clipping_start_override morning_start = min(morning_start, early_start) morning_start = max(self.minutes_now, morning_start) @@ -1242,8 +1249,12 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): if has_clearsky: pv_forecast_peak_step = {k: max(v, pv_clearsky_step.get(k, 0)) for k, v in pv_forecast_peak_step.items()} + self.clipping_mode = "Dynamic ClearSky" elif effective_amplification != 1.0: pv_forecast_peak_step = {k: v * effective_amplification for k, v in pv_forecast_peak_step.items()} + self.clipping_mode = "Amplified ({}x)".format(dp2(effective_amplification)) + else: + self.clipping_mode = "Base Forecast" # Apply start/end offsets to widen the clipping risk window start_offset = int(getattr(self, "clipping_buffer_start_offset", 0)) @@ -3794,7 +3805,7 @@ def optimise_all_windows(self, best_metric, metric_keep, debug_mode=False): best_metric, best_cost, best_keep, best_cycle, best_carbon, best_import = self.tweak_plan(self.end_record, best_metric, best_keep) # Export more solar - enable freeze export on idle solar windows if it doesn't cost too much - if self.export_more_solar: + if self.export_more_solar and not self.clipping_buffer_enable: best_metric, best_cost, best_keep, best_cycle, best_carbon, best_import = self.optimise_solar(best_metric, best_cost, best_keep, best_cycle, best_carbon, best_import, record_export_windows, debug_mode=debug_mode) self.plan_write_debug(debug_mode, "plan_raw.html", self.pv_forecast_minute_step, self.pv_forecast_minute10_step, self.load_minutes_step, self.load_minutes_step10, self.end_record) diff --git a/apps/predbat/prediction.py b/apps/predbat/prediction.py index 53eea67d3..06eb66623 100644 --- a/apps/predbat/prediction.py +++ b/apps/predbat/prediction.py @@ -850,10 +850,12 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi battery_to_max = max(soc_max - soc, 0) * battery_loss discharge_min = reserve + is_anti_clipping = False if export_window_active: discharge_min = max(soc_max * export_limit_now / 100.0, reserve, self.best_soc_min) + is_anti_clipping = "clipping_target_soc_pct" in export_window[export_window_n] - if not set_export_freeze_only and export_window_active and export_limit_now < 99.0: + if (not set_export_freeze_only or is_anti_clipping) and export_window_active and export_limit_now < 99.0 and (soc > discharge_min or is_anti_clipping): # Discharge enable, capped at export limit if self.set_export_low_power: export_rate_adjust = 1 - (export_limit_now - int(export_limit_now)) From 66f5e9ac8508a34c53d49cc8386058d984080493 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 29 Jun 2026 09:14:04 +0100 Subject: [PATCH 63/74] Fix simulation ignoring charge and export windows due to 5-minute time alignment mismatch --- apps/predbat/prediction.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/predbat/prediction.py b/apps/predbat/prediction.py index 06eb66623..1dc1da3c8 100644 --- a/apps/predbat/prediction.py +++ b/apps/predbat/prediction.py @@ -627,8 +627,9 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi best_soc_keep = max(best_soc_keep, min(alert_keep / 100.0 * soc_max, soc_max)) # Find charge & discharge windows - charge_window_n = charge_window_optimised.get(minute_absolute, -1) - export_window_n = export_window_optimised.get(minute_absolute, -1) + minute_absolute_aligned = int(minute_absolute / step) * step + charge_window_n = charge_window_optimised.get(minute_absolute_aligned, -1) + export_window_n = export_window_optimised.get(minute_absolute_aligned, -1) charge_window_active = charge_window_n >= 0 export_window_active = export_window_n >= 0 export_limit_now = export_limits[export_window_n] if export_window_active else 100.0 From df550f16739ab859d14eadeafb806ea43774af3b Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 29 Jun 2026 09:22:44 +0100 Subject: [PATCH 64/74] Fix first-cycle clipping window injection by relocating call site after clipping_buffer_forecast_kwh is calculated --- apps/predbat/plan.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index 28c09132f..62e60d7c2 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -1084,9 +1084,6 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): # Pre-fill best export enable with Off self.export_limits_best = [100.0 for i in range(len(self.export_window_best))] - # Inject export windows to create headroom for clipping peaks - self.inject_clipping_export_windows() - self.end_record = self.forecast_minutes # Show best windows self.log("Best charge window {}".format(self.window_as_text(self.charge_window_best, calc_percent_limit(self.charge_limit_best, self.soc_max)))) @@ -1342,6 +1339,9 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): if self.clipping_buffer_kwh > 0: self.clipping_remaining_today = max(self.clipping_remaining_today, self.clipping_buffer_kwh) + # Inject export windows to create headroom for clipping peaks + self.inject_clipping_export_windows() + # Save step data for debug self.load_minutes_step = load_minutes_step self.load_minutes_step10 = load_minutes_step10 From 631596272b6d521c6861c16a9dde187d7014b950 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 29 Jun 2026 10:03:00 +0100 Subject: [PATCH 65/74] clipping: implement dynamic eco/export switching during anti-clipping windows --- apps/predbat/execute.py | 2 +- apps/predbat/prediction.py | 2 +- apps/predbat/tests/test_execute.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/predbat/execute.py b/apps/predbat/execute.py index f5236085c..3164bf1c6 100644 --- a/apps/predbat/execute.py +++ b/apps/predbat/execute.py @@ -348,7 +348,7 @@ def execute_plan(self): self.log("Next export window will be: {} - {} at reserve {}".format(discharge_start_time, discharge_end_time, self.export_limits_best[0])) if (self.minutes_now >= minutes_start) and (self.minutes_now < minutes_end) and (self.export_limits_best[0] < 100.0): is_anti_clipping = self.export_window_best[0].get("clipping_target_soc_pct") is not None - if (not self.set_export_freeze_only or is_anti_clipping) and self.export_limits_best[0] < 99.0 and (self.soc_kw > discharge_soc or is_anti_clipping): + if (not self.set_export_freeze_only or is_anti_clipping) and self.export_limits_best[0] < 99.0 and (self.soc_kw > discharge_soc): if self.set_export_low_power: export_rate_adjust = 1 - (self.export_limits_best[0] - int(self.export_limits_best[0])) else: diff --git a/apps/predbat/prediction.py b/apps/predbat/prediction.py index 1dc1da3c8..30e0872c3 100644 --- a/apps/predbat/prediction.py +++ b/apps/predbat/prediction.py @@ -856,7 +856,7 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi discharge_min = max(soc_max * export_limit_now / 100.0, reserve, self.best_soc_min) is_anti_clipping = "clipping_target_soc_pct" in export_window[export_window_n] - if (not set_export_freeze_only or is_anti_clipping) and export_window_active and export_limit_now < 99.0 and (soc > discharge_min or is_anti_clipping): + if (not set_export_freeze_only or is_anti_clipping) and export_window_active and export_limit_now < 99.0 and (soc > discharge_min): # Discharge enable, capped at export limit if self.set_export_low_power: export_rate_adjust = 1 - (export_limit_now - int(export_limit_now)) diff --git a/apps/predbat/tests/test_execute.py b/apps/predbat/tests/test_execute.py index fe6aa9070..346cae834 100644 --- a/apps/predbat/tests/test_execute.py +++ b/apps/predbat/tests/test_execute.py @@ -2441,7 +2441,7 @@ def run_execute_tests(my_predbat): assert_force_export=True, set_charge_window=True, set_export_window=True, - soc_kw=7.9, + soc_kw=8.5, soc_max=10.0, assert_status="Exporting", assert_reserve=80, From 95984c3b1cd6b7474c4ae37472c2f4a498f16ec2 Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 2 Jul 2026 16:47:18 +0100 Subject: [PATCH 66/74] Fix UI reporting bug for overlapping windows --- apps/predbat/output.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/apps/predbat/output.py b/apps/predbat/output.py index 9fbc0426b..9286523d5 100644 --- a/apps/predbat/output.py +++ b/apps/predbat/output.py @@ -1251,6 +1251,15 @@ def publish_html_plan(self, pv_forecast_minute_step, pv_forecast_minute_step10, had_state = False + # PHYSICS ENGINE PRIORITY FIX: + # In prediction.py, a real export window (limit < 99) overrides a charge window. + # If both are active, suppress the charge window in the UI so we don't display a fake "Chrg" + # or a broken split-cell. + if charge_window_n >= 0 and export_window_n >= 0: + exp_limit = self.export_limits_best[export_window_n] + if exp_limit < 99.0: + charge_window_n = -1 + if charge_window_n >= 0: limit = self.charge_limit_best[charge_window_n] target = limit From d8483ef9baca9fb2046e73f9075118d4ad092986 Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 2 Jul 2026 16:52:54 +0100 Subject: [PATCH 67/74] Fix Windows unicode encoding in test_fill_load_from_power --- .../tests/test_fill_load_from_power.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/apps/predbat/tests/test_fill_load_from_power.py b/apps/predbat/tests/test_fill_load_from_power.py index 46c5701df..d06a7f437 100644 --- a/apps/predbat/tests/test_fill_load_from_power.py +++ b/apps/predbat/tests/test_fill_load_from_power.py @@ -82,7 +82,7 @@ def test_fill_load_from_power_basic(): total_consumption = result[0] - result[59] assert abs(total_consumption - 3.0) < 0.2, f"Total consumption should be near 3.0 kWh, got {dp4(total_consumption)} kWh" - print(f"✓ Two 30-minute periods (0-59): {dp4(result[0])} -> {dp4(result[59])}, consumption: {dp4(total_consumption)} kWh") + print(f"PASS: Two 30-minute periods (0-59): {dp4(result[0])} -> {dp4(result[59])}, consumption: {dp4(total_consumption)} kWh") print("Test 1 PASSED") @@ -119,7 +119,7 @@ def test_fill_load_from_power_no_power_data(): # Check warning was logged assert any("No power data" in msg for msg in fetch.log_messages), "Should log warning about no power data" - print(f"✓ Original data returned: {dp4(result[0])}, {dp4(result[4])}") + print(f"PASS: Original data returned: {dp4(result[0])}, {dp4(result[4])}") print("Test 2 PASSED") @@ -162,7 +162,7 @@ def test_fill_load_from_power_partial_power_data(): total_consumption = result[0] - result[59] assert abs(total_consumption - 4.0) < 0.2, f"Total consumption should be near 4.0 kWh, got {dp4(total_consumption)} kWh" - print(f"✓ Two 30-minute periods: Power in first period, distributed in second") + print(f"PASS: Two 30-minute periods: Power in first period, distributed in second") print(f" {dp4(result[0])} -> {dp4(result[29])} -> {dp4(result[59])}") print("Test 3 PASSED") @@ -197,7 +197,7 @@ def test_fill_load_from_power_single_minute_period(): # Last value should be around 4.0 assert abs(result[9] - 4.0) < 0.1, f"Minute 9: expected ~4.0, got {result[9]}" - print(f"✓ Short span handled: {dp4(result[0])} -> {dp4(result[9])}") + print(f"PASS: Short span handled: {dp4(result[0])} -> {dp4(result[9])}") print("Test 4 PASSED") @@ -225,7 +225,7 @@ def test_fill_load_from_power_zero_load(): for minute in range(0, 30): assert result[minute] == 0.0, f"Minute {minute} should be 0.0, got {result[minute]}" - print("✓ Zero load values preserved") + print("PASS: Zero load values preserved") print("Test 5 PASSED") @@ -301,9 +301,9 @@ def test_fill_load_from_power_backwards_time(): first_hour_consumption = result[0] - result[59] assert abs(first_hour_consumption - 1.5) < 0.5, f"First hour should have ~1.5 kWh consumption, got {dp4(first_hour_consumption)}" - print(f"✓ Backwards time: minute 0 (now) = {dp4(result[0])} kWh") - print(f"✓ Backwards time: minute 60 (1h ago) = {dp4(result[60])} kWh") - print(f"✓ Backwards time: minute 120 (2h ago) = {dp4(result[120])} kWh") + print(f"PASS: Backwards time: minute 0 (now) = {dp4(result[0])} kWh") + print(f"PASS: Backwards time: minute 60 (1h ago) = {dp4(result[60])} kWh") + print(f"PASS: Backwards time: minute 120 (2h ago) = {dp4(result[120])} kWh") print("Test 6 PASSED") @@ -346,7 +346,7 @@ def test_fill_load_from_power_data_extends_beyond_load(): # They may not exist in the result dict at all, which is the correct behaviour. assert 1440 not in result or result[1440] == 0, "Minute 1440 should not be populated from power-only range" - print("✓ No spurious zero-load warning when power data extends past load data range") + print("PASS: No spurious zero-load warning when power data extends past load data range") print("Test 7 PASSED") @@ -430,12 +430,12 @@ def run_all_tests(my_predbat=None): return 0 # Return 0 for success except AssertionError as e: print("\n" + "=" * 60) - print(f"❌ TEST FAILED: {e}") + print(f"FAIL: TEST FAILED: {e}") print("=" * 60) return 1 # Return 1 for failure except Exception as e: print("\n" + "=" * 60) - print(f"❌ ERROR: {e}") + print(f"FAIL: ERROR: {e}") import traceback traceback.print_exc() From 9e9775ba2408a51be26d3143c9b4088bbd30a56c Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 2 Jul 2026 16:53:37 +0100 Subject: [PATCH 68/74] Fix remaining Windows unicode encoding in test_fill_load_from_power --- apps/predbat/tests/test_fill_load_from_power.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/predbat/tests/test_fill_load_from_power.py b/apps/predbat/tests/test_fill_load_from_power.py index d06a7f437..a167e808b 100644 --- a/apps/predbat/tests/test_fill_load_from_power.py +++ b/apps/predbat/tests/test_fill_load_from_power.py @@ -425,7 +425,7 @@ def run_all_tests(my_predbat=None): test_fill_load_from_power_negative_power_corrupts_load_total() print("\n" + "=" * 60) - print("✅ ALL TESTS PASSED") + print("PASS: ALL TESTS PASSED") print("=" * 60) return 0 # Return 0 for success except AssertionError as e: From 731b61ece12eca6f11647a2feb4f6dadccc79a61 Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 2 Jul 2026 17:21:41 +0100 Subject: [PATCH 69/74] Round clipping auto_amp to 2 decimal places --- apps/predbat/plan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index 3a12edb7c..40f9cbba8 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -1183,7 +1183,7 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): if limit > 0: if max_pv_power >= limit * 0.98: - auto_amp = min(2.5, auto_amp + 0.05) + auto_amp = round(min(2.5, auto_amp + 0.05), 2) auto_start_offset = min(60, auto_start_offset + 5) auto_end_offset = min(60, auto_end_offset + 5) self.log( @@ -1192,7 +1192,7 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): ) ) else: - auto_amp = max(1.0, auto_amp - 0.01) + auto_amp = round(max(1.0, auto_amp - 0.01), 2) auto_start_offset = max(0, auto_start_offset - 5) auto_end_offset = max(0, auto_end_offset - 5) self.log("Clipping auto-tuner: No clipping detected. Decreased safety margins - amp: {}, start_offset: {}m, end_offset: {}m".format(auto_amp, auto_start_offset, auto_end_offset)) From 36df5b18c0cc0ff93f6d96509f48d806e2344164 Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 3 Jul 2026 07:37:43 +0100 Subject: [PATCH 70/74] Expose native inverter limit as entity --- apps/predbat/plan.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index 40f9cbba8..f76096df2 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -4621,6 +4621,15 @@ def format_time_human_abs(minute_absolute): "clipping_auto_tune": getattr(self, "clipping_auto_tune", False), }, ) + self.dashboard_item( + self.prefix + ".inverter_limit_watts", + state=dp2(self.inverter_limit), + attributes={ + "friendly_name": "Inverter AC Limit", + "unit_of_measurement": "W", + "icon": "mdi:flash-outline", + }, + ) self.dashboard_item(self.prefix + ".record", state=0.0, attributes={"results": self.filtered_times(record_time), "friendly_name": "Prediction window", "state_class": "measurement"}) self.dashboard_item( self.prefix + ".iboost_best", From 32ab0af8ea59e2f1395097b4a2eb0f8f3b7aa7cf Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 3 Jul 2026 08:18:17 +0100 Subject: [PATCH 71/74] Fix infinite feedback loop in clipping auto-tuner --- apps/predbat/plan.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index f76096df2..979ae516c 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -1182,12 +1182,12 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): limit = self.inverter_limit / MINUTE_WATT if limit > 0: - if max_pv_power >= limit * 0.98: + if limit * 0.98 <= max_pv_power <= limit * 1.02: auto_amp = round(min(2.5, auto_amp + 0.05), 2) auto_start_offset = min(60, auto_start_offset + 5) auto_end_offset = min(60, auto_end_offset + 5) self.log( - "Clipping auto-tuner: Real clipping detected (max PV {} kW >= limit {} kW). Increased safety margins - amp: {}, start_offset: {}m, end_offset: {}m".format( + "Clipping auto-tuner: Real mechanical clipping detected (max PV {} kW flatlined at limit {} kW). Increased safety margins - amp: {}, start_offset: {}m, end_offset: {}m".format( dp2(max_pv_power), dp2(limit), auto_amp, auto_start_offset, auto_end_offset ) ) @@ -1195,7 +1195,7 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): auto_amp = round(max(1.0, auto_amp - 0.01), 2) auto_start_offset = max(0, auto_start_offset - 5) auto_end_offset = max(0, auto_end_offset - 5) - self.log("Clipping auto-tuner: No clipping detected. Decreased safety margins - amp: {}, start_offset: {}m, end_offset: {}m".format(auto_amp, auto_start_offset, auto_end_offset)) + self.log("Clipping auto-tuner: No mechanical clipping detected (max PV {} kW). Decreased safety margins - amp: {}, start_offset: {}m, end_offset: {}m".format(dp2(max_pv_power), auto_amp, auto_start_offset, auto_end_offset)) try: with open(auto_tune_file, "w") as f: From 639d4c3ae670dce2e6b2fc1a4b880f9055ec5780 Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 3 Jul 2026 19:33:44 +0100 Subject: [PATCH 72/74] Fix bug where anti-clipping window drops out of Force Export when SOC reaches target --- apps/predbat/execute.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/predbat/execute.py b/apps/predbat/execute.py index 44dd46f34..0cd04e5e4 100644 --- a/apps/predbat/execute.py +++ b/apps/predbat/execute.py @@ -352,7 +352,7 @@ def execute_plan(self): self.log("Next export window will be: {} - {} at reserve {}".format(discharge_start_time, discharge_end_time, self.export_limits_best[0])) if (self.minutes_now >= minutes_start) and (self.minutes_now < minutes_end) and (self.export_limits_best[0] < 100.0): is_anti_clipping = self.export_window_best[0].get("clipping_target_soc_pct") is not None - if (not self.set_export_freeze_only or is_anti_clipping) and self.export_limits_best[0] < 99.0 and (self.soc_kw > discharge_soc): + if (not self.set_export_freeze_only or is_anti_clipping) and self.export_limits_best[0] < 99.0 and (self.soc_kw > discharge_soc or is_anti_clipping): if self.set_export_low_power: export_rate_adjust = 1 - (self.export_limits_best[0] - int(self.export_limits_best[0])) else: From d26416dc280eb09b5264fb77262cb18072d14148 Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 3 Jul 2026 20:02:17 +0100 Subject: [PATCH 73/74] Fix clipping window deletion bug in plan.py; revert execute.py to restore Eco mode --- apps/predbat/execute.py | 2 +- apps/predbat/plan.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/apps/predbat/execute.py b/apps/predbat/execute.py index 0cd04e5e4..44dd46f34 100644 --- a/apps/predbat/execute.py +++ b/apps/predbat/execute.py @@ -352,7 +352,7 @@ def execute_plan(self): self.log("Next export window will be: {} - {} at reserve {}".format(discharge_start_time, discharge_end_time, self.export_limits_best[0])) if (self.minutes_now >= minutes_start) and (self.minutes_now < minutes_end) and (self.export_limits_best[0] < 100.0): is_anti_clipping = self.export_window_best[0].get("clipping_target_soc_pct") is not None - if (not self.set_export_freeze_only or is_anti_clipping) and self.export_limits_best[0] < 99.0 and (self.soc_kw > discharge_soc or is_anti_clipping): + if (not self.set_export_freeze_only or is_anti_clipping) and self.export_limits_best[0] < 99.0 and (self.soc_kw > discharge_soc): if self.set_export_low_power: export_rate_adjust = 1 - (self.export_limits_best[0] - int(self.export_limits_best[0])) else: diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index 979ae516c..120ba0f57 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -973,9 +973,14 @@ def intersects(w, start, end): morning_start = int(morning_start / 30) * 30 # Align to nearest 30 mins - if morning_start >= peak_start: + if morning_start >= peak_end: continue + # Ensure the window covers the remaining duration of the clipping event + morning_start = min(morning_start, peak_start) + morning_start = max(self.minutes_now, morning_start) + morning_start = int(morning_start / 30) * 30 # Align to nearest 30 mins + # Calculate the precise export limit (Target SOC percentage) required to create this headroom target_soc_kwh = max(0.0, self.soc_max - total_kwh_loss) target_soc_pct = max(0.0, min(100.0, target_soc_kwh / self.soc_max * 100.0)) From 39e2eb04045ea1087e47919437317497267f6341 Mon Sep 17 00:00:00 2001 From: Robin Date: Sat, 4 Jul 2026 11:51:56 +0100 Subject: [PATCH 74/74] Fix unit mismatch bug in clipping penalty calculation that divided the penalty by 12x, allowing negative grid rates to override clipping buffers --- apps/predbat/prediction.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/predbat/prediction.py b/apps/predbat/prediction.py index 5cefc7e90..fca1f0ed5 100644 --- a/apps/predbat/prediction.py +++ b/apps/predbat/prediction.py @@ -737,7 +737,7 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi unmitigated_clip = max(potential_clip - absorbable, 0) if unmitigated_clip > 0: - clipping_penalty = unmitigated_clip * (step / 60.0) * export_rate * clipping_cost_weight + clipping_penalty = unmitigated_clip * export_rate * clipping_cost_weight metric += clipping_penalty clipping_penalty_total += clipping_penalty