Feature: Automated Solar Clipping Mitigation (using native clear-sky models)

.cspell/custom-dictionary-workspace.txt

@@ -57,6 +57,7 @@ chargehold
 chargelater
 Chrg
 citem
+clearsky
 Codespaces
 collapsable
 compareform
@@ -284,6 +285,7 @@ oncharge
 oninput
 onmouseout
 onmouseover
+openmeteo
 openweathermap
 overfitting
 overvoltage

.gitignore

@@ -108,3 +108,8 @@ it
 
 # Runtime-generated manifest
 apps/predbat/manifest.yaml
+
+# Local development and scratch files
+scratch/
+cases/
+apps_ha.yaml

AGENTS.md

@@ -125,3 +125,10 @@ mkdocs serve   # Live preview at http://localhost:8000
 ```
 
 When adding a new doc page, add it to `mkdocs.yml`. The published site at <https://springfall2008.github.io/batpred/> is built automatically from `main` via GitHub Actions.
+
+## Anti-Clipping Requirements
+
+When developing features, fixing bugs, or reviewing PRs related to the `clipping-cloud-model` or legacy `clipping-buffer` branches, **you MUST always consult the requirements validation document**:
+`../clipping_requirements_validation.md` (Note: This is located one directory above the repository root in the `Tech Projects` folder).
+
+This document serves as the absolute ground truth for how the clipping buffer prediction, generation overlay, dynamic SOC targeting, and freeze-export bypassing logic should behave. Ensure any changes align perfectly with the `REQ-xx` matrix.

apps/predbat/components.py

@@ -108,11 +108,15 @@
             "pv_forecast_tomorrow": {"required": False, "config": "pv_forecast_tomorrow"},
             "pv_forecast_d3": {"required": False, "config": "pv_forecast_d3"},
             "pv_forecast_d4": {"required": False, "config": "pv_forecast_d4"},
+            "pv_clearsky_today": {"required": False, "config": "pv_clearsky_today"},
+            "pv_clearsky_tomorrow": {"required": False, "config": "pv_clearsky_tomorrow"},
+            "pv_clearsky_d3": {"required": False, "config": "pv_clearsky_d3"},
+            "pv_clearsky_d4": {"required": False, "config": "pv_clearsky_d4"},
             "pv_scaling": {"required": False, "config": "pv_scaling", "default": 1.0},
             "open_meteo_forecast": {"required": False, "config": "open_meteo_forecast", "default": False},
             "open_meteo_forecast_max_age": {"required": False, "config": "open_meteo_forecast_max_age", "default": 4},
         },
-        "required_or": ["solcast_api_key", "forecast_solar", "pv_forecast_today", "open_meteo_forecast"],
+        "required_or": ["solcast_api_key", "forecast_solar", "pv_forecast_today", "open_meteo_forecast", "pv_clearsky_today"],
         "phase": 2,  # Solar component moved to phase 2 so that any Predbat cloud components (such as GEcloud) have been started and initialised pv_today, etc
     },
     "gecloud": {

apps/predbat/config.py

@@ -487,6 +487,120 @@
         "default": True,
         "enable": "expert_mode",
     },
+    {
+        "name": "clipping_buffer_enable",
+        "friendly_name": "Clipping Buffer Enable",
+        "type": "switch",
+        "default": False,
+        "icon": "mdi:chart-bell-curve-cumulative",
+    },
+    {
+        "name": "clipping_clearsky_source",
+        "friendly_name": "Clipping Clear-Sky Source",
+        "type": "select",
+        "options": ["auto", "ha_solcast_clearsky", "solcast_api", "openmeteo"],
+        "icon": "mdi:cloud-search",
+        "default": "auto",
+        "enable": "clipping_buffer_enable",
+    },
+    {
+        "name": "clipping_auto_tune",
+        "friendly_name": "Clipping Auto-Tune",
+        "type": "switch",
+        "default": True,
+        "icon": "mdi:auto-fix",
+        "enable": "clipping_buffer_enable",
+    },
+    {
+        "name": "clipping_cost_weight",
+        "friendly_name": "Clipping Cost Weight",
+        "type": "input_number",
+        "min": 0,
+        "max": 10.0,
+        "step": 0.1,
+        "unit": "x",
+        "icon": "mdi:multiplication",
+        "enable": "clipping_buffer_enable",
+        "default": 1.0,
+    },
+    {
+        "name": "clipping_amplification",
+        "friendly_name": "Clipping Amplification",
+        "type": "input_number",
+        "min": 0.5,
+        "max": 3.0,
+        "step": 0.1,
+        "unit": "x",
+        "icon": "mdi:arrow-expand-vertical",
+        "enable": "clipping_buffer_enable",
+        "default": 1.0,
+    },
+    {
+        "name": "clipping_buffer_start_offset",
+        "friendly_name": "Clipping Buffer Start Offset",
+        "type": "input_number",
+        "min": 0,
+        "max": 120,
+        "step": 5,
+        "unit": "minutes",
+        "icon": "mdi:clock-start",
+        "default": 0,
+        "enable": "clipping_buffer_enable",
+    },
+    {
+        "name": "clipping_buffer_end_offset",
+        "friendly_name": "Clipping Buffer End Offset",
+        "type": "input_number",
+        "min": 0,
+        "max": 120,
+        "step": 5,
+        "unit": "minutes",
+        "icon": "mdi:clock-end",
+        "default": 0,
+        "enable": "clipping_buffer_enable",
+    },
+    {
+        "name": "clipping_limit_override",
+        "friendly_name": "Clipping Limit Override",
+        "type": "input_number",
+        "min": 0,
+        "max": 50000,
+        "step": 100,
+        "unit": "W",
+        "icon": "mdi:flash-alert",
+        "enable": "clipping_buffer_enable",
+        "default": 0,
+    },
+    {
+        "name": "clipping_buffer_max_kwh",
+        "friendly_name": "Clipping Buffer Max kWh (Manual Override)",
+        "type": "input_number",
+        "min": 0,
+        "max": 50.0,
+        "step": 0.1,
+        "unit": "kWh",
+        "icon": "mdi:battery-50",
+        "default": 0,
+        "enable": "clipping_buffer_enable",
+    },
+    {
+        "name": "clipping_buffer_start_time",
+        "friendly_name": "Clipping Buffer Start Time",
+        "type": "select",
+        "options": ["None"] + OPTIONS_TIME,
+        "icon": "mdi:clock-start",
+        "default": "None",
+        "enable": "clipping_buffer_enable",
+    },
+    {
+        "name": "clipping_buffer_end_time",
+        "friendly_name": "Clipping Buffer End Time",
+        "type": "select",
+        "options": ["None"] + OPTIONS_TIME,
+        "icon": "mdi:clock-end",
+        "default": "None",
+        "enable": "clipping_buffer_enable",
+    },
     {
         "name": "metric_pv_calibration_enable",
         "friendly_name": "Enable use of Calibrated PV data",
@@ -2260,4 +2374,18 @@
     "gateway_mqtt_host": {"type": "string", "empty": False},
     "gateway_mqtt_port": {"type": "integer", "zero": False},
     "gateway_mqtt_token": {"type": "string", "empty": False},
+    "clipping_buffer_enable": {"type": "boolean"},
+    "clipping_use_clearsky_peaks": {"type": "boolean"},
+    "clipping_cost_weight": {"type": "float"},
+    "clipping_amplification": {"type": "float"},
+    "clipping_buffer_start_offset": {"type": "integer"},
+    "clipping_buffer_end_offset": {"type": "integer"},
+    "clipping_limit_override": {"type": "integer"},
+    "clipping_buffer_max_kwh": {"type": "float"},
+    "clipping_buffer_start_time": {"type": "string"},
+    "clipping_buffer_end_time": {"type": "string"},
+    "pv_clearsky_today": {"type": "sensor", "sensor_type": "float"},
+    "pv_clearsky_tomorrow": {"type": "sensor", "sensor_type": "float"},
+    "pv_clearsky_d3": {"type": "sensor", "sensor_type": "float"},
+    "pv_clearsky_d4": {"type": "sensor", "sensor_type": "float"},
 }

apps/predbat/config/apps.yaml

@@ -270,7 +270,6 @@ pred_bat:
   # Only use this for workarounds if your inverter time is correct but Predbat is somehow wrong (AppDaemon issue)
   # 1 means add 1 minute to AppDaemon time, -1 takes it away
   clock_skew: 0
-
   # Solcast cloud interface, set this or the local interface below
   #solcast_host: 'https://api.solcast.com.au/'
   #solcast_api_key: !secret solcast_api_key
@@ -283,7 +282,25 @@ pred_bat:
   #  azimuth: 180
   #  declination: 10
   #  api_key: !secret forecast_solar_api_key
-  #  azimuth_zero_south: false  # Set to true if azimuth is already in Forecast.solar/Open-Meteo convention (0=South) rather than Predbat convention (0=North)
+
+  # Open-Meteo interface (used for primary PV forecast OR for ClearSky baseline)
+  #open_meteo_forecast:
+  #  - latitude: 55.9711067
+  #    longitude: -3.1925157
+  #    declination: 35
+  #    azimuth: 180
+  #    kwp: 4.0
+
+  # Determines which source Predbat uses for the primary PV forecast.
+  # Options: "auto" (default), "ha", "forecast_solar", "openmeteo", "solcast_api"
+  # If you are using Open-Meteo ONLY for the ClearSky baseline below, it's recommended to explicitly set this to "ha" if using the HA integration.
+  #pv_forecast_primary: "auto"
+
+  # --- Automated Solar Clipping Mitigation ---
+  # Enable clipping mitigation to automatically force exports prior to peaks
+  #clipping_buffer_enable: True
+  # Set the source for the theoretical ClearSky maximum (e.g. "openmeteo", "ha_solcast_clearsky", "solcast_api")
+  #clipping_clearsky_source: "ha_solcast_clearsky"
 
   # Set these to match solcast sensor names if not using the cloud interface
   # The regular expression (re:) makes the solcast bit optional
@@ -293,6 +310,12 @@ pred_bat:
   pv_forecast_d3: re:(sensor.(solcast_|)(pv_forecast_|)forecast_(day_3|d3))
   pv_forecast_d4: re:(sensor.(solcast_|)(pv_forecast_|)forecast_(day_4|d4))
 
+  # ClearSky sensors (Required if clipping_clearsky_source is set to ha_solcast_clearsky)
+  #pv_clearsky_today: re:(sensor.(solcast_|)(pv_forecast_|)clearsky_today)
+  #pv_clearsky_tomorrow: re:(sensor.(solcast_|)(pv_forecast_|)clearsky_tomorrow)
+  #pv_clearsky_d3: re:(sensor.(solcast_|)(pv_forecast_|)clearsky_(day_3|d3))
+  #pv_clearsky_d4: re:(sensor.(solcast_|)(pv_forecast_|)clearsky_(day_4|d4))
+
   # Ohme EV charger cloud direct integration
   #ohme_login: !secret ohme_login
   #ohme_password: !secret ohme_password

apps/predbat/execute.py

@@ -351,7 +351,8 @@ def execute_plan(self):
                 discharge_soc = max((int(self.export_limits_best[0]) * self.soc_max) / 100.0, self.reserve, self.best_soc_min)
                 self.log("Next export window will be: {} - {} at reserve {}".format(discharge_start_time, discharge_end_time, self.export_limits_best[0]))
                 if (self.minutes_now >= minutes_start) and (self.minutes_now < minutes_end) and (self.export_limits_best[0] < 100.0):
-                    if not self.set_export_freeze_only and self.export_limits_best[0] < 99.0 and (self.soc_kw > discharge_soc):
+                    is_anti_clipping = self.export_window_best[0].get("clipping_target_soc_pct") is not None
+                    if (not self.set_export_freeze_only or is_anti_clipping) and self.export_limits_best[0] < 99.0 and (self.soc_kw > discharge_soc):
                         if self.set_export_low_power:
                             export_rate_adjust = 1 - (self.export_limits_best[0] - int(self.export_limits_best[0]))
                         else:
@@ -362,6 +363,11 @@ def execute_plan(self):
                         inverter.adjust_discharge_rate(inverter.battery_rate_max_export * export_rate_adjust * MINUTE_WATT)
                         resetDischarge = False
                         inverter.adjust_force_export(True, discharge_start_time, discharge_end_time)
+
+                        if self.set_reserve_enable and is_anti_clipping:
+                            inverter.adjust_reserve(int(self.export_limits_best[0]))
+                            resetReserve = False
+
                         if inverter.inv_charge_discharge_with_rate:
                             inverter.adjust_charge_rate(0)
                             resetCharge = False

apps/predbat/fetch.py

@@ -593,7 +593,7 @@ def minute_data_import_export(self, max_days_previous, now_utc, key, scale=1.0,
             if history and len(history) > 0 and len(history[0]) > 0:
                 item = history[0][0]
                 try:
-                    last_updated_time = str2time(item["last_updated"])
+                    last_updated_time = str2time(item.get("last_updated", item.get("state_class", "")))
                 except (ValueError, TypeError):
                     last_updated_time = now_utc
                 age_days = max_days_previous
@@ -660,7 +660,7 @@ def minute_data_load(self, now_utc, entity_name, max_days_previous, load_scaling
             if isinstance(history, list) and history and history[0]:
                 item = history[0][0]
                 try:
-                    last_updated_time = str2time(item["last_updated"])
+                    last_updated_time = str2time(item.get("last_updated", item.get("state_class", "")))
                 except (ValueError, TypeError):
                     last_updated_time = now_utc
                 age = now_utc - last_updated_time
@@ -1056,7 +1056,7 @@ def fetch_sensor_data(self, save=True):
             self.cost_today_sofar, self.carbon_today_sofar = self.today_cost(self.import_today, self.export_today, self.car_charging_energy, self.load_minutes, save=save)
 
         # Fetch PV forecast if enabled, today must be enabled, other days are optional
-        self.pv_forecast_minute, self.pv_forecast_minute10 = self.fetch_pv_forecast()
+        self.pv_forecast_minute, self.pv_forecast_minute10, self.pv_forecast_minute90, self.pv_forecast_minuteCS, self.pv_forecast_minuteMAX = self.fetch_pv_forecast()
 
         if self.load_minutes and not self.load_forecast_only and not self.load_forecast_history:
             # Apply modal filter to historical data. Skipped for days_previous_auto: the weighted-bucket
@@ -1282,11 +1282,21 @@ def fetch_pv_forecast(self):
         """
         pv_forecast_minute = {}
         pv_forecast_minute10 = {}
+        pv_forecast_minute90 = {}
+        pv_forecast_minuteCS = {}
+        pv_forecast_minuteHIST = {}
 
         # Get data from forecast sensor
         entity_id = "sensor." + self.prefix + "_pv_forecast_raw"
         pv_forecast_packed_ld = self.get_state_wrapper(entity_id=entity_id, attribute="forecast")
         pv_forecast10_packed_ld = self.get_state_wrapper(entity_id=entity_id, attribute="forecast10")
+        pv_forecast90_packed_ld = self.get_state_wrapper(entity_id=entity_id, attribute="forecast90")
+        pv_forecastCS_packed_ld = self.get_state_wrapper(entity_id=entity_id, attribute="forecast_clearsky")
+        pv_forecastHIST_packed_ld = self.get_state_wrapper(entity_id=entity_id, attribute="forecast_historical")
+        if pv_forecastCS_packed_ld is None:
+            pv_forecastCS_packed_ld = self.get_state_wrapper(entity_id=entity_id, attribute="forecastCS")
+        if pv_forecastHIST_packed_ld is None:
+            pv_forecastHIST_packed_ld = self.get_state_wrapper(entity_id=entity_id, attribute="forecastMAX")
         relative_time = self.get_state_wrapper(entity_id=entity_id, attribute="relative_time")
         try:
             relative_time = datetime.strptime(relative_time, TIME_FORMAT)
@@ -1297,6 +1307,9 @@ def fetch_pv_forecast(self):
         # Convert keys to integers and values to floats
         pv_forecast_packed = {}
         pv_forecast10_packed = {}
+        pv_forecast90_packed = {}
+        pv_forecastCS_packed = {}
+        pv_forecastHIST_packed = {}
 
         if pv_forecast_packed_ld:
             for key, value in pv_forecast_packed_ld.items():
@@ -1314,10 +1327,37 @@ def fetch_pv_forecast(self):
                 except (ValueError, TypeError):
                     pass
 
+        if pv_forecast90_packed_ld:
+            for key, value in pv_forecast90_packed_ld.items():
+                try:
+                    minute = int(key)
+                    pv_forecast90_packed[minute] = float(value)
+                except (ValueError, TypeError):
+                    pass
+
+        if pv_forecastCS_packed_ld:
+            for key, value in pv_forecastCS_packed_ld.items():
+                try:
+                    minute = int(key)
+                    pv_forecastCS_packed[minute] = float(value)
+                except (ValueError, TypeError):
+                    pass
+
+        if pv_forecastHIST_packed_ld:
+            for key, value in pv_forecastHIST_packed_ld.items():
+                try:
+                    minute = int(key)
+                    pv_forecastHIST_packed[minute] = float(value)
+                except (ValueError, TypeError):
+                    pass
+
         # Unpack the forecast data
         max_minute = max(pv_forecast_packed.keys()) if pv_forecast_packed else 0
         last_value = 0
         last_value10 = 0
+        last_value90 = 0
+        last_valueCS = 0
+        last_valueHIST = 0
         # The forecast could be for a different time to our relative time, so we need to offset the minutes to align with our midnight_utc.
         # relative_time is the midnight at which the forecast was saved, so stored minute keys are relative to that midnight.
         # We subtract the offset so that stored minute X (= relative_time + X) maps to (relative_time + X - midnight_utc) minutes from today's midnight.
@@ -1326,10 +1366,16 @@ def fetch_pv_forecast(self):
             target_minute = minute - minute_offset
             last_value = pv_forecast_packed.get(minute, last_value)
             last_value10 = pv_forecast10_packed.get(minute, last_value10)
+            last_value90 = pv_forecast90_packed.get(minute, last_value90)
+            last_valueCS = pv_forecastCS_packed.get(minute, last_valueCS)
+            last_valueHIST = pv_forecastHIST_packed.get(minute, last_valueHIST)
             pv_forecast_minute[target_minute] = last_value
             pv_forecast_minute10[target_minute] = last_value10
+            pv_forecast_minute90[target_minute] = last_value90
+            pv_forecast_minuteCS[target_minute] = last_valueCS
+            pv_forecast_minuteHIST[target_minute] = last_valueHIST
 
-        return pv_forecast_minute, pv_forecast_minute10
+        return pv_forecast_minute, pv_forecast_minute10, pv_forecast_minute90, pv_forecast_minuteCS, pv_forecast_minuteHIST
 
     def predict_battery_temperature(self, battery_temperature_history, step):
         """
@@ -2486,6 +2532,15 @@ def fetch_config_options(self):
         self.carbon_enable = self.get_arg("carbon_enable")
         self.carbon_metric = self.get_arg("carbon_metric")
 
+        # Clipping peak cost penalty model
+        self.clipping_buffer_enable = self.get_arg("clipping_buffer_enable")
+        self.clipping_auto_tune = self.get_arg("clipping_auto_tune")
+        self.clipping_cost_weight = self.get_arg("clipping_cost_weight")
+        self.clipping_amplification = self.get_arg("clipping_amplification")
+        self.clipping_buffer_start_offset = self.get_arg("clipping_buffer_start_offset", default=0)
+        self.clipping_buffer_end_offset = self.get_arg("clipping_buffer_end_offset", default=0)
+        self.clipping_limit_override = self.get_arg("clipping_limit_override") / MINUTE_WATT if self.get_arg("clipping_limit_override") else 0
+
         # iBoost solar diverter model
         self.iboost_enable = self.get_arg("iboost_enable")
         self.iboost_gas = self.get_arg("iboost_gas")