From f57a349660eb66742409f89429e809e36a5fedb4 Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Wed, 19 Aug 2026 15:03:25 -0700 Subject: [PATCH 1/6] init --- .../src/app/jobs/sfms_daily_actuals.py | 7 + .../src/app/jobs/sfms_daily_forecasts.py | 7 + .../app/tests/jobs/test_sfms_daily_actuals.py | 20 ++ .../tests/jobs/test_sfms_daily_forecasts.py | 20 ++ .../processors/foliar_moisture_content.py | 182 ++++++++++ .../wps-sfms/src/wps_sfms/raster_inputs.py | 11 + .../src/wps_sfms/sfmsng_raster_addresser.py | 29 +- .../tests/test_foliar_moisture_content.py | 328 ++++++++++++++++++ .../tests/test_sfmsng_raster_addresser.py | 23 +- 9 files changed, 624 insertions(+), 3 deletions(-) create mode 100644 backend/packages/wps-sfms/src/wps_sfms/processors/foliar_moisture_content.py create mode 100644 backend/packages/wps-sfms/src/wps_sfms/tests/test_foliar_moisture_content.py diff --git a/backend/packages/wps-api/src/app/jobs/sfms_daily_actuals.py b/backend/packages/wps-api/src/app/jobs/sfms_daily_actuals.py index 8dc2f42b2d..341c18b63b 100644 --- a/backend/packages/wps-api/src/app/jobs/sfms_daily_actuals.py +++ b/backend/packages/wps-api/src/app/jobs/sfms_daily_actuals.py @@ -13,6 +13,7 @@ from datetime import datetime, timezone from aiohttp import ClientSession +from wps_sfms.processors.foliar_moisture_content import ensure_fmc_rasters from wps_sfms.sfmsng_raster_addresser import SFMSNGRasterAddresser from wps_shared.chatops_notification import send_chatops_notification from wps_shared.db.crud.fuel_layer import get_fuel_type_raster_by_year @@ -64,6 +65,12 @@ async def run_sfms_daily_actuals(target_date: datetime) -> None: logger.info("Using reference raster: %s", fuel_raster_path) async with S3Client() as s3_client: + await ensure_fmc_rasters( + [datetime_to_process.date()], + raster_addresser, + s3_client, + ) + # Fetch station observations from WF1 async with ClientSession() as session: wfwx_api = WfwxApi(session) diff --git a/backend/packages/wps-api/src/app/jobs/sfms_daily_forecasts.py b/backend/packages/wps-api/src/app/jobs/sfms_daily_forecasts.py index 0fd3d744bc..6200bbe82f 100644 --- a/backend/packages/wps-api/src/app/jobs/sfms_daily_forecasts.py +++ b/backend/packages/wps-api/src/app/jobs/sfms_daily_forecasts.py @@ -13,6 +13,7 @@ from datetime import date, datetime, timedelta, timezone from aiohttp import ClientSession +from wps_sfms.processors.foliar_moisture_content import ensure_fmc_rasters from wps_sfms.sfmsng_raster_addresser import SFMSNGRasterAddresser from wps_shared.chatops_notification import send_chatops_notification from wps_shared.db.crud.fuel_layer import get_fuel_type_raster_by_year @@ -91,6 +92,12 @@ async def run_sfms_daily_forecasts(run_datetime: datetime) -> None: datetimes_to_process = forecast_datetimes(seed_actual_date) async with S3Client() as s3_client: + await ensure_fmc_rasters( + [datetime_to_process.date() for datetime_to_process in datetimes_to_process], + raster_addresser, + s3_client, + ) + missing_actual_seed_keys = await get_missing_fwi_seed_keys( datetimes_to_process[0], raster_addresser, diff --git a/backend/packages/wps-api/src/app/tests/jobs/test_sfms_daily_actuals.py b/backend/packages/wps-api/src/app/tests/jobs/test_sfms_daily_actuals.py index 9842dfbf4f..7f0b32b4bb 100644 --- a/backend/packages/wps-api/src/app/tests/jobs/test_sfms_daily_actuals.py +++ b/backend/packages/wps-api/src/app/tests/jobs/test_sfms_daily_actuals.py @@ -65,6 +65,7 @@ class MockDailyActualsDeps(NamedTuple): interpolation_processor: MagicMock fwi_processor: MagicMock sfc_processor: MagicMock + ensure_fmc_rasters: AsyncMock wfwx_api: MagicMock addresser: MagicMock @@ -120,6 +121,10 @@ async def _read_scope(): mock_addresser = MagicMock() mock_addresser.s3_prefix = "/vsis3/test-bucket" mocker.patch(f"{MODULE_PATH}.SFMSNGRasterAddresser", return_value=mock_addresser) + mock_ensure_fmc_rasters = mocker.patch( + f"{MODULE_PATH}.ensure_fmc_rasters", + new_callable=AsyncMock, + ) # Mock processors mock_temp_processor = MagicMock(spec=TemperatureInterpolator) mock_temp_processor.process = AsyncMock(return_value="sfms/interpolated/2024/07/04/temp.tif") @@ -185,6 +190,7 @@ async def _scope(): interpolation_processor=mock_interpolation_processor, fwi_processor=mock_fwi_processor, sfc_processor=mock_sfc_processor, + ensure_fmc_rasters=mock_ensure_fmc_rasters, wfwx_api=mock_wfwx_api, addresser=mock_addresser, ) @@ -224,6 +230,20 @@ async def test_runs_all_processors(self, mock_dependencies: MockDailyActualsDeps mock_dependencies.wind_direction_processor.process.assert_called_once() mock_dependencies.interpolation_processor.process.assert_called_once() + @pytest.mark.anyio + async def test_ensures_shared_fmc_for_target_date( + self, mock_dependencies: MockDailyActualsDeps + ): + target_date = datetime(2024, 7, 4, hour=10, minute=30, tzinfo=timezone.utc) + + await run_sfms_daily_actuals(target_date) + + mock_dependencies.ensure_fmc_rasters.assert_awaited_once_with( + [target_date.date()], + mock_dependencies.addresser, + mock_dependencies.s3_client, + ) + @pytest.mark.anyio async def test_runs_processors_in_order(self, mock_dependencies: MockDailyActualsDeps): """Test that weather processors run in the expected sequence.""" diff --git a/backend/packages/wps-api/src/app/tests/jobs/test_sfms_daily_forecasts.py b/backend/packages/wps-api/src/app/tests/jobs/test_sfms_daily_forecasts.py index 904d468594..333decfcd6 100644 --- a/backend/packages/wps-api/src/app/tests/jobs/test_sfms_daily_forecasts.py +++ b/backend/packages/wps-api/src/app/tests/jobs/test_sfms_daily_forecasts.py @@ -45,6 +45,7 @@ class MockDailyForecastsDeps(NamedTuple): interpolation_processor: MagicMock fwi_processor: MagicMock sfc_processor: MagicMock + ensure_fmc_rasters: AsyncMock wfwx_api: MagicMock addresser: MagicMock save_sfms_run: AsyncMock @@ -92,6 +93,10 @@ async def _read_scope(): mock_addresser = MagicMock() mock_addresser.s3_prefix = "/vsis3/test-bucket" mocker.patch(f"{MODULE_PATH}.SFMSNGRasterAddresser", return_value=mock_addresser) + mock_ensure_fmc_rasters = mocker.patch( + f"{MODULE_PATH}.ensure_fmc_rasters", + new_callable=AsyncMock, + ) mock_temp_processor = MagicMock(spec=TemperatureInterpolator) mock_temp_processor.process = AsyncMock(return_value="temperature.tif") @@ -148,6 +153,7 @@ async def _write_scope(): interpolation_processor=mock_interpolation_processor, fwi_processor=mock_fwi_processor, sfc_processor=mock_sfc_processor, + ensure_fmc_rasters=mock_ensure_fmc_rasters, wfwx_api=mock_wfwx_api, addresser=mock_addresser, save_sfms_run=mock_save_sfms_run, @@ -224,6 +230,20 @@ async def test_runs_three_days(self, mock_dependencies: MockDailyForecastsDeps): mock_dependencies.get_fuel_type_raster_by_year.assert_awaited_once() assert mock_dependencies.get_fuel_type_raster_by_year.call_args.args[1] == 2024 + @pytest.mark.anyio + async def test_ensures_shared_fmc_for_processed_forecast_dates( + self, mock_dependencies: MockDailyForecastsDeps + ): + target_date = datetime(2024, 7, 5, 0, 45, tzinfo=timezone.utc) + + await run_sfms_daily_forecasts(target_date) + + mock_dependencies.ensure_fmc_rasters.assert_awaited_once_with( + [date(2024, 7, 5), date(2024, 7, 6), date(2024, 7, 7)], + mock_dependencies.addresser, + mock_dependencies.s3_client, + ) + @pytest.mark.anyio async def test_saves_forecast_runs(self, mock_dependencies: MockDailyForecastsDeps): target_date = datetime(2024, 7, 5, 0, 45, tzinfo=timezone.utc) diff --git a/backend/packages/wps-sfms/src/wps_sfms/processors/foliar_moisture_content.py b/backend/packages/wps-sfms/src/wps_sfms/processors/foliar_moisture_content.py new file mode 100644 index 0000000000..624573b2e3 --- /dev/null +++ b/backend/packages/wps-sfms/src/wps_sfms/processors/foliar_moisture_content.py @@ -0,0 +1,182 @@ +"""Raster processor for shared daily Foliar Moisture Content calculations.""" + +import logging +from collections.abc import Iterable +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import date +from time import perf_counter +from typing import Callable, ContextManager, Generator + +import numpy as np +from cffdrs_vec.fbp import vectorized_foliar_moisture_content +from wps_shared.geospatial.geospatial import rasters_match +from wps_shared.geospatial.wps_dataset import WPSDataset, multi_wps_dataset_context +from wps_shared.utils.s3 import gdal_s3_context +from wps_shared.utils.s3_client import S3Client + +from wps_sfms.interpolation.common import SFMS_NO_DATA +from wps_sfms.publish import publish_dataset +from wps_sfms.raster_inputs import FoliarMoistureContentInputs +from wps_sfms.raster_output import create_masked_output_dataset +from wps_sfms.sfmsng_raster_addresser import SFMSNGRasterAddresser + +logger = logging.getLogger(__name__) + +MultiDatasetContext = Callable[[list[str]], ContextManager[list[WPSDataset]]] + + +@dataclass(frozen=True) +class FoliarMoistureContentResult: + values: np.ndarray + nodata_value: float = SFMS_NO_DATA + + +@dataclass(frozen=True) +class FoliarMoistureContentDatasets: + elevation: WPSDataset + latitude: WPSDataset + longitude: WPSDataset + + +def calculate_foliar_moisture_content( + datasets: FoliarMoistureContentDatasets, + target_date: date, +) -> FoliarMoistureContentResult: + """Calculate FMC for one calendar date wherever all static inputs are valid.""" + elevation, _ = datasets.elevation.replace_nodata_with(np.nan) + latitude, _ = datasets.latitude.replace_nodata_with(np.nan) + longitude, _ = datasets.longitude.replace_nodata_with(np.nan) + + calculation_mask = np.isfinite(elevation) & np.isfinite(latitude) & np.isfinite(longitude) + output = np.full(elevation.shape, SFMS_NO_DATA, dtype=np.float32) + if not np.any(calculation_mask): + return FoliarMoistureContentResult(output) + + start = perf_counter() + calculated = vectorized_foliar_moisture_content( + latitude[calculation_mask], + np.abs(longitude[calculation_mask]), + elevation[calculation_mask], + target_date.timetuple().tm_yday, + 0, + ) + logger.info( + "%f seconds to calculate vectorized FMC for %s", + perf_counter() - start, + target_date, + ) + output[calculation_mask] = np.where(np.isfinite(calculated), calculated, SFMS_NO_DATA) + return FoliarMoistureContentResult(output) + + +class FoliarMoistureContentProcessor: + """Load shared static inputs once and publish FMC for one or more dates.""" + + @staticmethod + async def _assert_dependencies_exist( + s3_client: S3Client, + inputs: FoliarMoistureContentInputs, + ) -> None: + dependency_keys = ( + inputs.elevation_key, + inputs.latitude_key, + inputs.longitude_key, + ) + if not await s3_client.all_objects_exist(*dependency_keys): + details = ", ".join(str(key) for key in dependency_keys) + raise RuntimeError(f"Missing FMC dependencies: {details}") + + @contextmanager + def _open_datasets( + self, + input_dataset_context: MultiDatasetContext, + inputs: FoliarMoistureContentInputs, + ) -> Generator[FoliarMoistureContentDatasets, None, None]: + keys = [inputs.elevation_key, inputs.latitude_key, inputs.longitude_key] + with input_dataset_context(keys) as input_datasets: + datasets_by_key = {dataset.ds_path: dataset for dataset in input_datasets} + yield FoliarMoistureContentDatasets( + elevation=datasets_by_key[inputs.elevation_key], + latitude=datasets_by_key[inputs.latitude_key], + longitude=datasets_by_key[inputs.longitude_key], + ) + + @staticmethod + def _validate_grids( + datasets: FoliarMoistureContentDatasets, + inputs: FoliarMoistureContentInputs, + ) -> None: + reference = datasets.elevation.as_gdal_ds() + candidates = ( + ("latitude", inputs.latitude_key, datasets.latitude), + ("longitude", inputs.longitude_key, datasets.longitude), + ) + for label, key, dataset in candidates: + if not rasters_match(dataset.as_gdal_ds(), reference): + raise ValueError( + f"{label} raster does not match the elevation grid: " + f"{key} vs {inputs.elevation_key}" + ) + + async def process( + self, + s3_client: S3Client, + input_dataset_context: MultiDatasetContext, + inputs: FoliarMoistureContentInputs, + ) -> None: + """Calculate and publish every requested FMC date from the shared static inputs.""" + if not inputs.output_keys: + return + + with gdal_s3_context(): + await self._assert_dependencies_exist(s3_client, inputs) + with self._open_datasets(input_dataset_context, inputs) as datasets: + self._validate_grids(datasets, inputs) + + for target_date, output_key in inputs.output_keys.items(): + result = calculate_foliar_moisture_content(datasets, target_date) + with create_masked_output_dataset( + result.values, + datasets.elevation, + result.nodata_value, + ) as output_ds: + output_band = output_ds.as_gdal_ds().GetRasterBand(1) + output_band.SetDescription("foliar_moisture_content") + output_band.SetUnitType("%") + published = await publish_dataset( + s3_client=s3_client, + dataset=output_ds, + output_key=output_key, + ) + + logger.info( + "Stored FMC for %s: %s (COG: %s)", + target_date, + published.output_key, + published.cog_key, + ) + + +async def ensure_fmc_rasters( + target_dates: Iterable[date], + raster_addresser: SFMSNGRasterAddresser, + s3_client: S3Client, +) -> None: + """Publish shared daily FMC rasters for dates without complete GeoTIFF and COG outputs.""" + unique_dates = tuple(dict.fromkeys(target_dates)) + missing_dates = [] + for target_date in unique_dates: + output_key = raster_addresser.get_fmc_key(target_date) + cog_key = raster_addresser.get_cog_key(output_key) + if await s3_client.all_objects_exist(output_key, cog_key): + logger.info("Skipping existing FMC raster for %s: %s", target_date, output_key) + else: + missing_dates.append(target_date) + + if not missing_dates: + return + + inputs = raster_addresser.get_fmc_inputs(missing_dates) + processor = FoliarMoistureContentProcessor() + await processor.process(s3_client, multi_wps_dataset_context, inputs) diff --git a/backend/packages/wps-sfms/src/wps_sfms/raster_inputs.py b/backend/packages/wps-sfms/src/wps_sfms/raster_inputs.py index c9a3c883ed..92de4a0440 100644 --- a/backend/packages/wps-sfms/src/wps_sfms/raster_inputs.py +++ b/backend/packages/wps-sfms/src/wps_sfms/raster_inputs.py @@ -1,6 +1,7 @@ """Typed raster dependency and output contracts for SFMS calculations.""" from dataclasses import dataclass +from datetime import date from typing import Mapping from wps_shared.run_type import RunType @@ -36,3 +37,13 @@ class SurfaceFuelConsumptionInputs: percent_conifer_key: GDALPath output_key: S3Key run_type: RunType + + +@dataclass(frozen=True) +class FoliarMoistureContentInputs: + """Static dependencies and date-specific outputs for shared daily FMC calculations.""" + + elevation_key: GDALPath + latitude_key: GDALPath + longitude_key: GDALPath + output_keys: Mapping[date, S3Key] diff --git a/backend/packages/wps-sfms/src/wps_sfms/sfmsng_raster_addresser.py b/backend/packages/wps-sfms/src/wps_sfms/sfmsng_raster_addresser.py index 71cf6c90e5..3e280983f1 100644 --- a/backend/packages/wps-sfms/src/wps_sfms/sfmsng_raster_addresser.py +++ b/backend/packages/wps-sfms/src/wps_sfms/sfmsng_raster_addresser.py @@ -5,7 +5,8 @@ `sfms/` storage used by RasterKeyAddresser. """ -from datetime import datetime, timedelta +from collections.abc import Iterable +from datetime import date, datetime, timedelta from wps_shared.run_type import RunType from wps_shared.sfms.raster_addresser import ( @@ -18,7 +19,11 @@ ) from wps_shared.utils.time import assert_all_utc -from wps_sfms.raster_inputs import FWIInputs, SurfaceFuelConsumptionInputs +from wps_sfms.raster_inputs import ( + FWIInputs, + FoliarMoistureContentInputs, + SurfaceFuelConsumptionInputs, +) class SFMSNGRasterAddresser(BaseRasterAddresser): @@ -99,6 +104,26 @@ def get_fbp_key( f"{fbp_param.value}_{date_str}.tif" ) + def get_fmc_key(self, target_date: date) -> S3Key: + """S3 key for the shared Foliar Moisture Content raster for one calendar date.""" + date_str = target_date.strftime("%Y%m%d") + return S3Key( + f"{self.root}/static/fmc/{target_date.year:04d}/{target_date.month:02d}/" + f"{target_date.day:02d}/fmc_{date_str}.tif" + ) + + def get_fmc_inputs(self, target_dates: Iterable[date]) -> FoliarMoistureContentInputs: + """Build the static dependencies and output keys for daily FMC calculations.""" + static_root = f"{self.root}/static" + return FoliarMoistureContentInputs( + elevation_key=self.gdal_path(S3Key(f"{static_root}/bc_elevation.tif")), + latitude_key=self.gdal_path(S3Key(f"{static_root}/latitude.tif")), + longitude_key=self.gdal_path(S3Key(f"{static_root}/longitude.tif")), + output_keys={ + target_date: self.get_fmc_key(target_date) for target_date in target_dates + }, + ) + def get_surface_fuel_consumption_inputs( self, datetime_to_process: datetime, diff --git a/backend/packages/wps-sfms/src/wps_sfms/tests/test_foliar_moisture_content.py b/backend/packages/wps-sfms/src/wps_sfms/tests/test_foliar_moisture_content.py new file mode 100644 index 0000000000..53604c22fe --- /dev/null +++ b/backend/packages/wps-sfms/src/wps_sfms/tests/test_foliar_moisture_content.py @@ -0,0 +1,328 @@ +from contextlib import contextmanager +from datetime import date +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import cffdrs.foliar_moisture_content +import numpy as np +import pytest +from osgeo import gdal, osr +from pytest_mock import MockerFixture +from wps_shared.geospatial.wps_dataset import WPSDataset + +from wps_sfms.interpolation.common import SFMS_NO_DATA +from wps_sfms.processors.foliar_moisture_content import ( + FoliarMoistureContentDatasets, + FoliarMoistureContentProcessor, + calculate_foliar_moisture_content, + ensure_fmc_rasters, +) +from wps_sfms.raster_inputs import FoliarMoistureContentInputs + +NODATA = -9999.0 +MODULE_PATH = "wps_sfms.processors.foliar_moisture_content" + + +def make_dataset(path: str, values: np.ndarray, nodata: float = NODATA) -> WPSDataset: + rows, columns = values.shape + dataset = gdal.GetDriverByName("MEM").Create("", columns, rows, 1, gdal.GDT_Float32) + dataset.SetGeoTransform((0, 2_000, 0, 10_000, 0, -2_000)) + spatial_reference = osr.SpatialReference() + spatial_reference.ImportFromEPSG(3005) + dataset.SetProjection(spatial_reference.ExportToWkt()) + band = dataset.GetRasterBand(1) + band.SetNoDataValue(nodata) + band.WriteArray(values) + return WPSDataset(ds_path=path, ds=dataset) + + +@pytest.fixture(autouse=True) +def output_mask(mocker: MockerFixture): + mask = make_dataset("mask.tif", np.ones((1, 1), dtype=np.float32)) + + @contextmanager + def mask_context(): + yield mask + + mocker.patch("wps_sfms.raster_output.open_bc_mask_dataset", side_effect=mask_context) + yield mask + mask.close() + + +def make_datasets( + elevation: np.ndarray, + latitude: np.ndarray | None = None, + longitude: np.ndarray | None = None, +) -> FoliarMoistureContentDatasets: + shape = elevation.shape + return FoliarMoistureContentDatasets( + elevation=make_dataset("elevation.tif", elevation), + latitude=make_dataset( + "latitude.tif", + latitude if latitude is not None else np.full(shape, 49.0), + ), + longitude=make_dataset( + "longitude.tif", + longitude if longitude is not None else np.full(shape, -123.0), + ), + ) + + +def test_calculation_matches_cffdrs_and_normalizes_western_longitude(): + target_date = date(2024, 5, 30) + datasets = make_datasets( + np.array([[100.0, 100.0]], dtype=np.float32), + longitude=np.array([[-123.0, 123.0]], dtype=np.float32), + ) + + result = calculate_foliar_moisture_content(datasets, target_date) + + expected = cffdrs.foliar_moisture_content.foliar_moisture_content( + 49.0, + 123.0, + 100.0, + 151, + 0, + ) + np.testing.assert_allclose(result.values, np.full((1, 2), expected, dtype=np.float32)) + + +def test_static_input_nodata_propagates_to_output(): + datasets = make_datasets( + np.array([[NODATA, 100.0, 100.0]], dtype=np.float32), + latitude=np.array([[49.0, NODATA, 49.0]], dtype=np.float32), + longitude=np.array([[-123.0, -123.0, NODATA]], dtype=np.float32), + ) + + result = calculate_foliar_moisture_content(datasets, date(2024, 7, 4)) + + np.testing.assert_array_equal( + result.values, + np.full((1, 3), SFMS_NO_DATA, dtype=np.float32), + ) + + +def make_inputs(*target_dates: date) -> FoliarMoistureContentInputs: + return FoliarMoistureContentInputs( + elevation_key="/vsis3/test/sfms_ng/static/bc_elevation.tif", + latitude_key="/vsis3/test/sfms_ng/static/latitude.tif", + longitude_key="/vsis3/test/sfms_ng/static/longitude.tif", + output_keys={ + target_date: f"sfms_ng/static/fmc/{target_date:%Y/%m/%d}/fmc_{target_date:%Y%m%d}.tif" + for target_date in target_dates + }, + ) + + +def make_dataset_context(datasets: FoliarMoistureContentDatasets, calls: list[list[str]]): + @contextmanager + def dataset_context(keys): + calls.append(keys) + input_datasets = [datasets.elevation, datasets.latitude, datasets.longitude] + for dataset, key in zip(input_datasets, keys, strict=True): + dataset.ds_path = key + yield input_datasets + + return dataset_context + + +@pytest.mark.anyio +async def test_processor_loads_static_inputs_once_and_publishes_each_date_with_metadata( + mocker: MockerFixture, +): + target_dates = (date(2024, 5, 30), date(2024, 5, 31)) + inputs = make_inputs(*target_dates) + datasets = make_datasets(np.array([[100.0]], dtype=np.float32)) + context_calls = [] + captured = [] + + async def capture_publish(*, dataset, output_key, **_kwargs): + band = dataset.as_gdal_ds().GetRasterBand(1) + captured.append( + { + "output_key": output_key, + "description": band.GetDescription(), + "unit": band.GetUnitType(), + "nodata": band.GetNoDataValue(), + "value": band.ReadAsArray()[0, 0], + } + ) + return SimpleNamespace(output_key=output_key, cog_key=f"{output_key}_cog") + + s3_client = SimpleNamespace(all_objects_exist=AsyncMock(return_value=True)) + publish = mocker.patch( + "wps_sfms.processors.foliar_moisture_content.publish_dataset", + side_effect=capture_publish, + ) + + await FoliarMoistureContentProcessor().process( + s3_client, + make_dataset_context(datasets, context_calls), + inputs, + ) + + assert len(context_calls) == 1 + assert [item["output_key"] for item in captured] == list(inputs.output_keys.values()) + assert all(item["description"] == "foliar_moisture_content" for item in captured) + assert all(item["unit"] == "%" for item in captured) + assert all(item["nodata"] == pytest.approx(SFMS_NO_DATA) for item in captured) + assert all(item["value"] != pytest.approx(SFMS_NO_DATA) for item in captured) + assert publish.await_count == 2 + + +@pytest.mark.anyio +async def test_processor_applies_bc_mask_to_published_output( + mocker: MockerFixture, + output_mask: WPSDataset, +): + target_date = date(2024, 7, 4) + inputs = make_inputs(target_date) + datasets = make_datasets(np.array([[100.0]], dtype=np.float32)) + context_calls = [] + captured_value = None + output_mask.as_gdal_ds().GetRasterBand(1).WriteArray(np.array([[0]], dtype=np.float32)) + + async def capture_publish(*, dataset, output_key, **_kwargs): + nonlocal captured_value + captured_value = dataset.as_gdal_ds().GetRasterBand(1).ReadAsArray()[0, 0] + return SimpleNamespace(output_key=output_key, cog_key=f"{output_key}_cog") + + s3_client = SimpleNamespace(all_objects_exist=AsyncMock(return_value=True)) + mocker.patch( + "wps_sfms.processors.foliar_moisture_content.publish_dataset", + side_effect=capture_publish, + ) + + await FoliarMoistureContentProcessor().process( + s3_client, + make_dataset_context(datasets, context_calls), + inputs, + ) + + assert captured_value == pytest.approx(SFMS_NO_DATA) + + +@pytest.mark.anyio +async def test_processor_rejects_mismatched_static_grid(mocker: MockerFixture): + inputs = make_inputs(date(2024, 7, 4)) + datasets = make_datasets(np.array([[100.0]], dtype=np.float32)) + context_calls = [] + s3_client = SimpleNamespace(all_objects_exist=AsyncMock(return_value=True)) + mocker.patch( + "wps_sfms.processors.foliar_moisture_content.rasters_match", + side_effect=[True, False], + ) + publish = mocker.patch( + "wps_sfms.processors.foliar_moisture_content.publish_dataset", + new=AsyncMock(), + ) + + with pytest.raises(ValueError, match="longitude raster does not match the elevation grid"): + await FoliarMoistureContentProcessor().process( + s3_client, + make_dataset_context(datasets, context_calls), + inputs, + ) + + publish.assert_not_awaited() + + +@pytest.mark.anyio +async def test_processor_rejects_missing_static_dependency(): + inputs = make_inputs(date(2024, 7, 4)) + s3_client = SimpleNamespace(all_objects_exist=AsyncMock(return_value=False)) + + with pytest.raises(RuntimeError, match="Missing FMC dependencies"): + await FoliarMoistureContentProcessor().process( + s3_client, + lambda _keys: None, + inputs, + ) + + s3_client.all_objects_exist.assert_awaited_once_with( + inputs.elevation_key, + inputs.latitude_key, + inputs.longitude_key, + ) + + +@pytest.mark.anyio +async def test_processor_publish_failure_propagates_and_clears_cache(mocker: MockerFixture): + inputs = make_inputs(date(2024, 7, 4)) + datasets = make_datasets(np.array([[100.0]], dtype=np.float32)) + context_calls = [] + s3_client = SimpleNamespace(all_objects_exist=AsyncMock(return_value=True)) + mocker.patch( + "wps_sfms.processors.foliar_moisture_content.publish_dataset", + new=AsyncMock(side_effect=RuntimeError("COG generation failed")), + ) + clear_cache = mocker.patch("wps_shared.utils.s3.gdal.VSICurlClearCache") + action = FoliarMoistureContentProcessor().process( + s3_client, + make_dataset_context(datasets, context_calls), + inputs, + ) + + with pytest.raises(RuntimeError, match="COG generation failed"): + await action + + clear_cache.assert_called_once_with() + + +@pytest.mark.anyio +async def test_ensure_fmc_rasters_skips_complete_dates_and_processes_missing_dates( + mocker: MockerFixture, +): + existing_date = date(2025, 7, 4) + missing_date = date(2025, 7, 5) + addresser = MagicMock() + addresser.get_fmc_key.side_effect = lambda target_date: f"fmc_{target_date:%Y%m%d}.tif" + addresser.get_cog_key.side_effect = lambda key: f"/vsis3/test/{key[:-4]}_cog.tif" + fmc_inputs = MagicMock() + addresser.get_fmc_inputs.return_value = fmc_inputs + s3_client = MagicMock() + s3_client.all_objects_exist = AsyncMock(side_effect=[True, False]) + processor = MagicMock() + processor.process = AsyncMock() + processor_class = mocker.patch( + f"{MODULE_PATH}.FoliarMoistureContentProcessor", + return_value=processor, + ) + + await ensure_fmc_rasters( + [existing_date, existing_date, missing_date], + addresser, + s3_client, + ) + + assert s3_client.all_objects_exist.await_args_list[0].args == ( + "fmc_20250704.tif", + "/vsis3/test/fmc_20250704_cog.tif", + ) + assert s3_client.all_objects_exist.await_args_list[1].args == ( + "fmc_20250705.tif", + "/vsis3/test/fmc_20250705_cog.tif", + ) + addresser.get_fmc_inputs.assert_called_once_with([missing_date]) + processor_class.assert_called_once_with() + processor.process.assert_awaited_once() + assert processor.process.await_args.args[0] is s3_client + assert processor.process.await_args.args[2] is fmc_inputs + + +@pytest.mark.anyio +async def test_ensure_fmc_rasters_does_not_open_inputs_when_all_outputs_exist( + mocker: MockerFixture, +): + target_date = date(2025, 7, 4) + addresser = MagicMock() + addresser.get_fmc_key.return_value = "fmc_20250704.tif" + addresser.get_cog_key.return_value = "/vsis3/test/fmc_20250704_cog.tif" + s3_client = MagicMock() + s3_client.all_objects_exist = AsyncMock(return_value=True) + processor_class = mocker.patch(f"{MODULE_PATH}.FoliarMoistureContentProcessor") + + await ensure_fmc_rasters([target_date], addresser, s3_client) + + addresser.get_fmc_inputs.assert_not_called() + processor_class.assert_not_called() diff --git a/backend/packages/wps-sfms/src/wps_sfms/tests/test_sfmsng_raster_addresser.py b/backend/packages/wps-sfms/src/wps_sfms/tests/test_sfmsng_raster_addresser.py index a67f9db379..1e9103471a 100644 --- a/backend/packages/wps-sfms/src/wps_sfms/tests/test_sfmsng_raster_addresser.py +++ b/backend/packages/wps-sfms/src/wps_sfms/tests/test_sfmsng_raster_addresser.py @@ -1,4 +1,4 @@ -from datetime import datetime, timezone +from datetime import date, datetime, timezone from zoneinfo import ZoneInfo import pytest @@ -21,6 +21,27 @@ def addresser(): return SFMSNGRasterAddresser() +class TestGetFoliarMoistureContentInputs: + def test_get_fmc_key_uses_shared_static_date_path(self, addresser: SFMSNGRasterAddresser): + assert addresser.get_fmc_key(date(2024, 4, 15)) == ( + "sfms_ng/static/fmc/2024/04/15/fmc_20240415.tif" + ) + + def test_get_fmc_inputs_uses_sfmsng_static_rasters(self, addresser: SFMSNGRasterAddresser): + target_dates = [date(2024, 4, 15), date(2024, 4, 16)] + + result = addresser.get_fmc_inputs(target_dates) + + s3 = addresser.s3_prefix + assert result.elevation_key == f"{s3}/sfms_ng/static/bc_elevation.tif" + assert result.latitude_key == f"{s3}/sfms_ng/static/latitude.tif" + assert result.longitude_key == f"{s3}/sfms_ng/static/longitude.tif" + assert result.output_keys == { + date(2024, 4, 15): "sfms_ng/static/fmc/2024/04/15/fmc_20240415.tif", + date(2024, 4, 16): "sfms_ng/static/fmc/2024/04/16/fmc_20240416.tif", + } + + class TestGetActualWeatherKey: @pytest.mark.parametrize( "weather_param,expected_key", From 1562aca28b358be156d6a445b010301be33414b3 Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Wed, 19 Aug 2026 16:21:54 -0700 Subject: [PATCH 2/6] validate fmc --- .../src/app/jobs/sfms_daily_actuals.py | 1 + .../src/app/jobs/sfms_daily_forecasts.py | 13 ++-- .../app/tests/jobs/test_sfms_daily_actuals.py | 1 + .../tests/jobs/test_sfms_daily_forecasts.py | 1 + .../processors/foliar_moisture_content.py | 52 +++++++++++-- .../wps-sfms/src/wps_sfms/raster_inputs.py | 1 + .../src/wps_sfms/sfmsng_raster_addresser.py | 7 +- .../tests/test_foliar_moisture_content.py | 73 +++++++++++++++++-- .../tests/test_sfmsng_raster_addresser.py | 4 +- 9 files changed, 130 insertions(+), 23 deletions(-) diff --git a/backend/packages/wps-api/src/app/jobs/sfms_daily_actuals.py b/backend/packages/wps-api/src/app/jobs/sfms_daily_actuals.py index 341c18b63b..52d998c186 100644 --- a/backend/packages/wps-api/src/app/jobs/sfms_daily_actuals.py +++ b/backend/packages/wps-api/src/app/jobs/sfms_daily_actuals.py @@ -67,6 +67,7 @@ async def run_sfms_daily_actuals(target_date: datetime) -> None: async with S3Client() as s3_client: await ensure_fmc_rasters( [datetime_to_process.date()], + fuel_raster_path, raster_addresser, s3_client, ) diff --git a/backend/packages/wps-api/src/app/jobs/sfms_daily_forecasts.py b/backend/packages/wps-api/src/app/jobs/sfms_daily_forecasts.py index 6200bbe82f..bad564f2c8 100644 --- a/backend/packages/wps-api/src/app/jobs/sfms_daily_forecasts.py +++ b/backend/packages/wps-api/src/app/jobs/sfms_daily_forecasts.py @@ -92,12 +92,6 @@ async def run_sfms_daily_forecasts(run_datetime: datetime) -> None: datetimes_to_process = forecast_datetimes(seed_actual_date) async with S3Client() as s3_client: - await ensure_fmc_rasters( - [datetime_to_process.date() for datetime_to_process in datetimes_to_process], - raster_addresser, - s3_client, - ) - missing_actual_seed_keys = await get_missing_fwi_seed_keys( datetimes_to_process[0], raster_addresser, @@ -124,6 +118,13 @@ async def run_sfms_daily_forecasts(run_datetime: datetime) -> None: fuel_raster_path = raster_addresser.gdal_path(fuel_type_raster.object_store_path) logger.info("Using reference raster: %s", fuel_raster_path) + await ensure_fmc_rasters( + [datetime_to_process.date() for datetime_to_process in datetimes_to_process], + fuel_raster_path, + raster_addresser, + s3_client, + ) + async with get_async_write_session_scope() as write_session: for index, datetime_to_process in enumerate(datetimes_to_process): sfms_forecasts = await wfwx_api.get_sfms_daily_forecasts_all_stations( diff --git a/backend/packages/wps-api/src/app/tests/jobs/test_sfms_daily_actuals.py b/backend/packages/wps-api/src/app/tests/jobs/test_sfms_daily_actuals.py index 7f0b32b4bb..e3bab4f1eb 100644 --- a/backend/packages/wps-api/src/app/tests/jobs/test_sfms_daily_actuals.py +++ b/backend/packages/wps-api/src/app/tests/jobs/test_sfms_daily_actuals.py @@ -240,6 +240,7 @@ async def test_ensures_shared_fmc_for_target_date( mock_dependencies.ensure_fmc_rasters.assert_awaited_once_with( [target_date.date()], + mock_dependencies.addresser.gdal_path.return_value, mock_dependencies.addresser, mock_dependencies.s3_client, ) diff --git a/backend/packages/wps-api/src/app/tests/jobs/test_sfms_daily_forecasts.py b/backend/packages/wps-api/src/app/tests/jobs/test_sfms_daily_forecasts.py index 333decfcd6..ce87390d7f 100644 --- a/backend/packages/wps-api/src/app/tests/jobs/test_sfms_daily_forecasts.py +++ b/backend/packages/wps-api/src/app/tests/jobs/test_sfms_daily_forecasts.py @@ -240,6 +240,7 @@ async def test_ensures_shared_fmc_for_processed_forecast_dates( mock_dependencies.ensure_fmc_rasters.assert_awaited_once_with( [date(2024, 7, 5), date(2024, 7, 6), date(2024, 7, 7)], + mock_dependencies.addresser.gdal_path.return_value, mock_dependencies.addresser, mock_dependencies.s3_client, ) diff --git a/backend/packages/wps-sfms/src/wps_sfms/processors/foliar_moisture_content.py b/backend/packages/wps-sfms/src/wps_sfms/processors/foliar_moisture_content.py index 624573b2e3..ad2002ceb3 100644 --- a/backend/packages/wps-sfms/src/wps_sfms/processors/foliar_moisture_content.py +++ b/backend/packages/wps-sfms/src/wps_sfms/processors/foliar_moisture_content.py @@ -6,12 +6,13 @@ from dataclasses import dataclass from datetime import date from time import perf_counter -from typing import Callable, ContextManager, Generator +from typing import Callable, ContextManager, Generator, Mapping import numpy as np from cffdrs_vec.fbp import vectorized_foliar_moisture_content from wps_shared.geospatial.geospatial import rasters_match from wps_shared.geospatial.wps_dataset import WPSDataset, multi_wps_dataset_context +from wps_shared.sfms.raster_addresser import GDALPath from wps_shared.utils.s3 import gdal_s3_context from wps_shared.utils.s3_client import S3Client @@ -34,6 +35,7 @@ class FoliarMoistureContentResult: @dataclass(frozen=True) class FoliarMoistureContentDatasets: + fuel: WPSDataset elevation: WPSDataset latitude: WPSDataset longitude: WPSDataset @@ -79,6 +81,7 @@ async def _assert_dependencies_exist( inputs: FoliarMoistureContentInputs, ) -> None: dependency_keys = ( + inputs.fuel_key, inputs.elevation_key, inputs.latitude_key, inputs.longitude_key, @@ -93,10 +96,16 @@ def _open_datasets( input_dataset_context: MultiDatasetContext, inputs: FoliarMoistureContentInputs, ) -> Generator[FoliarMoistureContentDatasets, None, None]: - keys = [inputs.elevation_key, inputs.latitude_key, inputs.longitude_key] + keys = [ + inputs.fuel_key, + inputs.elevation_key, + inputs.latitude_key, + inputs.longitude_key, + ] with input_dataset_context(keys) as input_datasets: datasets_by_key = {dataset.ds_path: dataset for dataset in input_datasets} yield FoliarMoistureContentDatasets( + fuel=datasets_by_key[inputs.fuel_key], elevation=datasets_by_key[inputs.elevation_key], latitude=datasets_by_key[inputs.latitude_key], longitude=datasets_by_key[inputs.longitude_key], @@ -107,16 +116,16 @@ def _validate_grids( datasets: FoliarMoistureContentDatasets, inputs: FoliarMoistureContentInputs, ) -> None: - reference = datasets.elevation.as_gdal_ds() + reference = datasets.fuel.as_gdal_ds() candidates = ( + ("elevation", inputs.elevation_key, datasets.elevation), ("latitude", inputs.latitude_key, datasets.latitude), ("longitude", inputs.longitude_key, datasets.longitude), ) for label, key, dataset in candidates: if not rasters_match(dataset.as_gdal_ds(), reference): raise ValueError( - f"{label} raster does not match the elevation grid: " - f"{key} vs {inputs.elevation_key}" + f"{label} raster does not match the fuel grid: {key} vs {inputs.fuel_key}" ) async def process( @@ -138,7 +147,7 @@ async def process( result = calculate_foliar_moisture_content(datasets, target_date) with create_masked_output_dataset( result.values, - datasets.elevation, + datasets.fuel, result.nodata_value, ) as output_ds: output_band = output_ds.as_gdal_ds().GetRasterBand(1) @@ -158,25 +167,52 @@ async def process( ) +def _validate_existing_fmc_grids( + fuel_key: GDALPath, + fmc_keys: Mapping[date, GDALPath], +) -> None: + """Validate that existing FMC rasters use the selected fuel raster's grid. + + Each FMC raster must have the same pixel resolution, top-left origin, row and column + dimensions, and equivalent projection as the fuel raster. This ensures its pixels can be + combined directly with the fuel grid and other aligned FBP inputs. + """ + with WPSDataset(fuel_key) as fuel: + for target_date, fmc_key in fmc_keys.items(): + with WPSDataset(fmc_key) as fmc: + if not rasters_match(fmc.as_gdal_ds(), fuel.as_gdal_ds()): + raise ValueError( + f"Existing FMC raster for {target_date} does not match the fuel grid: " + f"{fmc_key} vs {fuel_key}" + ) + + async def ensure_fmc_rasters( target_dates: Iterable[date], + fuel_key: GDALPath, raster_addresser: SFMSNGRasterAddresser, s3_client: S3Client, ) -> None: - """Publish shared daily FMC rasters for dates without complete GeoTIFF and COG outputs.""" + """Validate complete FMC rasters and publish dates without complete outputs.""" unique_dates = tuple(dict.fromkeys(target_dates)) missing_dates = [] + existing_fmc_keys: dict[date, GDALPath] = {} for target_date in unique_dates: output_key = raster_addresser.get_fmc_key(target_date) cog_key = raster_addresser.get_cog_key(output_key) if await s3_client.all_objects_exist(output_key, cog_key): logger.info("Skipping existing FMC raster for %s: %s", target_date, output_key) + existing_fmc_keys[target_date] = raster_addresser.gdal_path(output_key) else: missing_dates.append(target_date) + if existing_fmc_keys: + with gdal_s3_context(): + _validate_existing_fmc_grids(fuel_key, existing_fmc_keys) + if not missing_dates: return - inputs = raster_addresser.get_fmc_inputs(missing_dates) + inputs = raster_addresser.get_fmc_inputs(missing_dates, fuel_key) processor = FoliarMoistureContentProcessor() await processor.process(s3_client, multi_wps_dataset_context, inputs) diff --git a/backend/packages/wps-sfms/src/wps_sfms/raster_inputs.py b/backend/packages/wps-sfms/src/wps_sfms/raster_inputs.py index 92de4a0440..892c627823 100644 --- a/backend/packages/wps-sfms/src/wps_sfms/raster_inputs.py +++ b/backend/packages/wps-sfms/src/wps_sfms/raster_inputs.py @@ -43,6 +43,7 @@ class SurfaceFuelConsumptionInputs: class FoliarMoistureContentInputs: """Static dependencies and date-specific outputs for shared daily FMC calculations.""" + fuel_key: GDALPath elevation_key: GDALPath latitude_key: GDALPath longitude_key: GDALPath diff --git a/backend/packages/wps-sfms/src/wps_sfms/sfmsng_raster_addresser.py b/backend/packages/wps-sfms/src/wps_sfms/sfmsng_raster_addresser.py index 3e280983f1..5a0eceabd9 100644 --- a/backend/packages/wps-sfms/src/wps_sfms/sfmsng_raster_addresser.py +++ b/backend/packages/wps-sfms/src/wps_sfms/sfmsng_raster_addresser.py @@ -112,10 +112,15 @@ def get_fmc_key(self, target_date: date) -> S3Key: f"{target_date.day:02d}/fmc_{date_str}.tif" ) - def get_fmc_inputs(self, target_dates: Iterable[date]) -> FoliarMoistureContentInputs: + def get_fmc_inputs( + self, + target_dates: Iterable[date], + fuel_key: GDALPath, + ) -> FoliarMoistureContentInputs: """Build the static dependencies and output keys for daily FMC calculations.""" static_root = f"{self.root}/static" return FoliarMoistureContentInputs( + fuel_key=fuel_key, elevation_key=self.gdal_path(S3Key(f"{static_root}/bc_elevation.tif")), latitude_key=self.gdal_path(S3Key(f"{static_root}/latitude.tif")), longitude_key=self.gdal_path(S3Key(f"{static_root}/longitude.tif")), diff --git a/backend/packages/wps-sfms/src/wps_sfms/tests/test_foliar_moisture_content.py b/backend/packages/wps-sfms/src/wps_sfms/tests/test_foliar_moisture_content.py index 53604c22fe..5b9f28323f 100644 --- a/backend/packages/wps-sfms/src/wps_sfms/tests/test_foliar_moisture_content.py +++ b/backend/packages/wps-sfms/src/wps_sfms/tests/test_foliar_moisture_content.py @@ -56,6 +56,7 @@ def make_datasets( ) -> FoliarMoistureContentDatasets: shape = elevation.shape return FoliarMoistureContentDatasets( + fuel=make_dataset("fuel.tif", np.ones(shape, dtype=np.float32)), elevation=make_dataset("elevation.tif", elevation), latitude=make_dataset( "latitude.tif", @@ -104,6 +105,7 @@ def test_static_input_nodata_propagates_to_output(): def make_inputs(*target_dates: date) -> FoliarMoistureContentInputs: return FoliarMoistureContentInputs( + fuel_key="/vsis3/test/sfms/fuel/2024/fuel.tif", elevation_key="/vsis3/test/sfms_ng/static/bc_elevation.tif", latitude_key="/vsis3/test/sfms_ng/static/latitude.tif", longitude_key="/vsis3/test/sfms_ng/static/longitude.tif", @@ -118,7 +120,12 @@ def make_dataset_context(datasets: FoliarMoistureContentDatasets, calls: list[li @contextmanager def dataset_context(keys): calls.append(keys) - input_datasets = [datasets.elevation, datasets.latitude, datasets.longitude] + input_datasets = [ + datasets.fuel, + datasets.elevation, + datasets.latitude, + datasets.longitude, + ] for dataset, key in zip(input_datasets, keys, strict=True): dataset.ds_path = key yield input_datasets @@ -203,21 +210,33 @@ async def capture_publish(*, dataset, output_key, **_kwargs): @pytest.mark.anyio -async def test_processor_rejects_mismatched_static_grid(mocker: MockerFixture): +@pytest.mark.parametrize( + "mismatched_label,match_results", + [ + ("elevation", [False]), + ("latitude", [True, False]), + ("longitude", [True, True, False]), + ], +) +async def test_processor_rejects_static_grid_that_mismatches_fuel( + mocker: MockerFixture, + mismatched_label: str, + match_results: list[bool], +): inputs = make_inputs(date(2024, 7, 4)) datasets = make_datasets(np.array([[100.0]], dtype=np.float32)) context_calls = [] s3_client = SimpleNamespace(all_objects_exist=AsyncMock(return_value=True)) mocker.patch( "wps_sfms.processors.foliar_moisture_content.rasters_match", - side_effect=[True, False], + side_effect=match_results, ) publish = mocker.patch( "wps_sfms.processors.foliar_moisture_content.publish_dataset", new=AsyncMock(), ) - with pytest.raises(ValueError, match="longitude raster does not match the elevation grid"): + with pytest.raises(ValueError, match=f"{mismatched_label} raster does not match the fuel grid"): await FoliarMoistureContentProcessor().process( s3_client, make_dataset_context(datasets, context_calls), @@ -240,6 +259,7 @@ async def test_processor_rejects_missing_static_dependency(): ) s3_client.all_objects_exist.assert_awaited_once_with( + inputs.fuel_key, inputs.elevation_key, inputs.latitude_key, inputs.longitude_key, @@ -278,6 +298,7 @@ async def test_ensure_fmc_rasters_skips_complete_dates_and_processes_missing_dat addresser = MagicMock() addresser.get_fmc_key.side_effect = lambda target_date: f"fmc_{target_date:%Y%m%d}.tif" addresser.get_cog_key.side_effect = lambda key: f"/vsis3/test/{key[:-4]}_cog.tif" + addresser.gdal_path.side_effect = lambda key: f"/vsis3/test/{key}" fmc_inputs = MagicMock() addresser.get_fmc_inputs.return_value = fmc_inputs s3_client = MagicMock() @@ -288,9 +309,14 @@ async def test_ensure_fmc_rasters_skips_complete_dates_and_processes_missing_dat f"{MODULE_PATH}.FoliarMoistureContentProcessor", return_value=processor, ) + fuel = make_dataset("fuel.tif", np.ones((1, 1), dtype=np.float32)) + existing_fmc = make_dataset("fmc.tif", np.ones((1, 1), dtype=np.float32)) + + open_dataset = mocker.patch(f"{MODULE_PATH}.WPSDataset", side_effect=[fuel, existing_fmc]) await ensure_fmc_rasters( [existing_date, existing_date, missing_date], + "fuel.tif", addresser, s3_client, ) @@ -303,26 +329,59 @@ async def test_ensure_fmc_rasters_skips_complete_dates_and_processes_missing_dat "fmc_20250705.tif", "/vsis3/test/fmc_20250705_cog.tif", ) - addresser.get_fmc_inputs.assert_called_once_with([missing_date]) + addresser.get_fmc_inputs.assert_called_once_with([missing_date], "fuel.tif") processor_class.assert_called_once_with() processor.process.assert_awaited_once() assert processor.process.await_args.args[0] is s3_client assert processor.process.await_args.args[2] is fmc_inputs + assert [item.args[0] for item in open_dataset.call_args_list] == [ + "fuel.tif", + "/vsis3/test/fmc_20250704.tif", + ] @pytest.mark.anyio -async def test_ensure_fmc_rasters_does_not_open_inputs_when_all_outputs_exist( +async def test_ensure_fmc_rasters_does_not_load_static_inputs_when_all_outputs_match( mocker: MockerFixture, ): target_date = date(2025, 7, 4) addresser = MagicMock() addresser.get_fmc_key.return_value = "fmc_20250704.tif" addresser.get_cog_key.return_value = "/vsis3/test/fmc_20250704_cog.tif" + addresser.gdal_path.return_value = "/vsis3/test/fmc_20250704.tif" s3_client = MagicMock() s3_client.all_objects_exist = AsyncMock(return_value=True) processor_class = mocker.patch(f"{MODULE_PATH}.FoliarMoistureContentProcessor") + fuel = make_dataset("fuel.tif", np.ones((1, 1), dtype=np.float32)) + existing_fmc = make_dataset("fmc.tif", np.ones((1, 1), dtype=np.float32)) + + mocker.patch(f"{MODULE_PATH}.WPSDataset", side_effect=[fuel, existing_fmc]) - await ensure_fmc_rasters([target_date], addresser, s3_client) + await ensure_fmc_rasters([target_date], "fuel.tif", addresser, s3_client) addresser.get_fmc_inputs.assert_not_called() processor_class.assert_not_called() + + +@pytest.mark.anyio +async def test_ensure_fmc_rasters_rejects_existing_output_that_mismatches_fuel( + mocker: MockerFixture, +): + target_date = date(2025, 7, 4) + addresser = MagicMock() + addresser.get_fmc_key.return_value = "fmc_20250704.tif" + addresser.get_cog_key.return_value = "/vsis3/test/fmc_20250704_cog.tif" + addresser.gdal_path.return_value = "/vsis3/test/fmc_20250704.tif" + s3_client = MagicMock() + s3_client.all_objects_exist = AsyncMock(return_value=True) + processor_class = mocker.patch(f"{MODULE_PATH}.FoliarMoistureContentProcessor") + fuel = make_dataset("fuel.tif", np.ones((1, 1), dtype=np.float32)) + existing_fmc = make_dataset("fmc.tif", np.ones((2, 1), dtype=np.float32)) + + mocker.patch(f"{MODULE_PATH}.WPSDataset", side_effect=[fuel, existing_fmc]) + action = ensure_fmc_rasters([target_date], "fuel.tif", addresser, s3_client) + + with pytest.raises(ValueError, match="Existing FMC raster for 2025-07-04"): + await action + + processor_class.assert_not_called() diff --git a/backend/packages/wps-sfms/src/wps_sfms/tests/test_sfmsng_raster_addresser.py b/backend/packages/wps-sfms/src/wps_sfms/tests/test_sfmsng_raster_addresser.py index 1e9103471a..4bb182277a 100644 --- a/backend/packages/wps-sfms/src/wps_sfms/tests/test_sfmsng_raster_addresser.py +++ b/backend/packages/wps-sfms/src/wps_sfms/tests/test_sfmsng_raster_addresser.py @@ -29,10 +29,12 @@ def test_get_fmc_key_uses_shared_static_date_path(self, addresser: SFMSNGRasterA def test_get_fmc_inputs_uses_sfmsng_static_rasters(self, addresser: SFMSNGRasterAddresser): target_dates = [date(2024, 4, 15), date(2024, 4, 16)] + fuel_key = addresser.gdal_path(addresser.get_fuel_raster_key(TEST_DATETIME, 3)) - result = addresser.get_fmc_inputs(target_dates) + result = addresser.get_fmc_inputs(target_dates, fuel_key) s3 = addresser.s3_prefix + assert result.fuel_key == fuel_key assert result.elevation_key == f"{s3}/sfms_ng/static/bc_elevation.tif" assert result.latitude_key == f"{s3}/sfms_ng/static/latitude.tif" assert result.longitude_key == f"{s3}/sfms_ng/static/longitude.tif" From 47cd78ec0e62e86709f74f4cdaa21ce1e0be0087 Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Thu, 20 Aug 2026 08:32:48 -0700 Subject: [PATCH 3/6] comments --- .../src/wps_sfms/processors/foliar_moisture_content.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/packages/wps-sfms/src/wps_sfms/processors/foliar_moisture_content.py b/backend/packages/wps-sfms/src/wps_sfms/processors/foliar_moisture_content.py index ad2002ceb3..25d3056210 100644 --- a/backend/packages/wps-sfms/src/wps_sfms/processors/foliar_moisture_content.py +++ b/backend/packages/wps-sfms/src/wps_sfms/processors/foliar_moisture_content.py @@ -35,7 +35,7 @@ class FoliarMoistureContentResult: @dataclass(frozen=True) class FoliarMoistureContentDatasets: - fuel: WPSDataset + fuel: WPSDataset # fuel is only used for grid validation, not in the FMC calculation elevation: WPSDataset latitude: WPSDataset longitude: WPSDataset @@ -193,7 +193,7 @@ async def ensure_fmc_rasters( raster_addresser: SFMSNGRasterAddresser, s3_client: S3Client, ) -> None: - """Validate complete FMC rasters and publish dates without complete outputs.""" + """Validate complete FMC rasters and publish dates without exisiting FMC rasters.""" unique_dates = tuple(dict.fromkeys(target_dates)) missing_dates = [] existing_fmc_keys: dict[date, GDALPath] = {} From 3f07a6b1b373ff38b4ca56c5c001c30939c160fc Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Thu, 20 Aug 2026 09:09:46 -0700 Subject: [PATCH 4/6] tests --- .../wps-sfms/src/wps_sfms/tests/conftest.py | 23 ++++++- .../src/wps_sfms/tests/raster_test_utils.py | 27 ++++++++ .../tests/test_foliar_moisture_content.py | 61 ++++++------------- .../tests/test_surface_fuel_consumption.py | 56 ++++++----------- 4 files changed, 85 insertions(+), 82 deletions(-) create mode 100644 backend/packages/wps-sfms/src/wps_sfms/tests/raster_test_utils.py diff --git a/backend/packages/wps-sfms/src/wps_sfms/tests/conftest.py b/backend/packages/wps-sfms/src/wps_sfms/tests/conftest.py index 71940528a6..d6a363dda3 100644 --- a/backend/packages/wps-sfms/src/wps_sfms/tests/conftest.py +++ b/backend/packages/wps-sfms/src/wps_sfms/tests/conftest.py @@ -1,7 +1,28 @@ +from collections.abc import Generator +from contextlib import contextmanager from typing import Optional import numpy as np +import pytest from osgeo import gdal, osr +from pytest_mock import MockerFixture +from wps_shared.geospatial.wps_dataset import WPSDataset + +from wps_sfms.tests.raster_test_utils import TEST_NODATA, create_test_wps_dataset + + +@pytest.fixture +def output_mask(mocker: MockerFixture) -> Generator[WPSDataset, None, None]: + """Provide and patch the final BC output mask used by processor tests.""" + mask = create_test_wps_dataset("mask.tif", np.ones((1, 1), dtype=np.float32)) + + @contextmanager + def mask_context() -> Generator[WPSDataset, None, None]: + yield mask + + mocker.patch("wps_sfms.raster_output.open_bc_mask_dataset", side_effect=mask_context) + yield mask + mask.close() def create_test_raster( @@ -12,7 +33,7 @@ def create_test_raster( data: Optional[np.ndarray] = None, epsg: int = 4326, fill_value: float = 1.0, - nodata: float = -9999.0, + nodata: float = TEST_NODATA, ): """ Create a test GeoTIFF raster in memory using GDAL's /vsimem/ filesystem. diff --git a/backend/packages/wps-sfms/src/wps_sfms/tests/raster_test_utils.py b/backend/packages/wps-sfms/src/wps_sfms/tests/raster_test_utils.py new file mode 100644 index 0000000000..3658d6a51d --- /dev/null +++ b/backend/packages/wps-sfms/src/wps_sfms/tests/raster_test_utils.py @@ -0,0 +1,27 @@ +"""Shared raster construction helpers for SFMS processor tests.""" + +import numpy as np +from osgeo import gdal, osr +from wps_shared.geospatial.wps_dataset import WPSDataset + +TEST_NODATA = -9999.0 + + +def create_test_wps_dataset( + path: str, + values: np.ndarray, + nodata: float = TEST_NODATA, +) -> WPSDataset: + """Create an in-memory float raster on the standard SFMS test grid.""" + rows, columns = values.shape + dataset = gdal.GetDriverByName("MEM").Create("", columns, rows, 1, gdal.GDT_Float32) + dataset.SetGeoTransform((0, 2_000, 0, 10_000, 0, -2_000)) + + spatial_reference = osr.SpatialReference() + spatial_reference.ImportFromEPSG(3005) + dataset.SetProjection(spatial_reference.ExportToWkt()) + + band = dataset.GetRasterBand(1) + band.SetNoDataValue(nodata) + band.WriteArray(values) + return WPSDataset(ds_path=path, ds=dataset) diff --git a/backend/packages/wps-sfms/src/wps_sfms/tests/test_foliar_moisture_content.py b/backend/packages/wps-sfms/src/wps_sfms/tests/test_foliar_moisture_content.py index 5b9f28323f..9dc2c215d9 100644 --- a/backend/packages/wps-sfms/src/wps_sfms/tests/test_foliar_moisture_content.py +++ b/backend/packages/wps-sfms/src/wps_sfms/tests/test_foliar_moisture_content.py @@ -6,7 +6,6 @@ import cffdrs.foliar_moisture_content import numpy as np import pytest -from osgeo import gdal, osr from pytest_mock import MockerFixture from wps_shared.geospatial.wps_dataset import WPSDataset @@ -18,37 +17,11 @@ ensure_fmc_rasters, ) from wps_sfms.raster_inputs import FoliarMoistureContentInputs +from wps_sfms.tests.raster_test_utils import TEST_NODATA, create_test_wps_dataset -NODATA = -9999.0 MODULE_PATH = "wps_sfms.processors.foliar_moisture_content" -def make_dataset(path: str, values: np.ndarray, nodata: float = NODATA) -> WPSDataset: - rows, columns = values.shape - dataset = gdal.GetDriverByName("MEM").Create("", columns, rows, 1, gdal.GDT_Float32) - dataset.SetGeoTransform((0, 2_000, 0, 10_000, 0, -2_000)) - spatial_reference = osr.SpatialReference() - spatial_reference.ImportFromEPSG(3005) - dataset.SetProjection(spatial_reference.ExportToWkt()) - band = dataset.GetRasterBand(1) - band.SetNoDataValue(nodata) - band.WriteArray(values) - return WPSDataset(ds_path=path, ds=dataset) - - -@pytest.fixture(autouse=True) -def output_mask(mocker: MockerFixture): - mask = make_dataset("mask.tif", np.ones((1, 1), dtype=np.float32)) - - @contextmanager - def mask_context(): - yield mask - - mocker.patch("wps_sfms.raster_output.open_bc_mask_dataset", side_effect=mask_context) - yield mask - mask.close() - - def make_datasets( elevation: np.ndarray, latitude: np.ndarray | None = None, @@ -56,13 +29,13 @@ def make_datasets( ) -> FoliarMoistureContentDatasets: shape = elevation.shape return FoliarMoistureContentDatasets( - fuel=make_dataset("fuel.tif", np.ones(shape, dtype=np.float32)), - elevation=make_dataset("elevation.tif", elevation), - latitude=make_dataset( + fuel=create_test_wps_dataset("fuel.tif", np.ones(shape, dtype=np.float32)), + elevation=create_test_wps_dataset("elevation.tif", elevation), + latitude=create_test_wps_dataset( "latitude.tif", latitude if latitude is not None else np.full(shape, 49.0), ), - longitude=make_dataset( + longitude=create_test_wps_dataset( "longitude.tif", longitude if longitude is not None else np.full(shape, -123.0), ), @@ -90,9 +63,9 @@ def test_calculation_matches_cffdrs_and_normalizes_western_longitude(): def test_static_input_nodata_propagates_to_output(): datasets = make_datasets( - np.array([[NODATA, 100.0, 100.0]], dtype=np.float32), - latitude=np.array([[49.0, NODATA, 49.0]], dtype=np.float32), - longitude=np.array([[-123.0, -123.0, NODATA]], dtype=np.float32), + np.array([[TEST_NODATA, 100.0, 100.0]], dtype=np.float32), + latitude=np.array([[49.0, TEST_NODATA, 49.0]], dtype=np.float32), + longitude=np.array([[-123.0, -123.0, TEST_NODATA]], dtype=np.float32), ) result = calculate_foliar_moisture_content(datasets, date(2024, 7, 4)) @@ -136,6 +109,7 @@ def dataset_context(keys): @pytest.mark.anyio async def test_processor_loads_static_inputs_once_and_publishes_each_date_with_metadata( mocker: MockerFixture, + output_mask: WPSDataset, ): target_dates = (date(2024, 5, 30), date(2024, 5, 31)) inputs = make_inputs(*target_dates) @@ -267,7 +241,10 @@ async def test_processor_rejects_missing_static_dependency(): @pytest.mark.anyio -async def test_processor_publish_failure_propagates_and_clears_cache(mocker: MockerFixture): +async def test_processor_publish_failure_propagates_and_clears_cache( + mocker: MockerFixture, + output_mask: WPSDataset, +): inputs = make_inputs(date(2024, 7, 4)) datasets = make_datasets(np.array([[100.0]], dtype=np.float32)) context_calls = [] @@ -309,8 +286,8 @@ async def test_ensure_fmc_rasters_skips_complete_dates_and_processes_missing_dat f"{MODULE_PATH}.FoliarMoistureContentProcessor", return_value=processor, ) - fuel = make_dataset("fuel.tif", np.ones((1, 1), dtype=np.float32)) - existing_fmc = make_dataset("fmc.tif", np.ones((1, 1), dtype=np.float32)) + fuel = create_test_wps_dataset("fuel.tif", np.ones((1, 1), dtype=np.float32)) + existing_fmc = create_test_wps_dataset("fmc.tif", np.ones((1, 1), dtype=np.float32)) open_dataset = mocker.patch(f"{MODULE_PATH}.WPSDataset", side_effect=[fuel, existing_fmc]) @@ -352,8 +329,8 @@ async def test_ensure_fmc_rasters_does_not_load_static_inputs_when_all_outputs_m s3_client = MagicMock() s3_client.all_objects_exist = AsyncMock(return_value=True) processor_class = mocker.patch(f"{MODULE_PATH}.FoliarMoistureContentProcessor") - fuel = make_dataset("fuel.tif", np.ones((1, 1), dtype=np.float32)) - existing_fmc = make_dataset("fmc.tif", np.ones((1, 1), dtype=np.float32)) + fuel = create_test_wps_dataset("fuel.tif", np.ones((1, 1), dtype=np.float32)) + existing_fmc = create_test_wps_dataset("fmc.tif", np.ones((1, 1), dtype=np.float32)) mocker.patch(f"{MODULE_PATH}.WPSDataset", side_effect=[fuel, existing_fmc]) @@ -375,8 +352,8 @@ async def test_ensure_fmc_rasters_rejects_existing_output_that_mismatches_fuel( s3_client = MagicMock() s3_client.all_objects_exist = AsyncMock(return_value=True) processor_class = mocker.patch(f"{MODULE_PATH}.FoliarMoistureContentProcessor") - fuel = make_dataset("fuel.tif", np.ones((1, 1), dtype=np.float32)) - existing_fmc = make_dataset("fmc.tif", np.ones((2, 1), dtype=np.float32)) + fuel = create_test_wps_dataset("fuel.tif", np.ones((1, 1), dtype=np.float32)) + existing_fmc = create_test_wps_dataset("fmc.tif", np.ones((2, 1), dtype=np.float32)) mocker.patch(f"{MODULE_PATH}.WPSDataset", side_effect=[fuel, existing_fmc]) action = ensure_fmc_rasters([target_date], "fuel.tif", addresser, s3_client) diff --git a/backend/packages/wps-sfms/src/wps_sfms/tests/test_surface_fuel_consumption.py b/backend/packages/wps-sfms/src/wps_sfms/tests/test_surface_fuel_consumption.py index ff8edfa029..472289149e 100644 --- a/backend/packages/wps-sfms/src/wps_sfms/tests/test_surface_fuel_consumption.py +++ b/backend/packages/wps-sfms/src/wps_sfms/tests/test_surface_fuel_consumption.py @@ -6,7 +6,6 @@ import cffdrs.surface_fuel_consumption import numpy as np import pytest -from osgeo import gdal, osr from pytest_mock import MockerFixture from wps_shared.geospatial.wps_dataset import WPSDataset from wps_shared.run_type import RunType @@ -18,35 +17,9 @@ calculate_surface_fuel_consumption, ) from wps_sfms.raster_inputs import SurfaceFuelConsumptionInputs +from wps_sfms.tests.raster_test_utils import TEST_NODATA, create_test_wps_dataset TEST_DATETIME = datetime(2024, 7, 4, 20, tzinfo=timezone.utc) -NODATA = -9999.0 - - -def make_dataset(path: str, values: np.ndarray, nodata: float = NODATA) -> WPSDataset: - rows, columns = values.shape - dataset = gdal.GetDriverByName("MEM").Create("", columns, rows, 1, gdal.GDT_Float32) - dataset.SetGeoTransform((0, 2_000, 0, 10_000, 0, -2_000)) - spatial_reference = osr.SpatialReference() - spatial_reference.ImportFromEPSG(3005) - dataset.SetProjection(spatial_reference.ExportToWkt()) - band = dataset.GetRasterBand(1) - band.SetNoDataValue(nodata) - band.WriteArray(values) - return WPSDataset(ds_path=path, ds=dataset) - - -@pytest.fixture(autouse=True) -def output_mask(mocker: MockerFixture): - mask = make_dataset("mask.tif", np.ones((1, 1), dtype=np.float32)) - - @contextmanager - def mask_context(): - yield mask - - mocker.patch("wps_sfms.raster_output.open_bc_mask_dataset", side_effect=mask_context) - yield mask - mask.close() def make_datasets( @@ -57,12 +30,14 @@ def make_datasets( ) -> SurfaceFuelConsumptionDatasets: shape = fuel.shape return SurfaceFuelConsumptionDatasets( - fuel=make_dataset("fuel.tif", fuel), - ffmc=make_dataset("ffmc.tif", ffmc if ffmc is not None else np.full(shape, 90.0)), - bui=make_dataset("bui.tif", bui if bui is not None else np.full(shape, 60.0)), - percent_conifer=make_dataset( + fuel=create_test_wps_dataset("fuel.tif", fuel), + ffmc=create_test_wps_dataset( + "ffmc.tif", ffmc if ffmc is not None else np.full(shape, 90.0) + ), + bui=create_test_wps_dataset("bui.tif", bui if bui is not None else np.full(shape, 60.0)), + percent_conifer=create_test_wps_dataset( "percent_conifer.tif", - percent_conifer if percent_conifer is not None else np.full(shape, NODATA), + percent_conifer if percent_conifer is not None else np.full(shape, TEST_NODATA), ), ) @@ -103,7 +78,7 @@ def test_calculation_matches_cffdrs_reference( def test_non_fuel_becomes_zero_and_source_nodata_remains_sfms_nodata(): - fuel = np.array([[99, 102, NODATA]], dtype=np.float32) + fuel = np.array([[99, 102, TEST_NODATA]], dtype=np.float32) datasets = make_datasets(fuel) result = calculate_surface_fuel_consumption(datasets) @@ -114,8 +89,8 @@ def test_non_fuel_becomes_zero_and_source_nodata_remains_sfms_nodata(): def test_non_fuel_becomes_zero_when_weather_is_nodata(): datasets = make_datasets( np.array([[99, 102]], dtype=np.float32), - ffmc=np.full((1, 2), NODATA, dtype=np.float32), - bui=np.full((1, 2), NODATA, dtype=np.float32), + ffmc=np.full((1, 2), TEST_NODATA, dtype=np.float32), + bui=np.full((1, 2), TEST_NODATA, dtype=np.float32), ) result = calculate_surface_fuel_consumption(datasets) @@ -126,8 +101,8 @@ def test_non_fuel_becomes_zero_when_weather_is_nodata(): def test_weather_nodata_propagates_to_output(): datasets = make_datasets( np.array([[1, 2]], dtype=np.float32), - ffmc=np.array([[NODATA, 90]], dtype=np.float32), - bui=np.array([[60, NODATA]], dtype=np.float32), + ffmc=np.array([[TEST_NODATA, 90]], dtype=np.float32), + bui=np.array([[60, TEST_NODATA]], dtype=np.float32), ) result = calculate_surface_fuel_consumption(datasets) @@ -229,7 +204,10 @@ async def capture_publish(*, dataset, output_key, **_kwargs): @pytest.mark.anyio -async def test_processor_publish_failure_propagates_and_clears_cache(mocker: MockerFixture): +async def test_processor_publish_failure_propagates_and_clears_cache( + mocker: MockerFixture, + output_mask: WPSDataset, +): datasets = make_datasets(np.array([[1]], dtype=np.float32)) inputs = make_inputs() s3_client = SimpleNamespace(all_objects_exist=AsyncMock(return_value=True)) From 5abdd713521855df9919c5a7633391401af26514 Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Thu, 20 Aug 2026 09:28:21 -0700 Subject: [PATCH 5/6] test cleanup --- .../wps-sfms/src/wps_sfms/tests/conftest.py | 4 +-- .../src/wps_sfms/tests/raster_test_utils.py | 5 ++-- .../tests/test_foliar_moisture_content.py | 25 ++++++++----------- .../tests/test_surface_fuel_consumption.py | 14 +++++------ 4 files changed, 23 insertions(+), 25 deletions(-) diff --git a/backend/packages/wps-sfms/src/wps_sfms/tests/conftest.py b/backend/packages/wps-sfms/src/wps_sfms/tests/conftest.py index d6a363dda3..be1a626d0e 100644 --- a/backend/packages/wps-sfms/src/wps_sfms/tests/conftest.py +++ b/backend/packages/wps-sfms/src/wps_sfms/tests/conftest.py @@ -8,7 +8,7 @@ from pytest_mock import MockerFixture from wps_shared.geospatial.wps_dataset import WPSDataset -from wps_sfms.tests.raster_test_utils import TEST_NODATA, create_test_wps_dataset +from wps_sfms.tests.raster_test_utils import TEST_INPUT_NODATA, create_test_wps_dataset @pytest.fixture @@ -33,7 +33,7 @@ def create_test_raster( data: Optional[np.ndarray] = None, epsg: int = 4326, fill_value: float = 1.0, - nodata: float = TEST_NODATA, + nodata: float = TEST_INPUT_NODATA, ): """ Create a test GeoTIFF raster in memory using GDAL's /vsimem/ filesystem. diff --git a/backend/packages/wps-sfms/src/wps_sfms/tests/raster_test_utils.py b/backend/packages/wps-sfms/src/wps_sfms/tests/raster_test_utils.py index 3658d6a51d..ca1800086d 100644 --- a/backend/packages/wps-sfms/src/wps_sfms/tests/raster_test_utils.py +++ b/backend/packages/wps-sfms/src/wps_sfms/tests/raster_test_utils.py @@ -4,13 +4,14 @@ from osgeo import gdal, osr from wps_shared.geospatial.wps_dataset import WPSDataset -TEST_NODATA = -9999.0 +# use an input nodata value that differs from SFMS_NO_DATA to verify output normalization +TEST_INPUT_NODATA = -9999.0 def create_test_wps_dataset( path: str, values: np.ndarray, - nodata: float = TEST_NODATA, + nodata: float = TEST_INPUT_NODATA, ) -> WPSDataset: """Create an in-memory float raster on the standard SFMS test grid.""" rows, columns = values.shape diff --git a/backend/packages/wps-sfms/src/wps_sfms/tests/test_foliar_moisture_content.py b/backend/packages/wps-sfms/src/wps_sfms/tests/test_foliar_moisture_content.py index 9dc2c215d9..849606200b 100644 --- a/backend/packages/wps-sfms/src/wps_sfms/tests/test_foliar_moisture_content.py +++ b/backend/packages/wps-sfms/src/wps_sfms/tests/test_foliar_moisture_content.py @@ -17,7 +17,7 @@ ensure_fmc_rasters, ) from wps_sfms.raster_inputs import FoliarMoistureContentInputs -from wps_sfms.tests.raster_test_utils import TEST_NODATA, create_test_wps_dataset +from wps_sfms.tests.raster_test_utils import TEST_INPUT_NODATA, create_test_wps_dataset MODULE_PATH = "wps_sfms.processors.foliar_moisture_content" @@ -63,9 +63,9 @@ def test_calculation_matches_cffdrs_and_normalizes_western_longitude(): def test_static_input_nodata_propagates_to_output(): datasets = make_datasets( - np.array([[TEST_NODATA, 100.0, 100.0]], dtype=np.float32), - latitude=np.array([[49.0, TEST_NODATA, 49.0]], dtype=np.float32), - longitude=np.array([[-123.0, -123.0, TEST_NODATA]], dtype=np.float32), + np.array([[TEST_INPUT_NODATA, 100.0, 100.0]], dtype=np.float32), + latitude=np.array([[49.0, TEST_INPUT_NODATA, 49.0]], dtype=np.float32), + longitude=np.array([[-123.0, -123.0, TEST_INPUT_NODATA]], dtype=np.float32), ) result = calculate_foliar_moisture_content(datasets, date(2024, 7, 4)) @@ -209,13 +209,12 @@ async def test_processor_rejects_static_grid_that_mismatches_fuel( "wps_sfms.processors.foliar_moisture_content.publish_dataset", new=AsyncMock(), ) + processor = FoliarMoistureContentProcessor() + input_context = make_dataset_context(datasets, context_calls) + action = processor.process(s3_client, input_context, inputs) with pytest.raises(ValueError, match=f"{mismatched_label} raster does not match the fuel grid"): - await FoliarMoistureContentProcessor().process( - s3_client, - make_dataset_context(datasets, context_calls), - inputs, - ) + await action publish.assert_not_awaited() @@ -224,13 +223,11 @@ async def test_processor_rejects_static_grid_that_mismatches_fuel( async def test_processor_rejects_missing_static_dependency(): inputs = make_inputs(date(2024, 7, 4)) s3_client = SimpleNamespace(all_objects_exist=AsyncMock(return_value=False)) + processor = FoliarMoistureContentProcessor() + action = processor.process(s3_client, lambda _keys: None, inputs) with pytest.raises(RuntimeError, match="Missing FMC dependencies"): - await FoliarMoistureContentProcessor().process( - s3_client, - lambda _keys: None, - inputs, - ) + await action s3_client.all_objects_exist.assert_awaited_once_with( inputs.fuel_key, diff --git a/backend/packages/wps-sfms/src/wps_sfms/tests/test_surface_fuel_consumption.py b/backend/packages/wps-sfms/src/wps_sfms/tests/test_surface_fuel_consumption.py index 472289149e..88def39ff4 100644 --- a/backend/packages/wps-sfms/src/wps_sfms/tests/test_surface_fuel_consumption.py +++ b/backend/packages/wps-sfms/src/wps_sfms/tests/test_surface_fuel_consumption.py @@ -17,7 +17,7 @@ calculate_surface_fuel_consumption, ) from wps_sfms.raster_inputs import SurfaceFuelConsumptionInputs -from wps_sfms.tests.raster_test_utils import TEST_NODATA, create_test_wps_dataset +from wps_sfms.tests.raster_test_utils import TEST_INPUT_NODATA, create_test_wps_dataset TEST_DATETIME = datetime(2024, 7, 4, 20, tzinfo=timezone.utc) @@ -37,7 +37,7 @@ def make_datasets( bui=create_test_wps_dataset("bui.tif", bui if bui is not None else np.full(shape, 60.0)), percent_conifer=create_test_wps_dataset( "percent_conifer.tif", - percent_conifer if percent_conifer is not None else np.full(shape, TEST_NODATA), + percent_conifer if percent_conifer is not None else np.full(shape, TEST_INPUT_NODATA), ), ) @@ -78,7 +78,7 @@ def test_calculation_matches_cffdrs_reference( def test_non_fuel_becomes_zero_and_source_nodata_remains_sfms_nodata(): - fuel = np.array([[99, 102, TEST_NODATA]], dtype=np.float32) + fuel = np.array([[99, 102, TEST_INPUT_NODATA]], dtype=np.float32) datasets = make_datasets(fuel) result = calculate_surface_fuel_consumption(datasets) @@ -89,8 +89,8 @@ def test_non_fuel_becomes_zero_and_source_nodata_remains_sfms_nodata(): def test_non_fuel_becomes_zero_when_weather_is_nodata(): datasets = make_datasets( np.array([[99, 102]], dtype=np.float32), - ffmc=np.full((1, 2), TEST_NODATA, dtype=np.float32), - bui=np.full((1, 2), TEST_NODATA, dtype=np.float32), + ffmc=np.full((1, 2), TEST_INPUT_NODATA, dtype=np.float32), + bui=np.full((1, 2), TEST_INPUT_NODATA, dtype=np.float32), ) result = calculate_surface_fuel_consumption(datasets) @@ -101,8 +101,8 @@ def test_non_fuel_becomes_zero_when_weather_is_nodata(): def test_weather_nodata_propagates_to_output(): datasets = make_datasets( np.array([[1, 2]], dtype=np.float32), - ffmc=np.array([[TEST_NODATA, 90]], dtype=np.float32), - bui=np.array([[60, TEST_NODATA]], dtype=np.float32), + ffmc=np.array([[TEST_INPUT_NODATA, 90]], dtype=np.float32), + bui=np.array([[60, TEST_INPUT_NODATA]], dtype=np.float32), ) result = calculate_surface_fuel_consumption(datasets) From 0410f47ef82838f559397aa3b14f71259efceb24 Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Thu, 20 Aug 2026 11:28:32 -0700 Subject: [PATCH 6/6] todo update --- docs/architecture/fbp-todo.md | 79 +++++++++++++++++++---------------- 1 file changed, 42 insertions(+), 37 deletions(-) diff --git a/docs/architecture/fbp-todo.md b/docs/architecture/fbp-todo.md index c6881a11db..9f12ad1766 100644 --- a/docs/architecture/fbp-todo.md +++ b/docs/architecture/fbp-todo.md @@ -13,12 +13,12 @@ required fields from its result. ## Input TODOs - [ ] Bring the existing legacy SFMS ground-slope and aspect rasters into the new pipeline, - following the same approach used for the legacy DEM. + following the same approach used for the legacy DEM. - Ground slope (`gs`) must be expressed as percent slope, not degrees. - Aspect is the direction the slope faces. Convert it to radians before calling CFFDRS. - Define nodata handling and the aspect value used for flat pixels. - [ ] Confirm whether production fuel grids contain the M3/M4 classification before sourcing - percent dead balsam fir (`pdf`). + percent dead balsam fir (`pdf`). - The temporary classification mapping reserves value `13` for M3/M4, but the temporary 2025 raster currently contains no value `13` pixels. - If a selected fuel grid contains M3/M4 pixels, identify an appropriate PDF source and decide @@ -30,7 +30,12 @@ required fields from its result. - It is only meaningful for O1A/O1B pixels. - The initial source and update cadence still need to be determined. - Define staleness and fallback rules once the source is selected. -- [ ] Integrate the daily Foliar Moisture Content (FMC) raster. +- [x] Generate one shared Foliar Moisture Content (FMC) raster per calendar date from the + SFMSNG elevation, latitude, and longitude grids. + - Actual jobs ensure their target date exists; forecast jobs ensure their three processed + dates exist. Existing GeoTIFF and COG pairs are reused. + - FMC rasters are stored under `sfms_ng/static/fmc/YYYY/MM/DD/` +- [ ] Integrate the daily FMC raster into the shared primary FBP calculation. - Treat valid daily FMC values as authoritative rather than asking CFFDRS to derive them. - Require FMC to be finite and greater than `0` and at most `120` on pixels being calculated. - Exclude missing or invalid FMC pixels with the common valid-pixel mask. Passing them into @@ -46,49 +51,49 @@ required fields from its result. ## Inputs Already Available or Derivable -| CFFDRS argument | Source or policy | Units and notes | -| --- | --- | --- | -| `fuel_type_code` | Year-specific fuel raster and SFMS classification mapping | Apply seasonal variants before converting to CFFDRS codes. | -| `ffmc` | Same-day FFMC raster | Existing FWI output. | -| `bui` | Same-day BUI raster | Existing FWI output. | -| `ws` | Same-day interpolated wind-speed raster | km/h. | -| `wd_rad` | Same-day interpolated wind-direction raster | Existing raster is meteorological degrees; convert to radians. | -| `gs` | Existing legacy SFMS slope raster | Percent slope; migrate and address it in the new pipeline. | -| `aspect_rad` | Existing legacy SFMS aspect raster | Downslope aspect converted to radians; migrate and address it in the new pipeline. | -| `pc` | Percent-conifer raster paired with the fuel-grid year | Required and validated on M1/M2 pixels. Use zero elsewhere. | -| `pdf` | Conditional percent-dead-balsam-fir source | First confirm M3/M4 occurs in the selected fuel grid. If it does, require and validate PDF on those pixels; use zero elsewhere. | -| `cc` | Grass-curing source to be determined | Required and validated on O1A/O1B pixels. Use zero elsewhere. | -| `gfl` | Fixed value | `0.35 kg/m²`, matching the existing SFC calculation. | -| `cbh` | Default policy to confirm | Candidate value: `0`, which selects the CFFDRS fuel-type default; confirm before implementation. | -| `cfl` | Default policy to confirm | Candidate value: `0`, which selects the CFFDRS fuel-type default; confirm before implementation. | -| `fmc` | Daily FMC raster | Require a finite value in `(0, 120]`; missing or invalid pixels become output nodata. | -| `isi` | Policy to be decided | Pass a positive value to use the existing daily ISI, or `0` to have CFFDRS derive it from FFMC and effective wind. | -| `lat` | Unused-input policy to confirm | Candidate placeholder: `0`; valid FMC prevents CFFDRS from reading it. Confirm before implementation. | -| `lon` | Unused-input policy to confirm | Candidate placeholder: `0`; valid FMC prevents CFFDRS from reading it. Confirm before implementation. | -| `elv` | Unused-input policy to confirm | Candidate placeholder: `0`; valid FMC prevents CFFDRS from reading it. Confirm before implementation. | -| `dj` | Unused-input policy to confirm | Candidate placeholder: `0`; valid FMC prevents CFFDRS from reading it. Confirm before implementation. | -| `d0` | Unused-input policy to confirm | Candidate placeholder: `0`; valid FMC prevents CFFDRS from reading it. Confirm before implementation. | -| `sd` | Default policy to confirm | Candidate value: `0`, which makes C6 use its fuel-type CBH default; confirm before implementation. | -| `sh` | Default policy to confirm | Candidate value: `0`, which makes C6 use its fuel-type CBH default; confirm before implementation. | -| `hr` | Primary-control policy to confirm | Candidate value: `0`; elapsed time is not used by the planned primary products. Confirm before implementation. | -| `theta_rad` | Primary-control policy to confirm | Candidate value: `0`; directional secondary outputs are not planned. Confirm before implementation. | -| `accel` | Primary-control policy to confirm | Candidate value: `0`, which produces equilibrium ROS; confirm before implementation. | -| `buieff` | Fixed calculation control | Pass `1` to apply the BUI effect. | +| CFFDRS argument | Source or policy | Units and notes | +| ---------------- | --------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | +| `fuel_type_code` | Year-specific fuel raster and SFMS classification mapping | Apply seasonal variants before converting to CFFDRS codes. | +| `ffmc` | Same-day FFMC raster | Existing FWI output. | +| `bui` | Same-day BUI raster | Existing FWI output. | +| `ws` | Same-day interpolated wind-speed raster | km/h. | +| `wd_rad` | Same-day interpolated wind-direction raster | Existing raster is meteorological degrees; convert to radians. | +| `gs` | Existing legacy SFMS slope raster | Percent slope; migrate and address it in the new pipeline. | +| `aspect_rad` | Existing legacy SFMS aspect raster | Downslope aspect converted to radians; migrate and address it in the new pipeline. | +| `pc` | Percent-conifer raster paired with the fuel-grid year | Required and validated on M1/M2 pixels. Use zero elsewhere. | +| `pdf` | Conditional percent-dead-balsam-fir source | First confirm M3/M4 occurs in the selected fuel grid. If it does, require and validate PDF on those pixels; use zero elsewhere. | +| `cc` | Grass-curing source to be determined | Required and validated on O1A/O1B pixels. Use zero elsewhere. | +| `gfl` | Fixed value | `0.35 kg/m²`, matching the existing SFC calculation. | +| `cbh` | Default policy to confirm | Candidate value: `0`, which selects the CFFDRS fuel-type default; confirm before implementation. | +| `cfl` | Default policy to confirm | Candidate value: `0`, which selects the CFFDRS fuel-type default; confirm before implementation. | +| `fmc` | Daily FMC raster | Require a finite value in `(0, 120]`; missing or invalid pixels become output nodata. | +| `isi` | Policy to be decided | Pass a positive value to use the existing daily ISI, or `0` to have CFFDRS derive it from FFMC and effective wind. | +| `lat` | Unused-input policy to confirm | Candidate placeholder: `0`; valid FMC prevents CFFDRS from reading it. Confirm before implementation. | +| `lon` | Unused-input policy to confirm | Candidate placeholder: `0`; valid FMC prevents CFFDRS from reading it. Confirm before implementation. | +| `elv` | Unused-input policy to confirm | Candidate placeholder: `0`; valid FMC prevents CFFDRS from reading it. Confirm before implementation. | +| `dj` | Unused-input policy to confirm | Candidate placeholder: `0`; valid FMC prevents CFFDRS from reading it. Confirm before implementation. | +| `d0` | Unused-input policy to confirm | Candidate placeholder: `0`; valid FMC prevents CFFDRS from reading it. Confirm before implementation. | +| `sd` | Default policy to confirm | Candidate value: `0`, which makes C6 use its fuel-type CBH default; confirm before implementation. | +| `sh` | Default policy to confirm | Candidate value: `0`, which makes C6 use its fuel-type CBH default; confirm before implementation. | +| `hr` | Primary-control policy to confirm | Candidate value: `0`; elapsed time is not used by the planned primary products. Confirm before implementation. | +| `theta_rad` | Primary-control policy to confirm | Candidate value: `0`; directional secondary outputs are not planned. Confirm before implementation. | +| `accel` | Primary-control policy to confirm | Candidate value: `0`, which produces equilibrium ROS; confirm before implementation. | +| `buieff` | Fixed calculation control | Pass `1` to apply the BUI effect. | ## Pipeline Requirements - [ ] Define a shared `FBPInputs` raster contract once the unresolved data sources are known. - [ ] Require all input rasters to match the selected fuel grid's extent, resolution, projection, - and geotransform. + and geotransform. - [ ] Validate fuel-specific inputs only where they are meaningful: PC on M1/M2, PDF on M3/M4, - and grass curing on O1A/O1B. + and grass curing on O1A/O1B. - [ ] Apply the BC mask as the final mask for every primary output. Publish nodata outside BC and - where required inputs are missing or invalid; publish `0` for recognized non-combustible fuel - pixels inside BC. + where required inputs are missing or invalid; publish `0` for recognized non-combustible fuel + pixels inside BC. - [ ] Replace `SurfaceFuelConsumptionProcessor` rather than running both the standalone SFC and - shared primary FBP calculations. + shared primary FBP calculations. - [ ] During that transition, verify the shared primary calculation's SFC output matches the - standalone SFC calculation for every supported fuel type. + standalone SFC calculation for every supported fuel type. ## Deferred Outputs