From b82ee017f5eb33883054c0619ec8e6a26b7fb86f Mon Sep 17 00:00:00 2001 From: Abdelrahman Katkat Date: Sun, 1 Mar 2026 23:54:04 +0200 Subject: [PATCH 01/30] feat(ramp): add Dockerfile and pipeline for EfficientNetB0 + U-Net semantic segmentation model - Introduced Dockerfile for building the RAMP model environment with GPU and CPU support. - Added pipeline.py for defining the ZenML pipeline, including preprocessing, training, inference, and postprocessing steps. - Created README.md to document the model architecture, usage, and data layout. - Implemented stac-item.json for STAC catalog integration. - Included smoke tests to validate the Docker runtime and model functionality. - Updated .gitignore to exclude new data directories. --- .gitignore | 1 + models/ramp/Dockerfile | 103 +++ models/ramp/README.md | 195 ++++++ models/ramp/pipeline.py | 645 ++++++++++++++++++ models/ramp/stac-item.json | 154 +++++ models/ramp/tests/CODE_EXPLAINED.md | 465 +++++++++++++ models/ramp/tests/README.md | 137 ++++ .../ramp/tests/inside_container_smoke_test.py | 403 +++++++++++ models/ramp/tests/run_docker_tests.ps1 | 34 + models/ramp/tests/run_docker_tests.sh | 33 + models/ramp/tests/test_plan.yaml | 58 ++ 11 files changed, 2228 insertions(+) create mode 100644 models/ramp/Dockerfile create mode 100644 models/ramp/README.md create mode 100644 models/ramp/pipeline.py create mode 100644 models/ramp/stac-item.json create mode 100644 models/ramp/tests/CODE_EXPLAINED.md create mode 100644 models/ramp/tests/README.md create mode 100644 models/ramp/tests/inside_container_smoke_test.py create mode 100644 models/ramp/tests/run_docker_tests.ps1 create mode 100644 models/ramp/tests/run_docker_tests.sh create mode 100644 models/ramp/tests/test_plan.yaml diff --git a/.gitignore b/.gitignore index d3dc4e47..7ee5ae04 100644 --- a/.gitignore +++ b/.gitignore @@ -220,6 +220,7 @@ implementation.md # data data/sample/predict/predictions +data/sample/ramp_work stac_catalog # hatch-vcs generated version file diff --git a/models/ramp/Dockerfile b/models/ramp/Dockerfile new file mode 100644 index 00000000..45bc49f6 --- /dev/null +++ b/models/ramp/Dockerfile @@ -0,0 +1,103 @@ +# Build instructions (from fAIr-models repo root): +# GPU: docker build -f models/ramp/Dockerfile --build-arg BUILD_TYPE=gpu -t ramp-v1:gpu . +# CPU: docker build -f models/ramp/Dockerfile --build-arg BUILD_TYPE=cpu -t ramp-v1:cpu . + +ARG PY_VER=3.10 +ARG TF_VER=2.9.3 +ARG BUILD_TYPE=gpu +ARG CUDA_TAG=11.8.0-cudnn8-runtime-ubuntu22.04 + +# ============================================================================== +# === CPU base image (minimal) ================================================= +FROM python:${PY_VER}-slim-bookworm AS cpu-base +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + git build-essential gcc g++ python3-dev python3-rtree \ + gdal-bin libgdal-dev python3-gdal python3-opencv libspatialindex-dev libgeos-dev \ + libgl1 libglib2.0-0 \ + && rm -rf /var/lib/apt/lists/* + +# ============================================================================== +# === GPU base image (CUDA + runtime-only Python & GDAL) ======================= +FROM nvidia/cuda:${CUDA_TAG} AS gpu-base +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + git python3 python3-pip build-essential gcc g++ python3-dev python3-rtree python-is-python3 \ + gdal-bin libgdal-dev python3-gdal python3-opencv libspatialindex-dev libgeos-dev \ + libgl1 libglib2.0-0 \ + && rm -rf /var/lib/apt/lists/* && \ + python3 -m pip install --upgrade pip + +# ============================================================================== +# === Builder stage (installs everything) ====================================== +FROM ${BUILD_TYPE}-base AS builder +ENV DEBIAN_FRONTEND=noninteractive \ + TF_CPP_MIN_LOG_LEVEL=2 +ARG TF_VER + +ENV CPLUS_INCLUDE_PATH=/usr/include/gdal \ + C_INCLUDE_PATH=/usr/include/gdal + +# Use pip cache and install Python packages +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install --no-cache-dir --upgrade pip more-itertools && \ + pip install --no-cache-dir "numpy>=1.22,<2.0" && \ + pip install --no-cache-dir "GDAL==$(gdal-config --version)" && \ + pip install --no-cache-dir tensorflow==${TF_VER} && \ + pip install --no-cache-dir \ + "efficientnet==1.0.0" \ + "image-classifiers==1.0.0" \ + "segmentation-models==1.0.1" && \ + pip install --no-cache-dir \ + "geopandas==0.10.2" \ + "rasterio>=1.3.10" \ + "Shapely>=1.8.1" \ + "pyproj==3.3.0" \ + "scikit-image==0.19.2" \ + "scikit-learn==1.0.2" \ + "scipy==1.8.0" \ + "tqdm==4.62.3" \ + "pandas==1.4.1" \ + "matplotlib==3.5.1" \ + "albumentations==1.0.3" \ + "pyyaml>=6.0" \ + "requests>=2.27.1" \ + "tinydb==4.7.0" \ + "gdown>=5.0" + +# Compatibility pin for geopandas/raster stack +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install --no-cache-dir "fiona>=1.8.22,<1.10" + +# Install solaris from ramp-code-fair (equivalent to Dockerfile.pb local install) +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install --no-cache-dir \ + git+https://github.com/hotosm/ramp-code-fair.git#subdirectory=solaris + +# Install scikit-fmm and ramp-fair +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install --no-cache-dir scikit-fmm && \ + pip install --no-cache-dir "ramp-fair==0.1.2" + +# Install hot-fair-utilities for shared preprocessing path used by pipeline.py +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install --no-cache-dir "opencv-python-headless<=4.7.0.68" && \ + pip install --no-cache-dir "hot-fair-utilities==2.0.12" && \ + pip install --no-cache-dir "numpy>=1.22,<2.0" + +# ============================================================================== +# === Final minimal runtime image ============================================== +FROM ${BUILD_TYPE}-base AS final +ENV DEBIAN_FRONTEND=noninteractive + +ENV CPLUS_INCLUDE_PATH=/usr/include/gdal \ + C_INCLUDE_PATH=/usr/include/gdal \ + RAMP_HOME=/workspace + +COPY --from=builder /usr/local /usr/local +COPY --from=builder /usr/lib/python*/ /usr/lib/python*/ +COPY --from=builder /usr/include/gdal /usr/include/gdal + +WORKDIR /workspace diff --git a/models/ramp/README.md b/models/ramp/README.md new file mode 100644 index 00000000..95cd58c6 --- /dev/null +++ b/models/ramp/README.md @@ -0,0 +1,195 @@ +# RAMP — EfficientNetB0 + U-Net Building Semantic Segmentation + +RAMP (Replicable AI for Microplanning) is an EfficientNetB0 encoder + U-Net decoder +for pixel-wise 4-class building segmentation on 256 × 256 px aerial image chips. + +## Architecture overview + +| Component | Detail | +| --- | --- | +| Encoder | EfficientNet-B0 (ImageNet pre-trained via segmentation_models) | +| Decoder | U-Net symmetric decoder with skip connections | +| Output | 4-class sparse categorical mask (channels-last, uint8) | +| Classes | 0=background, 1=building, 2=boundary, 3=contact-point | +| Loss | Sparse categorical crossentropy | +| Metric | Sparse categorical accuracy (`val_sparse_categorical_accuracy`) | +| Framework | TensorFlow 2.9.3 / Keras | + +The boundary (class 2) and contact-point (class 3) channels help the model cleanly +separate adjacent buildings at inference time, even when they share a wall. + +## Key difference from YOLO packs + +| Stage | YOLO v8 v1/v2 | RAMP | +| --- | --- | --- | +| Preprocessing | `hot_fair_utilities.preprocess` | same | +| Training | `hot_fair_utilities.training.yolo_v8_*` (ultralytics) | `ramp.training.*` (TF/Keras) | +| Inference | `hot_fair_utilities.predict` (ultralytics) | `tf.keras.models.load_model` | +| Postprocessing | `hot_fair_utilities.polygonize` (AutoBFE) | `ramp.utils.mask_to_vec_utils` (GDAL) | + +## Model pack contents + +| File | Purpose | +| --- | --- | +| `pipeline.py` | ZenML `@step` / `@pipeline` entrypoints (pre → train → infer → post); `resolve_model_href()` for URLs | +| `stac-item.json` | STAC MLM item — model weights (mlm:model href), entrypoints; weights from Google Drive/S3 | +| `Dockerfile` | Isolated runtime (CUDA 11.8 + TF 2.9.3 + ramp-fair + hot-fair-utilities + gdown) | + +## Data directory layout + +**Option A: Use shared `data/sample`** (recommended for tests) + +The smoke tests use `data/sample` (train/oam + train/osm). The test script +auto-converts OAM GeoTIFFs to PNG and merges OSM labels into the RAMP format. + +**Option B: Legacy layout** + +```text +dataset/ +├── input/ +│ ├── *.png # OAM chips (PNG, no geo-reference) +│ └── labels.geojson # combined building polygon labels +├── preprocessed/ # created by run_preprocessing +│ ├── chips/ # georeferenced .tif chips (EPSG:3857) +│ ├── labels/ # per-chip .geojson labels +│ ├── multimasks/ # 4-class .mask.tif targets +│ ├── val_chips/ # validation split chips +│ ├── val_multimasks/ # validation split masks +│ └── checkpoints// # Keras SavedModel checkpoints +└── prediction/ + ├── input/ # chips for inference (GeoTIFFs; typically copy from preprocessed/chips/) + ├── output/ # .pred.tif predicted masks + └── vectors/ # per-chip .geojson building polygons +``` + +## Pipeline steps + +```text +input/ (PNG chips + labels.geojson) + │ + ▼ +[run_preprocessing] hot_fair_utilities.preprocess (georeference + multimask) + │ preprocessed/chips/ + preprocessed/multimasks/ + ▼ +[train_model] ramp.training.* (EfficientNetB0 U-Net, TF/Keras) + │ best checkpoint (.tf SavedModel dir) + val_sparse_categorical_accuracy + ▼ +[run_inference] tf.keras.models.load_model (chip-by-chip predict) + │ prediction/output/*.pred.tif + ▼ +[run_postprocessing] ramp.utils.mask_to_vec_utils (GDAL Polygonize) + │ + ▼ +prediction/vectors/*.geojson (per-chip building footprints) +``` + +## Running locally (outside ZenML) + +```python +from models.ramp.pipeline import training_pipeline, inference_pipeline + +# Full training run (use your dataset path) +training_pipeline( + input_path="data/sample/ramp_work/input", # or your dataset/input + output_path="data/sample/ramp_work", # or your dataset + backbone="efficientnetb0", + num_epochs=100, + batch_size=16, + learning_rate=3e-4, + early_stopping_patience=35, + val_fraction=0.15, +) + +# Inference run (model_uri from STAC or local path) +inference_pipeline( + model_uri="data/sample/ramp_work/preprocessed_test/checkpoints//smoke_.tf", + input_path="data/sample/ramp_work/preprocessed_test/chips", + prediction_path="data/sample/ramp_work/prediction_test/output", + output_dir="data/sample/ramp_work/prediction_test/vectors", +) +# model_uri can also be: Google Drive folder URL, HTTP URL to .zip +``` + +## Building the Docker image + +```bash +# GPU image (default) +docker build -t ramp-v1:gpu \ + --build-arg BUILD_TYPE=gpu \ + -f models/ramp/Dockerfile . + +# CPU-only (development / CI) +docker build -t ramp-v1:cpu \ + --build-arg BUILD_TYPE=cpu \ + -f models/ramp/Dockerfile . +``` + +## Running the smoke tests + +```powershell +# PowerShell (Windows) +.\models\ramp\tests\run_docker_tests.ps1 -BuildImage +``` + +```bash +# Bash (Linux / macOS) +BUILD_IMAGE=1 ./models/ramp/tests/run_docker_tests.sh +``` + +> **Note**: The smoke tests use `data/sample` (train/oam OAM tiles + train/osm labels). +> Run from the fAIr-models repo root so `/workspace/data/sample` is available in the container. + +## Model weights (STAC mlm:model asset) + +The STAC Item's `assets.model.href` points to pretrained weights. Supported sources: + +| Source | Example | +| --- | --- | +| Local path | `/workspace/checkpoints/model.tf` | +| Google Drive folder | `https://drive.google.com/drive/folders/FOLDER_ID` | +| HTTP .zip | `https://example.com/ramp_model.zip` | + +**Google Drive**: Upload the **full** Keras SavedModel directory (saved_model.pb + variables/ + assets/). The pipeline downloads via gdown and caches to `/workspace/.ramp_model_cache/`. + +## Registering in the STAC catalog + +```python +from fair.stac.catalog_manager import CatalogManager + +cm = CatalogManager() +cm.register_model("models/ramp/stac-item.json") +``` + +## Dependencies from hot_fair_utilities + +Only the **preprocessing** path of `hot-fair-utilities` is used in this pack: + +| Used | Not used | +| --- | --- | +| `hot_fair_utilities.preprocess` (+ `multimasks_from_polygons`) | `hot_fair_utilities.predict` (YOLO / ultralytics) | +| `ramp.utils.multimask_utils` via transitive import | `hot_fair_utilities.polygonize` (AutoBFE, YOLO output) | + +The `ultralytics` and `torch` packages are installed transitively by +`hot-fair-utilities` but are never imported at runtime for RAMP. + +## Key design decisions + +**Why ramp-fair and not inline model code?** +`ramp-fair` provides the complete EfficientNetB0 U-Net + training utilities. +This pack is thin: it declares *how* to run the model (pipeline.py) and +*what* it is (stac-item.json). Upgrading means bumping the `ramp-fair` pin. + +**Why solaris from GitHub source?** +Solaris is a geospatial ML toolkit vendored inside `ramp-code-fair`. It is +not on PyPI. The Dockerfile installs it directly from the +`hotosm/ramp-code-fair` GitHub repository's `solaris/` subdirectory. + +**Why separate val split in pipeline.py?** +RAMP's `data_generator` requires explicitly separate `train_img_dir` and +`val_img_dir`. The pipeline handles this automatically by shuffling a +configurable fraction (`val_fraction`, default 15%) of chips out of the +training set after preprocessing. + +**Why one Dockerfile per model?** +TF 2.9.3 required for RAMP is incompatible with the TF 2.13 needed by the +YOLO packs' base image. Per-model images prevent version conflicts. diff --git a/models/ramp/pipeline.py b/models/ramp/pipeline.py new file mode 100644 index 00000000..848e840b --- /dev/null +++ b/models/ramp/pipeline.py @@ -0,0 +1,645 @@ +"""ZenML pipeline for RAMP (EfficientNetB0 + U-Net) building semantic segmentation. + +Entrypoints referenced by models/ramp/stac-item.json. +Runtime: ramp-fair (TensorFlow/Keras), hot-fair-utilities (preprocessing only). + +Implements the fAIr 3.0 contract (FAIr_3.0_Optimized_Pipeline.md): + - pre_processing_function → preprocess() + - post_processing_function → postprocess() + - mlm:entrypoint (training) → training_pipeline() + - inference (model from STAC mlm:model asset href) → inference_pipeline() + +Key architecture difference from YOLO packs: + - Preprocessing → hot_fair_utilities.preprocess (shared) + - Training → ramp.training.* (TF/Keras, NOT ultralytics) + - Inference → tf.keras.models.load_model (TF SavedModel) + - Postprocessing → ramp.utils.mask_to_vec_utils (GDAL polygonize) + +Model weights: Backend passes model_uri from STAC Item (assets.model.href). +Supports Google Drive, direct HTTP URLs, S3 (future), and local paths. +Weights are downloaded on first use and cached locally. + +All heavy imports are lazy: this module is importable in the fAIr-models +host environment where tensorflow, ramp, and solaris are not installed. +""" + +from __future__ import annotations + +import datetime +import random +import re +import shutil +import zipfile +from pathlib import Path +from typing import Annotated +from urllib.request import urlretrieve + +from annotated_types import Ge, Le +from zenml import log_metadata, pipeline, step + +# Cache directory for downloaded models (inside container, typically /workspace) +_DEFAULT_MODEL_CACHE = Path("/workspace/.ramp_model_cache") + +# Google Drive folder URL pattern +_GDRIVE_FOLDER_RE = re.compile( + r"https?://drive\.google\.com/drive/folders/([a-zA-Z0-9_-]+)", + re.IGNORECASE, +) +_GDRIVE_FILE_RE = re.compile( + r"https?://drive\.google\.com/file/d/([a-zA-Z0-9_-]+)", + re.IGNORECASE, +) + + +def resolve_model_href( + model_uri: str, + cache_dir: Path | None = None, +) -> str: + """Resolve model_uri to a local SavedModel directory path. + + Supports: + - Local path: returned as-is if it exists and contains saved_model.pb + - Google Drive folder URL: downloaded via gdown, cached + - Direct HTTP(S) URL to .zip: downloaded, extracted, cached + - S3 URLs (s3://...): placeholder for future; raise if not implemented + + Returns the absolute path to the SavedModel directory (contains saved_model.pb). + """ + cache_dir = cache_dir or _DEFAULT_MODEL_CACHE + path = Path(model_uri) + + # Local path: must exist and look like a SavedModel dir + if not ( + model_uri.startswith("http://") + or model_uri.startswith("https://") + or model_uri.startswith("s3://") + ): + resolved = path.resolve() + if resolved.is_dir() and (resolved / "saved_model.pb").is_file(): + return str(resolved) + if resolved.exists(): + return str(resolved) + raise FileNotFoundError(f"Model path not found or invalid: {resolved}") + + # S3: future support (backend could download before calling) + if model_uri.startswith("s3://"): + raise NotImplementedError( + "S3 model URIs require backend pre-download or boto3 in container. " + "Use Google Drive or HTTP URL for now." + ) + + # Google Drive folder + folder_match = _GDRIVE_FOLDER_RE.search(model_uri) + if folder_match: + folder_id = folder_match.group(1) + dest_dir = cache_dir / f"gdrive_{folder_id}" + dest_dir.mkdir(parents=True, exist_ok=True) + + # Check if already downloaded + if (dest_dir / "saved_model.pb").is_file(): + return str(dest_dir) + + try: + import gdown + + gdown.download_folder( + id=folder_id, + output=str(dest_dir), + quiet=True, + ) + except ImportError as e: + raise ImportError( + "gdown is required to download models from Google Drive. " + "Add 'gdown' to the RAMP Dockerfile." + ) from e + + if not (dest_dir / "saved_model.pb").is_file(): + # gdown may create a subfolder with the Drive folder name + subdirs = [d for d in dest_dir.iterdir() if d.is_dir()] + if len(subdirs) == 1 and (subdirs[0] / "saved_model.pb").is_file(): + return str(subdirs[0]) + raise RuntimeError( + f"Downloaded folder {dest_dir} does not contain saved_model.pb. " + "Ensure the Drive folder contains the full Keras SavedModel (saved_model.pb + variables/)." + ) + return str(dest_dir) + + # Google Drive single file (less common for SavedModel) + file_match = _GDRIVE_FILE_RE.search(model_uri) + if file_match: + file_id = file_match.group(1) + dest_dir = cache_dir / f"gdrive_file_{file_id}" + dest_dir.mkdir(parents=True, exist_ok=True) + out_file = dest_dir / "downloaded" + try: + import gdown + + gdown.download(id=file_id, output=str(out_file), quiet=True) + except ImportError as e: + raise ImportError("gdown required for Google Drive downloads.") from e + if out_file.suffix == ".zip": + with zipfile.ZipFile(out_file, "r") as zf: + zf.extractall(dest_dir) + out_file.unlink() + saved_pb = dest_dir / "saved_model.pb" + if saved_pb.is_file(): + return str(dest_dir) + for sub in dest_dir.rglob("saved_model.pb"): + return str(sub.parent) + raise RuntimeError(f"Downloaded file did not yield a valid SavedModel in {dest_dir}") + + # Direct HTTP(S) URL to .zip + if model_uri.lower().endswith(".zip"): + base_name = Path(model_uri.split("/")[-1]).stem + dest_dir = cache_dir / base_name + dest_dir.mkdir(parents=True, exist_ok=True) + zip_path = cache_dir / (base_name + ".zip") + if not any(dest_dir.rglob("saved_model.pb")): + urlretrieve(model_uri, zip_path) + with zipfile.ZipFile(zip_path, "r") as zf: + zf.extractall(dest_dir) + zip_path.unlink(missing_ok=True) + for sub in dest_dir.rglob("saved_model.pb"): + return str(sub.parent) + raise RuntimeError(f"Zip from {model_uri} did not contain a valid SavedModel") + + raise ValueError( + f"Unsupported model_uri format: {model_uri}. " + "Use: local path, Google Drive folder URL, or HTTP(S) URL to a .zip file." + ) + + +# --------------------------------------------------------------------------- +# STAC MLM processing-expression callables +# --------------------------------------------------------------------------- + + +def preprocess( + input_path: str, + output_path: str, + boundary_width: int = 3, + contact_spacing: int = 8, +) -> str: + """Preprocess OAM chips + labels for RAMP training. + + Step 1 — Georeference PNGs → chips/*.tif (EPSG:3857) + Step 2 — Reproject + clip labels → labels/*.geojson (per chip) + Step 3 — Generate 4-channel sparse multimasks → multimasks/*.mask.tif + Classes: 0=background, 1=building, 2=boundary, 3=contact-point + + Returns the preprocessed output directory path. + """ + from hot_fair_utilities import preprocess as _preprocess + + _preprocess( + input_path=input_path, + output_path=output_path, + rasterize=True, + rasterize_options=["binary"], + georeference_images=True, + multimasks=True, + input_boundary_width=boundary_width, + input_contact_spacing=contact_spacing, + ) + return output_path + + +def postprocess(prediction_masks_dir: str, output_dir: str) -> str: + """Convert RAMP multichannel predicted masks to per-chip GeoJSON polygons. + + For each .pred.tif in prediction_masks_dir: + 1. Reads the 4-class sparse multimask (uint8, channels-first, 1 band). + 2. Extracts a binary building footprint mask (class == 1, ignores boundary/contact). + 3. Polygonizes via GDAL and writes a matching .geojson file. + + Returns the output_dir path containing per-chip GeoJSONs. + """ + from osgeo import gdal + + from ramp.utils.img_utils import gdal_get_mask_tensor + from ramp.utils.mask_to_vec_utils import ( + binary_mask_from_multichannel_mask, + binary_mask_to_geojson, + ) + + pred_dir = Path(prediction_masks_dir) + out_dir = Path(output_dir) + out_dir.mkdir(parents=True, exist_ok=True) + + pred_tifs = sorted(pred_dir.glob("*.pred.tif")) + if not pred_tifs: + raise RuntimeError(f"No *.pred.tif files found in {pred_dir}") + + for mask_path in pred_tifs: + json_name = mask_path.stem.replace(".pred", "") + ".geojson" + json_path = str(out_dir / json_name) + + ref_ds = gdal.Open(str(mask_path)) + if ref_ds is None: + raise RuntimeError(f"GDAL could not open {mask_path}") + + multimask = gdal_get_mask_tensor(str(mask_path)) + bin_mask = binary_mask_from_multichannel_mask(multimask) + binary_mask_to_geojson(bin_mask, ref_ds, json_path) + + return output_dir + + +# --------------------------------------------------------------------------- +# Private helpers +# --------------------------------------------------------------------------- + + +def _make_val_split( + chips_dir: Path, + masks_dir: Path, + val_chips_dir: Path, + val_masks_dir: Path, + val_fraction: float = 0.15, +) -> None: + """Move val_fraction of chip+mask pairs to validation directories. + + RAMP mask filenames follow the convention .mask.tif so we + match chips to their masks by stem before moving. + """ + chip_files = sorted(chips_dir.glob("*.tif")) + if not chip_files: + raise RuntimeError(f"No .tif chips found in {chips_dir}") + + n_val = max(1, int(len(chip_files) * val_fraction)) + random.shuffle(chip_files) + val_chips = chip_files[:n_val] + + val_chips_dir.mkdir(parents=True, exist_ok=True) + val_masks_dir.mkdir(parents=True, exist_ok=True) + + moved = 0 + for chip_path in val_chips: + mask_path = masks_dir / (chip_path.stem + ".mask.tif") + if mask_path.is_file(): + shutil.move(str(chip_path), val_chips_dir / chip_path.name) + shutil.move(str(mask_path), val_masks_dir / mask_path.name) + moved += 1 + + if moved == 0: + raise RuntimeError( + f"Val split produced 0 pairs. " + f"Check that chips in {chips_dir} match masks in {masks_dir}." + ) + + +def _build_train_config( + chips_subdir: str, + masks_subdir: str, + val_chips_subdir: str, + val_masks_subdir: str, + checkpts_subdir: str, + backbone: str, + num_epochs: int, + batch_size: int, + learning_rate: float, + early_stopping_patience: int, + timestamp: str, +) -> dict: + """Build a RAMP JSON training config dict from pipeline parameters.""" + return { + "experiment_name": "RAMP EffUnet multimask training", + "discard_experiment": False, + "logging": {"log_experiment": False}, + "datasets": { + "train_img_dir": chips_subdir, + "train_mask_dir": masks_subdir, + "val_img_dir": val_chips_subdir, + "val_mask_dir": val_masks_subdir, + }, + "num_classes": 4, + "num_epochs": num_epochs, + "batch_size": batch_size, + "input_img_shape": [256, 256], + "output_img_shape": [256, 256], + "loss": { + "get_loss_fn_name": "get_sparse_categorical_crossentropy_fn", + "loss_fn_parms": {}, + }, + "metrics": { + "use_metrics": True, + "get_metrics_fn_names": ["get_sparse_categorical_accuracy_fn"], + "metrics_fn_parms": [{}], + }, + "optimizer": { + "get_optimizer_fn_name": "get_adam_optimizer", + "optimizer_fn_parms": {"learning_rate": learning_rate}, + }, + "model": { + "get_model_fn_name": "get_effunet_model", + "model_fn_parms": { + "backbone": backbone, + "classes": ["background", "building", "boundary", "contact"], + }, + }, + "saved_model": {"use_saved_model": False}, + "augmentation": {"use_aug": False}, + "early_stopping": { + "use_early_stopping": True, + "early_stopping_parms": { + "monitor": "val_loss", + "min_delta": 0.005, + "patience": early_stopping_patience, + "verbose": 1, + "mode": "auto", + "restore_best_weights": True, + }, + }, + "cyclic_learning_scheduler": {"use_clr": False}, + "tensorboard": {"use_tb": False}, + "prediction_logging": {"use_prediction_logging": False}, + "model_checkpts": { + "use_model_checkpts": True, + "model_checkpts_dir": checkpts_subdir, + "get_model_checkpt_callback_fn_name": "get_model_checkpt_callback_fn", + "model_checkpt_callback_parms": {"mode": "max", "save_best_only": True}, + }, + "random_seed": 20220523, + "timestamp": timestamp, + } + + +# --------------------------------------------------------------------------- +# ZenML steps +# --------------------------------------------------------------------------- + + +@step +def run_preprocessing( + input_path: str, + output_path: str, + boundary_width: int = 3, + contact_spacing: int = 8, +) -> str: + """Georeference OAM chips and generate 4-class multimasks. Returns preprocessed dir.""" + return preprocess(input_path, output_path, boundary_width, contact_spacing) + + +@step +def train_model( + data_base_path: str, + preprocessed_path: str, + backbone: str = "efficientnetb0", + num_epochs: int = 100, + batch_size: int = 16, + learning_rate: float = 3e-4, + early_stopping_patience: int = 35, + val_fraction: float = 0.15, +) -> str: + """Fine-tune EfficientNetB0 + U-Net on 4-class multimask chips. + + 1. Splits preprocessed chips/masks into train and val sets. + 2. Builds the EfficientNet-B0 U-Net from segmentation_models. + 3. Trains with sparse categorical crossentropy loss. + 4. Returns the path to the best Keras SavedModel directory. + + val_sparse_categorical_accuracy is logged as ZenML step metadata. + """ + import os + + import segmentation_models as sm + + sm.set_framework("tf.keras") + + from ramp.data_mgmt.data_generator import ( + test_batches_from_gtiff_dirs, + training_batches_from_gtiff_dirs, + ) + from ramp.training import ( + callback_constructors, + loss_constructors, + metric_constructors, + model_constructors, + optimizer_constructors, + ) + from ramp.utils.model_utils import get_best_model_value_and_epoch + + os.environ["RAMP_HOME"] = data_base_path + + pre_path = Path(preprocessed_path) + chips_dir = pre_path / "chips" + masks_dir = pre_path / "multimasks" + val_chips_dir = pre_path / "val_chips" + val_masks_dir = pre_path / "val_multimasks" + checkpts_dir = pre_path / "checkpoints" + + if not val_chips_dir.is_dir(): + _make_val_split(chips_dir, masks_dir, val_chips_dir, val_masks_dir, val_fraction) + + def _rel(d: Path) -> str: + return str(d.relative_to(data_base_path)) + + timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + cfg = _build_train_config( + chips_subdir=_rel(chips_dir), + masks_subdir=_rel(masks_dir), + val_chips_subdir=_rel(val_chips_dir), + val_masks_subdir=_rel(val_masks_dir), + checkpts_subdir=_rel(checkpts_dir), + backbone=backbone, + num_epochs=num_epochs, + batch_size=batch_size, + learning_rate=learning_rate, + early_stopping_patience=early_stopping_patience, + timestamp=timestamp, + ) + + loss_fn = loss_constructors.get_sparse_categorical_crossentropy_fn(cfg) + optimizer = optimizer_constructors.get_adam_optimizer(cfg) + accuracy_metric = metric_constructors.get_sparse_categorical_accuracy_fn({}) + the_model = model_constructors.get_effunet_model(cfg) + the_model.compile(optimizer=optimizer, loss=loss_fn, metrics=[accuracy_metric]) + + n_train = len(list(chips_dir.glob("*.tif"))) + n_val = len(list(val_chips_dir.glob("*.tif"))) + steps_per_epoch = max(1, n_train // batch_size) + validation_steps = max(1, n_val // batch_size) + cfg["runtime"] = { + "n_training": n_train, + "n_val": n_val, + "steps_per_epoch": steps_per_epoch, + "validation_steps": validation_steps, + } + + img_shape = cfg["input_img_shape"] + mask_shape = cfg["output_img_shape"] + + train_batches = training_batches_from_gtiff_dirs( + chips_dir, masks_dir, batch_size, img_shape, mask_shape + ) + val_batches = test_batches_from_gtiff_dirs( + val_chips_dir, val_masks_dir, batch_size, img_shape, mask_shape + ) + + callbacks = [ + callback_constructors.get_early_stopping_callback_fn(cfg), + callback_constructors.get_model_checkpt_callback_fn(cfg), + ] + + history = the_model.fit( + train_batches, + epochs=num_epochs, + steps_per_epoch=steps_per_epoch, + validation_data=val_batches, + validation_steps=validation_steps, + callbacks=callbacks, + ) + + best_epoch, best_val_acc = get_best_model_value_and_epoch(history) + log_metadata( + metadata={ + "best_val_sparse_categorical_accuracy": float(best_val_acc), + "best_epoch": int(best_epoch), + } + ) + + checkpts_ts_dir = checkpts_dir / timestamp + checkpoints = sorted(checkpts_ts_dir.glob("*.tf")) + if not checkpoints: + raise RuntimeError(f"No .tf checkpoint found in {checkpts_ts_dir}") + return str(checkpoints[-1]) + + +@step +def run_inference( + model_uri: str, + input_path: str, + prediction_path: str, + model_cache_dir: str | None = None, +) -> str: + """Run RAMP EfficientNetB0 U-Net inference on georeferenced chips. + + model_uri: From STAC Item (assets.model.href). Can be: + - Local path to SavedModel directory + - Google Drive folder URL + - HTTP(S) URL to .zip containing SavedModel + Resolves URLs to local path (downloads and caches if needed). + + Loads Keras SavedModel, runs prediction chip-by-chip, and writes + one .pred.tif (single-band uint8 sparse mask) per input chip. + + Returns prediction_path containing the .pred.tif files. + """ + import numpy as np + import rasterio as rio + import tensorflow as tf + from tqdm import tqdm + + from ramp.data_mgmt.display_data import get_mask_from_prediction + from ramp.utils.file_utils import get_basename + from ramp.utils.img_utils import to_channels_first, to_channels_last + + cache = Path(model_cache_dir) if model_cache_dir else None + model_dir = resolve_model_href(model_uri, cache_dir=cache) + model = tf.keras.models.load_model(model_dir, compile=False) + out_dir = Path(prediction_path) + out_dir.mkdir(parents=True, exist_ok=True) + + chip_files = sorted(Path(input_path).glob("**/*.tif")) + if not chip_files: + png_files = sorted(Path(input_path).glob("**/*.png")) + if png_files: + raise RuntimeError( + "No GeoTIFF chips (*.tif) found for inference. " + f"Found {len(png_files)} PNG(s) in {input_path}. " + "RAMP inference expects georeferenced RGB GeoTIFF chips (typically the output of " + "`preprocess(...)` under `/chips/`). " + "Run preprocessing and point `input_path` to the resulting `chips/` directory." + ) + raise RuntimeError( + f"No GeoTIFF chips (*.tif) found in {input_path}. " + "RAMP inference expects georeferenced RGB GeoTIFF chips (typically under `/chips/`)." + ) + + for chip_file in tqdm(chip_files, desc="RAMP inference"): + bname = get_basename(str(chip_file)) + mask_name = bname + ".pred.tif" + with rio.open(chip_file) as src: + dst_profile = src.profile.copy() + dst_profile["count"] = 1 + dst_profile["dtype"] = "uint8" + img = to_channels_last(src.read()).astype("float32") + max_val = float(img.max()) + if max_val > 0: + img = img / max_val + predicted = get_mask_from_prediction(model.predict(np.expand_dims(img, 0))) + predicted = np.squeeze(predicted, axis=0) + with rio.open(out_dir / mask_name, "w", **dst_profile) as dst: + dst.write(to_channels_first(predicted)) + + return prediction_path + + +@step +def run_postprocessing( + prediction_path: str, + output_dir: str, +) -> str: + """Polygonize RAMP predicted masks into per-chip building GeoJSONs.""" + return postprocess(prediction_path, output_dir) + + +# --------------------------------------------------------------------------- +# ZenML pipelines +# --------------------------------------------------------------------------- + + +@pipeline +def training_pipeline( + input_path: str, + output_path: str, + backbone: str = "efficientnetb0", + num_epochs: Annotated[int, Ge(1), Le(2000)] = 100, + batch_size: Annotated[int, Ge(1), Le(64)] = 16, + learning_rate: Annotated[float, Ge(1e-6), Le(1e-2)] = 3e-4, + early_stopping_patience: Annotated[int, Ge(5), Le(100)] = 35, + val_fraction: Annotated[float, Ge(0.05), Le(0.4)] = 0.15, + boundary_width: int = 3, + contact_spacing: int = 8, +) -> None: + """Full RAMP training: georeference + multimask → val split → EfficientNetB0 U-Net.""" + preprocessed_path = run_preprocessing( + input_path=input_path, + output_path=f"{output_path}/preprocessed", + boundary_width=boundary_width, + contact_spacing=contact_spacing, + ) + train_model( + data_base_path=output_path, + preprocessed_path=preprocessed_path, + backbone=backbone, + num_epochs=num_epochs, + batch_size=batch_size, + learning_rate=learning_rate, + early_stopping_patience=early_stopping_patience, + val_fraction=val_fraction, + ) + + +@pipeline +def inference_pipeline( + model_uri: str, + input_path: str, + prediction_path: str, + output_dir: str, + model_cache_dir: str | None = None, +) -> None: + """RAMP inference: load model (from STAC) → predict → polygonize to GeoJSONs. + + model_uri: From STAC Item assets.model.href. Supports: + - Local path (e.g. /workspace/checkpoints/model.tf) + - Google Drive folder URL + - HTTP URL to .zip containing SavedModel + """ + pred_path = run_inference( + model_uri=model_uri, + input_path=input_path, + prediction_path=prediction_path, + model_cache_dir=model_cache_dir, + ) + run_postprocessing(prediction_path=pred_path, output_dir=output_dir) diff --git a/models/ramp/stac-item.json b/models/ramp/stac-item.json new file mode 100644 index 00000000..bf1e201b --- /dev/null +++ b/models/ramp/stac-item.json @@ -0,0 +1,154 @@ +{ + "type": "Feature", + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/mlm/v1.5.1/schema.json", + "https://stac-extensions.github.io/version/v1.2.0/schema.json", + "https://stac-extensions.github.io/classification/v2.0.0/schema.json", + "https://stac-extensions.github.io/file/v2.1.0/schema.json", + "https://stac-extensions.github.io/raster/v1.1.0/schema.json" + ], + "id": "ramp-v1", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [-180, -90], + [180, -90], + [180, 90], + [-180, 90], + [-180, -90] + ] + ] + }, + "bbox": [-180, -90, 180, 90], + "properties": { + "datetime": "2024-01-01T00:00:00Z", + "mlm:name": "ramp-v1", + "mlm:architecture": "EffUnet", + "mlm:tasks": [ + "semantic-segmentation" + ], + "mlm:framework": "tensorflow", + "mlm:framework_version": "2.9.3", + "mlm:pretrained": false, + "mlm:pretrained_source": null, + "keywords": [ + "building", + "semantic-segmentation", + "multimask", + "efficientnet", + "unet" + ], + "version": "1", + "mlm:input": [ + { + "name": "RGB GeoTIFF chips", + "bands": [ + {"name": "red"}, + {"name": "green"}, + {"name": "blue"} + ], + "input": { + "shape": [-1, 256, 256, 3], + "dim_order": ["batch", "height", "width", "bands"], + "data_type": "float32" + }, + "norm_by_channel": false, + "norm_clip": [0.0, 1.0], + "pre_processing_function": { + "format": "python", + "expression": "models.ramp.pipeline:preprocess" + } + } + ], + "mlm:output": [ + { + "name": "4-class sparse categorical multimask", + "tasks": ["semantic-segmentation"], + "result": { + "shape": [-1, 256, 256, 1], + "dim_order": ["batch", "height", "width", "channel"], + "data_type": "uint8" + }, + "classification:classes": [ + { + "name": "background", + "value": 0, + "description": "Non-building pixels" + }, + { + "name": "building", + "value": 1, + "description": "Building interior pixels" + }, + { + "name": "boundary", + "value": 2, + "description": "Building boundary pixels (helps separate adjacent buildings)" + }, + { + "name": "contact", + "value": 3, + "description": "Close-contact points between neighbouring buildings" + } + ], + "post_processing_function": { + "format": "python", + "expression": "models.ramp.pipeline:postprocess" + } + } + ], + "mlm:hyperparameters": { + "backbone": "efficientnetb0", + "num_classes": 4, + "epochs": 100, + "batch_size": 16, + "learning_rate": 0.0003, + "loss": "sparse_categorical_crossentropy", + "optimizer": "adam", + "input_img_shape": [256, 256], + "output_img_shape": [256, 256], + "boundary_width": 3, + "contact_spacing": 8, + "val_fraction": 0.15, + "early_stopping_patience": 35, + "augmentation": false + } + }, + "assets": { + "model": { + "href": "https://drive.google.com/drive/folders/1VHFPkAWVtI5UEyzbD6e1uZlqe80YtQpn", + "type": "application/octet-stream; framework=tensorflow", + "title": "RAMP EfficientNetB0 U-Net weights (Keras SavedModel)", + "roles": ["mlm:model"], + "mlm:artifact_type": "tf.keras.Model", + "raster:bands": [ + {"name": "red"}, + {"name": "green"}, + {"name": "blue"} + ] + }, + "source-code": { + "href": "https://github.com/hotosm/fAIr-models/tree/main/models/ramp", + "type": "text/html", + "title": "RAMP model pack source", + "roles": ["code"], + "mlm:entrypoint": "models.ramp.pipeline:training_pipeline", + "mlm:inference_entrypoint": "models.ramp.pipeline:inference_pipeline" + }, + "mlm:training": { + "href": "local", + "type": "text/plain", + "title": "Training runtime (Docker)", + "roles": ["mlm:training-runtime", "runtime"] + }, + "mlm:inference": { + "href": "local", + "type": "text/plain", + "title": "Inference runtime (Docker)", + "roles": ["mlm:inference-runtime", "runtime"] + } + }, + "links": [] +} diff --git a/models/ramp/tests/CODE_EXPLAINED.md b/models/ramp/tests/CODE_EXPLAINED.md new file mode 100644 index 00000000..84cfb1b7 --- /dev/null +++ b/models/ramp/tests/CODE_EXPLAINED.md @@ -0,0 +1,465 @@ +# Line-by-Line Explanation of `inside_container_smoke_test.py` + +A beginner-friendly walkthrough of the RAMP smoke test script. + +--- + +## What is `# noqa: F401`? + +**F401** is a **Flake8** (Python linter) rule code. It means: + +- **F** = Pyflakes (the part of Flake8 that checks for logic/import issues) +- **401** = "Module imported but unused" + +So **F401** = "You imported a module but never use any name from it." + +When we write: + +```python +import tensorflow as tf # noqa: F401 +``` + +we are: + +1. **Importing** the `tensorflow` package so Python loads it (and we check it’s installed). +2. **Not using** `tf` later in a way the linter sees (we do use `tf.__version__` and `tf.keras` elsewhere, but on *this* line we only care that the import works). +3. **Silencing the linter** with `# noqa: F401` so it doesn’t report "imported but unused" for this line. + +So: **F401 = "imported but unused"**, and **noqa: F401** means "don’t warn about F401 on this line." + +--- + +## Top of the file (docstring and imports) + +```python +"""End-to-end smoke tests for models/ramp Docker runtime. +... +""" +``` + +- **Triple-quoted string** at the top of a file is the **docstring** for the module. It describes what the script does. Tools and `help()` can show it. + +```python +from __future__ import annotations +``` + +- Makes type hints (like `str`, `Path`) be treated as strings by the interpreter. Helps with forward references and cleaner type hints. + +```python +import argparse +import os +import shutil +from pathlib import Path +``` + +- **argparse**: read command-line arguments (e.g. `--dataset-root`, `--epochs`). +- **os**: environment variables, path checks (e.g. `os.environ`, `os.path`). +- **shutil**: copy/delete trees (e.g. `shutil.rmtree`, `shutil.copy`). +- **pathlib.Path**: object-oriented paths (`Path("a") / "b"` → `a/b`), `.is_dir()`, `.glob()`, etc. + +--- + +## Helper functions + +```python +def _assert(condition: bool, message: str) -> None: + if not condition: + raise RuntimeError(message) +``` + +- **def**: define a function. +- **condition: bool**: argument must be a boolean (True/False). +- **message: str**: argument must be a string. +- **-> None**: function returns nothing. +- If `condition` is False, we raise `RuntimeError(message)` so the test fails with a clear message. The leading `_` means "internal/private" by convention. + +```python +def _stage(name: str) -> None: + print(f"\n=== {name} ===") +``` + +- Prints a section header like `\n=== Test 1: Critical imports ===`. The **f-string** `f"..."` lets you embed `{name}` in the string. + +```python +def _count_files(path: Path, pattern: str) -> int: + return len(list(path.glob(pattern))) +``` + +- **path.glob(pattern)**: finds all files under `path` matching `pattern` (e.g. `"*.png"`). Returns an iterator. +- **list(...)**: turn that iterator into a list. +- **len(...)**: number of items. So this returns how many files match the pattern. + +--- + +## Parsing command-line arguments + +```python +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Run RAMP container smoke tests.") +``` + +- **ArgumentParser**: object that will define and parse CLI arguments. +- **argparse.Namespace**: type of the object you get back (e.g. `args.dataset_root`). + +```python + parser.add_argument( + "--dataset-root", + default="/workspace/data/sample", + help="...", + ) +``` + +- Adds an option `--dataset-root`. If the user doesn’t pass it, `args.dataset_root` is `"/workspace/data/sample"`. +- **help**: text shown when you run `python script.py --help`. + +```python + parser.add_argument("--epochs", type=int, default=2) +``` + +- **type=int**: value is converted to an integer. +- Same idea for **--batch-size** and **--backbone** (string). + +```python + return parser.parse_args() +``` + +- Reads `sys.argv` (the actual command line), fills in the values, and returns an object with attributes like `args.dataset_root`, `args.epochs`, etc. + +--- + +## Preparing data/sample layout + +```python +def _prepare_data_sample_layout(dataset_root: Path) -> Path: +``` + +- Takes a path to the dataset root, returns the path to the "work" root (where we’ll put prepared input). + +```python + oam_dir = dataset_root / "train" / "oam" + osm_dir = dataset_root / "train" / "osm" +``` + +- **Path / "train" / "oam"**: builds path like `dataset_root/train/oam`. `/` on `Path` joins path parts. + +```python + _assert(oam_dir.is_dir(), f"train/oam not found under {dataset_root}") +``` + +- Fails fast if `train/oam` (or `train/osm`) doesn’t exist. + +```python + work_root = dataset_root / "ramp_work" + input_dir = work_root / "input" + input_dir.mkdir(parents=True, exist_ok=True) +``` + +- **mkdir(parents=True, exist_ok=True)**: create the directory and any missing parents; don’t error if it already exists. + +```python + import numpy as np + import rasterio + from PIL import Image +``` + +- Imports used only in this function. **numpy**: arrays/math. **rasterio**: read/write rasters (e.g. GeoTIFF). **PIL.Image**: image I/O (e.g. save PNG). + +```python + tif_files = sorted(oam_dir.glob("OAM-*.tif")) +``` + +- **glob("OAM-*.tif")**: all files whose names match that pattern. **sorted()**: stable order for reproducibility. + +```python + for tif_path in tif_files: + png_path = input_dir / (tif_path.stem + ".png") +``` + +- **tif_path.stem**: filename without extension (e.g. `OAM-386695-220244-19`). So we get a path like `input_dir/OAM-386695-220244-19.png`. + +```python + with rasterio.open(tif_path) as src: + data = src.read() +``` + +- **with ... as src**: open the file and automatically close it when the block ends. **src.read()**: read all bands as a 3D array (bands, height, width). + +```python + if data.shape[0] >= 3: + rgb = np.transpose(data[:3], (1, 2, 0)) +``` + +- **data.shape[0]**: number of bands. We need at least 3 (R, G, B). +- **data[:3]**: first 3 bands. **np.transpose(..., (1, 2, 0))**: change layout from (bands, H, W) to (H, W, bands) for image-style arrays. + +```python + if rgb.max() <= 1.0: + rgb = (rgb * 255).astype(np.uint8) + Image.fromarray(rgb).save(png_path) +``` + +- If values are in 0–1, scale to 0–255 and convert to 8-bit integer. **Image.fromarray(rgb).save(png_path)**: write a PNG file. + +```python + geojson_files = sorted(osm_dir.glob("*.geojson")) + ... + gdfs = [gpd.read_file(p) for p in geojson_files] +``` + +- **gpd.read_file(p)**: read a GeoJSON into a GeoDataFrame. The list comprehension builds a list of one GeoDataFrame per file. + +```python + if len(gdfs) == 1: + merged = gdfs[0] + else: + import pandas as pd + crs = gdfs[0].crs or "EPSG:4326" + merged = gpd.GeoDataFrame( + pd.concat([g.to_crs(crs) for g in gdfs], ignore_index=True), + crs=crs, + ) +``` + +- One file → use it as-is. Multiple → convert all to the same CRS (**crs**), concatenate with **pd.concat**, wrap in **GeoDataFrame**, and assign **crs**. + +```python + labels_path = input_dir / "labels.geojson" + merged.to_file(labels_path, driver="GeoJSON") +``` + +- Write the (possibly merged) labels to **input/labels.geojson** so the rest of the pipeline sees a single labels file. + +```python + return work_root +``` + +- Return the path to the directory that now contains **input/** with PNGs and **labels.geojson**. + +--- + +## main() — entry point + +```python +def main() -> None: + args = parse_args() +``` + +- Parse CLI into **args**. + +```python + os.environ.setdefault("RAMP_HOME", "/workspace") +``` + +- Set **RAMP_HOME** only if it’s not already set. Some RAMP code uses this. + +```python + dataset_root = Path(args.dataset_root).resolve() +``` + +- **Path(...)**: turn the string into a Path. **.resolve()**: make it absolute and normalize (e.g. resolve `..`). + +```python + if (dataset_root / "train" / "oam").is_dir() and (dataset_root / "train" / "osm").is_dir(): + dataset_root = _prepare_data_sample_layout(dataset_root) +``` + +- If we see the **data/sample** layout (train/oam and train/osm), we prepare it and then **dataset_root** becomes the work root (**ramp_work**). + +```python + input_dir = dataset_root / "input" + preprocessed_dir = dataset_root / "preprocessed_test" + chips_dir = preprocessed_dir / "chips" + ... +``` + +- Define all the paths we’ll use for inputs and outputs (chips, masks, val split, checkpoints, prediction output, vectors). No files are created yet; these are just path variables. + +--- + +## Test 1: Critical imports + +```python + import segmentation_models as sm # noqa: F401 + import tensorflow as tf # noqa: F401 + from osgeo import gdal # noqa: F401 + import ramp # noqa: F401 + import hot_fair_utilities # noqa: F401 +``` + +- We import these **to check they are installed and loadable**. If any import fails, the script crashes and the test fails. We use **noqa: F401** because the linter would otherwise say "imported but unused" on some of these lines (we do use `sm`, `tf`, etc. later; the comment keeps the linter happy where it still complains). + +```python + sm.set_framework("tf.keras") +``` + +- Tell **segmentation_models** to use TensorFlow/Keras as the backend. + +```python + print(f"PASS: tensorflow {tf.__version__} ...") +``` + +- **tf.__version__**: the TensorFlow version string. Printing it confirms the import worked and helps with debugging. + +--- + +## Test 2: Dataset layout + +```python + _assert(dataset_root.is_dir(), ...) + _assert(input_dir.is_dir(), ...) + _assert((input_dir / "labels.geojson").is_file(), ...) +``` + +- Ensure the expected folders and **labels.geojson** exist; otherwise raise with a clear message. + +```python + n_png = _count_files(input_dir, "*.png") + _assert(n_png > 0, ...) +``` + +- Count PNGs in **input/** and require at least one. + +--- + +## Test 3: Preprocessing + +```python + shutil.rmtree(preprocessed_dir, ignore_errors=True) +``` + +- Remove any previous preprocessed output so we start clean. **ignore_errors=True**: don’t fail if the directory doesn’t exist. + +```python + from hot_fair_utilities import preprocess as _preprocess + _preprocess( + input_path=str(input_dir), + output_path=str(preprocessed_dir), + ... + ) +``` + +- **import ... as _preprocess**: we only call the function; the leading `_` suggests "used locally." +- **str(...)**: some APIs expect strings, not **Path**; we convert so it works everywhere. + +```python + n_chips = _count_files(chips_dir, "*.tif") + n_masks = _count_files(masks_dir, "*.mask.tif") + _assert(n_chips == n_masks, ...) +``` + +- After preprocessing we expect one chip and one mask per image; we check that the counts match. + +--- + +## Test 4: Train/val split + +```python + chip_files = sorted(chips_dir.glob("*.tif")) + n_val = max(1, int(len(chip_files) * 0.2)) +``` + +- **n_val**: 20% of chips for validation, but at least 1. + +```python + random.shuffle(chip_files) + val_chip_files = chip_files[:n_val] +``` + +- Shuffle and take the first **n_val** as validation; the rest stay as training. + +```python + for chip_path in val_chip_files: + mask_path = masks_dir / (chip_path.stem + ".mask.tif") + if mask_path.is_file(): + shutil.copy(str(chip_path), val_chips_dir / chip_path.name) + shutil.copy(str(mask_path), val_masks_dir / mask_path.name) + moved += 1 +``` + +- For each validation chip, find the matching mask by name (e.g. `chip.tif` → `chip.mask.tif`). If it exists, copy both chip and mask into the val directories and count. + +--- + +## Test 5: Training + +- **cfg**: a big dictionary that RAMP uses for training (epochs, batch size, loss, optimizer, callbacks, etc.). This is the "config" for the run. +- **loss_fn**, **optimizer**, **acc_metric**: built from RAMP’s constructors using **cfg**. +- **sm.Unet(...)**: build the U-Net model. **encoder_weights=None** avoids downloading pretrained weights (which can 404 in CI). +- **the_model.compile(...)**: attach optimizer, loss, and metric for training. +- **training_batches_from_gtiff_dirs** / **test_batches_from_gtiff_dirs**: RAMP functions that yield batches from the chip and mask directories. +- **the_model.fit(...)**: run training for **args.epochs** (e.g. 2) with the given batches and callbacks. +- **the_model.save(model_save_path)**: save the trained model so we can load it in the next stage. + +--- + +## Test 6: Inference + +```python + inference_model = tf.keras.models.load_model(model_save_path, compile=False) +``` + +- Load the SavedModel we just saved. **compile=False**: we don’t need the training graph, only forward pass. + +```python + for chip_file in chip_files[:3]: +``` + +- Run prediction on only the first 3 chips to keep the smoke test fast. + +```python + with rio.open(chip_file) as src: + dst_profile = src.profile.copy() + ... + img = to_channels_last(src.read()).astype("float32") +``` + +- **src.profile**: metadata (size, dtype, etc.) we reuse for the output. **to_channels_last**: convert from (C, H, W) to (H, W, C) if needed for the model. + +```python + predicted = get_mask_from_prediction(inference_model.predict(np.expand_dims(img, 0))) +``` + +- **np.expand_dims(img, 0)**: add a batch dimension (1, H, W, C). **predict(...)**: run the model. **get_mask_from_prediction**: turn model output into a class mask. Then we write the result as **.pred.tif**. + +--- + +## Test 7: Polygonization + +```python + ref_ds = gdal.Open(str(pred_tif)) + multimask = gdal_get_mask_tensor(str(pred_tif)) + bin_mask = binary_mask_from_multichannel_mask(multimask) + binary_mask_to_geojson(bin_mask, ref_ds, json_path) +``` + +- **gdal.Open**: open the predicted GeoTIFF (for georeference info). +- **gdal_get_mask_tensor**: read the mask array. +- **binary_mask_from_multichannel_mask**: reduce multi-class mask to a single binary mask (e.g. building vs non-building). +- **binary_mask_to_geojson**: raster-to-vector (polygonize) and write building footprints to GeoJSON. + +--- + +## End of script + +```python +if __name__ == "__main__": + main() +``` + +- **__name__**: special variable. When you run this file as a script, **__name__** is **"__main__"**. When the file is imported, **__name__** is the module name. +- So **main()** runs only when you execute the script (e.g. `python inside_container_smoke_test.py`), not when another file does `import inside_container_smoke_test`. + +--- + +## Quick reference + +| Concept | Meaning | +|--------|--------| +| **F401** | Flake8: "module imported but unused" | +| **noqa: F401** | "Don’t report F401 on this line" | +| **Path / "a" / "b"** | Path joining: `path/a/b` | +| **path.glob("*.png")** | All files matching pattern under **path** | +| **with open(...) as f:** | Use **f** and close the resource when the block ends | +| **f"text {var}"** | f-string: insert **var** into the string | +| **-> None** | This function returns nothing | +| **type=int** | Convert CLI argument to int | diff --git a/models/ramp/tests/README.md b/models/ramp/tests/README.md new file mode 100644 index 00000000..d26b08a1 --- /dev/null +++ b/models/ramp/tests/README.md @@ -0,0 +1,137 @@ +# RAMP Smoke Tests — In-Depth Guide + +This folder contains end-to-end smoke tests that validate the RAMP Docker runtime. +Tests run **inside** the container after the repo is mounted at `/workspace`. + +## Test Files Overview + +| File | Purpose | +| --- | --- | +| `inside_container_smoke_test.py` | Main test script. Run inside the container. Validates 7 stages (imports → preprocessing → train → inference → polygonization). | +| `run_docker_tests.ps1` | PowerShell runner: builds image (optional), runs container, executes the smoke test. | +| `run_docker_tests.sh` | Bash runner: same as above for Linux/macOS. | +| `test_plan.yaml` | Declarative description of what each stage checks. Used for documentation and CI planning. | + +## Data Source: `data/sample` + +Tests use **`data/sample`** (fAIr-models shared sample) instead of model-specific `ramp-data`. + +**Layout:** + +```text +data/sample/ +├── train/oam/ # OAM GeoTIFF tiles (OAM-{x}-{y}-{z}.tif) +├── train/osm/ # OSM building labels (*.geojson) +└── predict/oam/ # (Optional) images for inference +``` + +**Adapter logic:** `hot_fair_utilities.preprocess` expects `input/*.png` and `input/labels.geojson`. +The test script detects the `data/sample` layout and: + +1. Creates `data/sample/ramp_work/input/` +2. Converts `train/oam/*.tif` → PNG (rasterio + PIL) +3. Merges `train/osm/*.geojson` → `labels.geojson` +4. Runs the pipeline on `ramp_work/input/` + +Outputs go to `data/sample/ramp_work/preprocessed_test/` and `prediction_test/`. + +## Seven Test Stages (inside_container_smoke_test.py) + +### 1. Critical Imports + +Verifies runtime packages: `tensorflow`, `segmentation_models`, `ramp`, `hot_fair_utilities`, `osgeo.gdal`, `solaris`. +Sets `segmentation_models` to use `tf.keras` backend. + +### 2. Input Dataset Layout + +- **data/sample**: Requires `train/oam/`, `train/osm/`, at least one `OAM-*.tif`, and one `*.geojson`. +- **Legacy**: Requires `input/`, `input/labels.geojson`, and `input/*.png`. + +### 3. Preprocessing + +Calls `hot_fair_utilities.preprocess()` with: + +- `georeference_images=True` — converts PNG to GeoTIFF (EPSG:3857) +- `rasterize=True`, `rasterize_options=["binary"]` +- `multimasks=True` — 4-class masks (background, building, boundary, contact) + +Checks: `chips/*.tif` and `multimasks/*.mask.tif` exist and counts match. + +### 4. Train / Val Split + +Shuffles chip/mask pairs, moves ~20% to `val_chips/` and `val_multimasks/`. +RAMP’s data generator needs separate train and validation directories. + +### 5. Training Smoke Run + +Builds EfficientNetB0 U-Net (`encoder_weights=None` to avoid 404 on pretrained weights). +Runs 2 epochs, saves SavedModel to `checkpoints/`. + +### 6. Inference + +Loads SavedModel, runs prediction on first 3 chips. +Produces `*.pred.tif` (4-class uint8 masks). + +### 7. Polygonization + +Uses `ramp.utils.mask_to_vec_utils` (GDAL Polygonize) to convert predicted masks to GeoJSON building footprints. +Checks for `*.geojson` in `prediction_test/vectors/`. + +## How to Run + +**Prerequisite:** Ensure `data/sample` exists with `train/oam/` and `train/osm/`. + +### PowerShell (Windows) + +```powershell +cd fAIr-models # repo root + +# Build image and run tests +.\models\ramp\tests\run_docker_tests.ps1 -BuildImage + +# Or run tests only (image must already exist) +.\models\ramp\tests\run_docker_tests.ps1 + +# CPU-only (no GPU) +.\models\ramp\tests\run_docker_tests.ps1 -BuildImage -CpuOnly +``` + +### Bash (Linux / macOS) + +```bash +cd fAIr-models + +# Build and run +BUILD_IMAGE=1 ./models/ramp/tests/run_docker_tests.sh + +# CPU-only +CPU_ONLY=1 BUILD_IMAGE=1 ./models/ramp/tests/run_docker_tests.sh +``` + +### Manually Inside Container + +```bash +docker run --rm -v /path/to/fAIr-models:/workspace ramp-v1:gpu \ + python /workspace/models/ramp/tests/inside_container_smoke_test.py \ + --dataset-root /workspace/data/sample +``` + +### With Custom Dataset (Legacy Layout) + +```bash +# Legacy layout: dataset/input/*.png and dataset/input/labels.geojson +docker run --rm -v /path/to/data:/workspace/data ramp-v1:gpu \ + python /workspace/models/ramp/tests/inside_container_smoke_test.py \ + --dataset-root /workspace/data/your_dataset +``` + +## What Gets Created + +Running tests creates: + +- `data/sample/ramp_work/input/` — prepared PNG chips + labels.geojson (data/sample only) +- `data/sample/ramp_work/preprocessed_test/` — chips, multimasks, val split, checkpoints +- `data/sample/ramp_work/prediction_test/output/` — `*.pred.tif` +- `data/sample/ramp_work/prediction_test/vectors/` — `*.geojson` building polygons + +These paths are in `.gitignore`. diff --git a/models/ramp/tests/inside_container_smoke_test.py b/models/ramp/tests/inside_container_smoke_test.py new file mode 100644 index 00000000..d4270f57 --- /dev/null +++ b/models/ramp/tests/inside_container_smoke_test.py @@ -0,0 +1,403 @@ +"""End-to-end smoke tests for models/ramp Docker runtime. + +Run this script INSIDE the container. It validates: +1) Critical imports (tensorflow, ramp, segmentation_models, osgeo.gdal, solaris) +2) hot-fair-utilities preprocessing (georeference + multimask generation) +3) Train/val split +4) Short training run (2 epochs, checkpoint creation) +5) Inference output generation (.pred.tif per chip) +6) Polygonization (per-chip GeoJSON from predicted masks) + +Data layouts supported: + - data/sample layout: --dataset-root /workspace/data/sample + Uses train/oam/*.tif + train/osm/*.geojson. Converts TIF→PNG and merges + OSM labels into a temporary input directory (hot_fair_utilities expects PNG). + - Legacy RAMP layout: --dataset-root /path/to/dataset + Expects dataset/input/*.png and dataset/input/labels.geojson. + +Usage inside container: + python /workspace/models/ramp/tests/inside_container_smoke_test.py \\ + --dataset-root /workspace/data/sample +""" + +from __future__ import annotations + +import argparse +import os +import shutil +from pathlib import Path + + +def _assert(condition: bool, message: str) -> None: + if not condition: + raise RuntimeError(message) + + +def _stage(name: str) -> None: + print(f"\n=== {name} ===") + + +def _count_files(path: Path, pattern: str) -> int: + return len(list(path.glob(pattern))) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Run RAMP container smoke tests.") + parser.add_argument( + "--dataset-root", + default="/workspace/data/sample", + help="Dataset root: data/sample (train/oam + train/osm) or legacy (input/ with PNG + labels.geojson).", + ) + parser.add_argument("--epochs", type=int, default=2) + parser.add_argument("--batch-size", type=int, default=4) + parser.add_argument("--backbone", default="efficientnetb0") + return parser.parse_args() + + +def _prepare_data_sample_layout(dataset_root: Path) -> Path: + """Convert data/sample (train/oam + train/osm) to RAMP input layout. + + hot_fair_utilities.preprocess expects input_path with *.png and labels.geojson. + data/sample has train/oam/*.tif and train/osm/*.geojson. + Creates dataset_root/ramp_work/input/ with PNG chips and labels.geojson. + Returns the working root (ramp_work) for the test. + """ + oam_dir = dataset_root / "train" / "oam" + osm_dir = dataset_root / "train" / "osm" + _assert(oam_dir.is_dir(), f"train/oam not found under {dataset_root}") + _assert(osm_dir.is_dir(), f"train/osm not found under {dataset_root}") + + work_root = dataset_root / "ramp_work" + input_dir = work_root / "input" + input_dir.mkdir(parents=True, exist_ok=True) + + import numpy as np + import rasterio + from PIL import Image + + tif_files = sorted(oam_dir.glob("OAM-*.tif")) + _assert(len(tif_files) > 0, f"No OAM-*.tif files in {oam_dir}") + + for tif_path in tif_files: + png_path = input_dir / (tif_path.stem + ".png") + with rasterio.open(tif_path) as src: + data = src.read() + if data.shape[0] >= 3: + rgb = np.transpose(data[:3], (1, 2, 0)) + if rgb.max() <= 1.0: + rgb = (rgb * 255).astype(np.uint8) + Image.fromarray(rgb).save(png_path) + + geojson_files = sorted(osm_dir.glob("*.geojson")) + _assert(len(geojson_files) > 0, f"No .geojson files in {osm_dir}") + + import geopandas as gpd + + gdfs = [gpd.read_file(p) for p in geojson_files] + if len(gdfs) == 1: + merged = gdfs[0] + else: + import pandas as pd + + crs = gdfs[0].crs or "EPSG:4326" + merged = gpd.GeoDataFrame( + pd.concat([g.to_crs(crs) for g in gdfs], ignore_index=True), + crs=crs, + ) + labels_path = input_dir / "labels.geojson" + merged.to_file(labels_path, driver="GeoJSON") + print(f"PASS: prepared {len(tif_files)} PNG chip(s) + labels.geojson from data/sample") + + return work_root + + +def main() -> None: + args = parse_args() + + os.environ.setdefault("RAMP_HOME", "/workspace") + + dataset_root = Path(args.dataset_root).resolve() + + # Support data/sample layout (train/oam + train/osm) or legacy (input/ with PNG + labels) + if (dataset_root / "train" / "oam").is_dir() and (dataset_root / "train" / "osm").is_dir(): + dataset_root = _prepare_data_sample_layout(dataset_root) + + input_dir = dataset_root / "input" + preprocessed_dir = dataset_root / "preprocessed_test" + chips_dir = preprocessed_dir / "chips" + masks_dir = preprocessed_dir / "multimasks" + val_chips_dir = preprocessed_dir / "val_chips" + val_masks_dir = preprocessed_dir / "val_multimasks" + checkpts_dir = preprocessed_dir / "checkpoints" + pred_input_dir = preprocessed_dir / "chips" + pred_output_dir = dataset_root / "prediction_test" / "output" + vectors_dir = dataset_root / "prediction_test" / "vectors" + + # ------------------------------------------------------------------------- + _stage("Test 1: Critical imports") + + import segmentation_models as sm # noqa: F401 + import tensorflow as tf # noqa: F401 + from osgeo import gdal # noqa: F401 + import ramp # noqa: F401 + import hot_fair_utilities # noqa: F401 + + sm.set_framework("tf.keras") + print(f"PASS: tensorflow {tf.__version__} / segmentation_models / ramp / gdal imported") + + import solaris # noqa: F401 + print(f"PASS: solaris {solaris.__version__} imported") + + # ------------------------------------------------------------------------- + _stage("Test 2: Input dataset layout checks") + + _assert(dataset_root.is_dir(), f"Dataset root not found: {dataset_root}") + _assert(input_dir.is_dir(), f"Input folder not found: {input_dir}") + _assert( + (input_dir / "labels.geojson").is_file(), + f"labels.geojson not found in {input_dir}", + ) + n_png = _count_files(input_dir, "*.png") + _assert(n_png > 0, f"No PNG chips found in {input_dir}. Add OAM PNG chips to run this test.") + print(f"PASS: found {n_png} PNG chip(s) + labels.geojson") + + # ------------------------------------------------------------------------- + _stage("Test 3: Preprocessing (georeference + multimask generation)") + + shutil.rmtree(preprocessed_dir, ignore_errors=True) + + from hot_fair_utilities import preprocess as _preprocess + + _preprocess( + input_path=str(input_dir), + output_path=str(preprocessed_dir), + rasterize=True, + rasterize_options=["binary"], + georeference_images=True, + multimasks=True, + input_boundary_width=3, + input_contact_spacing=8, + ) + + n_chips = _count_files(chips_dir, "*.tif") + n_masks = _count_files(masks_dir, "*.mask.tif") + _assert(n_chips > 0, f"No chips produced in {chips_dir}") + _assert(n_masks > 0, f"No multimasks produced in {masks_dir}") + _assert(n_chips == n_masks, f"Chip count ({n_chips}) != mask count ({n_masks})") + print(f"PASS: preprocessing produced {n_chips} chip(s) + {n_masks} multimask(s)") + + # ------------------------------------------------------------------------- + _stage("Test 4: Train / val split") + + import random + + chip_files = sorted(chips_dir.glob("*.tif")) + n_val = max(1, int(len(chip_files) * 0.2)) + random.shuffle(chip_files) + val_chip_files = chip_files[:n_val] + + val_chips_dir.mkdir(parents=True, exist_ok=True) + val_masks_dir.mkdir(parents=True, exist_ok=True) + + moved = 0 + for chip_path in val_chip_files: + mask_path = masks_dir / (chip_path.stem + ".mask.tif") + if mask_path.is_file(): + shutil.copy(str(chip_path), val_chips_dir / chip_path.name) + shutil.copy(str(mask_path), val_masks_dir / mask_path.name) + moved += 1 + + _assert(moved > 0, "Val split produced 0 pairs — check chip/mask naming") + print(f"PASS: val split created {moved} chip+mask pair(s)") + + # ------------------------------------------------------------------------- + _stage("Test 5: Training smoke run (2 epochs)") + + checkpts_dir.mkdir(parents=True, exist_ok=True) + + import datetime + + import segmentation_models as sm + + sm.set_framework("tf.keras") + + from ramp.data_mgmt.data_generator import ( + test_batches_from_gtiff_dirs, + training_batches_from_gtiff_dirs, + ) + from ramp.training import ( + callback_constructors, + loss_constructors, + metric_constructors, + optimizer_constructors, + ) + + timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + cfg = { + "experiment_name": "smoke_test", + "num_classes": 4, + "num_epochs": args.epochs, + "batch_size": args.batch_size, + "input_img_shape": [256, 256], + "output_img_shape": [256, 256], + "loss": {"get_loss_fn_name": "get_sparse_categorical_crossentropy_fn", "loss_fn_parms": {}}, + "metrics": { + "use_metrics": True, + "get_metrics_fn_names": ["get_sparse_categorical_accuracy_fn"], + "metrics_fn_parms": [{}], + }, + "optimizer": { + "get_optimizer_fn_name": "get_adam_optimizer", + "optimizer_fn_parms": {"learning_rate": 3e-4}, + }, + "model": { + "get_model_fn_name": "get_effunet_model", + "model_fn_parms": { + "backbone": args.backbone, + "classes": ["background", "building", "boundary", "contact"], + }, + }, + "saved_model": {"use_saved_model": False}, + "augmentation": {"use_aug": False}, + "early_stopping": { + "use_early_stopping": True, + "early_stopping_parms": { + "monitor": "val_loss", + "min_delta": 0.005, + "patience": 2, + "verbose": 0, + "mode": "auto", + "restore_best_weights": True, + }, + }, + "cyclic_learning_scheduler": {"use_clr": False}, + "tensorboard": {"use_tb": False}, + "prediction_logging": {"use_prediction_logging": False}, + "model_checkpts": { + "use_model_checkpts": True, + "model_checkpts_dir": str(checkpts_dir), + "get_model_checkpt_callback_fn_name": "get_model_checkpt_callback_fn", + "model_checkpt_callback_parms": {"mode": "max", "save_best_only": True}, + }, + "random_seed": 42, + "timestamp": timestamp, + } + + loss_fn = loss_constructors.get_sparse_categorical_crossentropy_fn(cfg) + optimizer = optimizer_constructors.get_adam_optimizer(cfg) + acc_metric = metric_constructors.get_sparse_categorical_accuracy_fn({}) + # NOTE: For the container smoke test we intentionally disable any pretrained + # EfficientNet weight downloads (the old Keras Applications URL can 404). + # This keeps the test self-contained while still validating the end-to-end + # training + checkpoint + inference + polygonization flow. + the_model = sm.Unet( + backbone_name=args.backbone, + encoder_weights=None, + classes=4, + activation="softmax", + ) + the_model.compile(optimizer=optimizer, loss=loss_fn, metrics=[acc_metric]) + + n_train = _count_files(chips_dir, "*.tif") + n_val_c = _count_files(val_chips_dir, "*.tif") + steps_per_epoch = max(1, n_train // args.batch_size) + validation_steps = max(1, n_val_c // args.batch_size) + cfg["runtime"] = { + "n_training": n_train, + "n_val": n_val_c, + "steps_per_epoch": steps_per_epoch, + "validation_steps": validation_steps, + } + + train_batches = training_batches_from_gtiff_dirs( + chips_dir, masks_dir, args.batch_size, [256, 256], [256, 256] + ) + val_batches = test_batches_from_gtiff_dirs( + val_chips_dir, val_masks_dir, args.batch_size, [256, 256], [256, 256] + ) + + callbacks = [callback_constructors.get_early_stopping_callback_fn(cfg)] + + the_model.fit( + train_batches, + epochs=args.epochs, + steps_per_epoch=steps_per_epoch, + validation_data=val_batches, + validation_steps=validation_steps, + callbacks=callbacks, + ) + + model_save_path = str(checkpts_dir / f"smoke_{timestamp}.tf") + the_model.save(model_save_path) + _assert(Path(model_save_path).is_dir(), f"Saved model not found at {model_save_path}") + print(f"PASS: 2-epoch training completed; model saved to {model_save_path}") + + # ------------------------------------------------------------------------- + _stage("Test 6: Inference smoke run") + + import numpy as np + import rasterio as rio + + from ramp.data_mgmt.display_data import get_mask_from_prediction + from ramp.utils.file_utils import get_basename + from ramp.utils.img_utils import to_channels_first, to_channels_last + + pred_output_dir.mkdir(parents=True, exist_ok=True) + shutil.rmtree(pred_output_dir, ignore_errors=True) + pred_output_dir.mkdir(parents=True, exist_ok=True) + + inference_model = tf.keras.models.load_model(model_save_path, compile=False) + + chip_files = sorted(pred_input_dir.glob("*.tif")) + for chip_file in chip_files[:3]: + bname = get_basename(str(chip_file)) + mask_name = bname + ".pred.tif" + with rio.open(chip_file) as src: + dst_profile = src.profile.copy() + dst_profile["count"] = 1 + dst_profile["dtype"] = "uint8" + img = to_channels_last(src.read()).astype("float32") + max_val = float(img.max()) + if max_val > 0: + img = img / max_val + predicted = get_mask_from_prediction(inference_model.predict(np.expand_dims(img, 0))) + predicted = np.squeeze(predicted, axis=0) + with rio.open(pred_output_dir / mask_name, "w", **dst_profile) as dst: + dst.write(to_channels_first(predicted)) + + n_pred = _count_files(pred_output_dir, "*.pred.tif") + _assert(n_pred > 0, f"No .pred.tif files produced in {pred_output_dir}") + print(f"PASS: inference produced {n_pred} .pred.tif file(s)") + + # ------------------------------------------------------------------------- + _stage("Test 7: Polygonization") + + from osgeo import gdal + + from ramp.utils.img_utils import gdal_get_mask_tensor + from ramp.utils.mask_to_vec_utils import ( + binary_mask_from_multichannel_mask, + binary_mask_to_geojson, + ) + + vectors_dir.mkdir(parents=True, exist_ok=True) + + for pred_tif in sorted(pred_output_dir.glob("*.pred.tif")): + json_name = pred_tif.stem.replace(".pred", "") + ".geojson" + json_path = str(vectors_dir / json_name) + ref_ds = gdal.Open(str(pred_tif)) + _assert(ref_ds is not None, f"GDAL could not open {pred_tif}") + multimask = gdal_get_mask_tensor(str(pred_tif)) + bin_mask = binary_mask_from_multichannel_mask(multimask) + binary_mask_to_geojson(bin_mask, ref_ds, json_path) + + n_geojson = _count_files(vectors_dir, "*.geojson") + _assert(n_geojson > 0, f"No GeoJSON files produced in {vectors_dir}") + print(f"PASS: polygonization produced {n_geojson} GeoJSON file(s)") + + # ------------------------------------------------------------------------- + _stage("ALL TESTS PASSED") + + +if __name__ == "__main__": + main() diff --git a/models/ramp/tests/run_docker_tests.ps1 b/models/ramp/tests/run_docker_tests.ps1 new file mode 100644 index 00000000..25189917 --- /dev/null +++ b/models/ramp/tests/run_docker_tests.ps1 @@ -0,0 +1,34 @@ +param( + [string]$RepoRoot = "E:\On Going Projects\Work Growth\HOTOSM\fAIr_Repos\fAIr-models", + [string]$Image = "ramp-v1:gpu", + [switch]$BuildImage, + [switch]$CpuOnly +) + +$ErrorActionPreference = "Stop" + +if ($BuildImage) { + $buildArg = "" + if ($CpuOnly) { + $buildArg = "--build-arg BUILD_TYPE=cpu" + $Image = "ramp-v1:cpu" + } else { + $buildArg = "--build-arg BUILD_TYPE=gpu" + } + Write-Host "Building image $Image ..." + Invoke-Expression "docker build $buildArg -t $Image -f `"$RepoRoot\models\ramp\Dockerfile`" `"$RepoRoot`"" +} + +$gpuArgs = @() +if (-not $CpuOnly) { + $gpuArgs = @("--gpus", "all") +} + +Write-Host "Running container smoke tests..." +docker run --rm @gpuArgs ` + -v "${RepoRoot}:/workspace" ` + $Image ` + python /workspace/models/ramp/tests/inside_container_smoke_test.py ` + --dataset-root /workspace/data/sample + +Write-Host "Done." diff --git a/models/ramp/tests/run_docker_tests.sh b/models/ramp/tests/run_docker_tests.sh new file mode 100644 index 00000000..b08498a9 --- /dev/null +++ b/models/ramp/tests/run_docker_tests.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO_ROOT="${1:-$(pwd)}" +IMAGE="${2:-ramp-v1:gpu}" +BUILD_IMAGE="${BUILD_IMAGE:-0}" +CPU_ONLY="${CPU_ONLY:-0}" + +if [[ "$BUILD_IMAGE" == "1" ]]; then + BUILD_ARGS="" + if [[ "$CPU_ONLY" == "1" ]]; then + BUILD_ARGS="--build-arg BUILD_TYPE=cpu" + IMAGE="ramp-v1:cpu" + else + BUILD_ARGS="--build-arg BUILD_TYPE=gpu" + fi + echo "Building image $IMAGE ..." + docker build $BUILD_ARGS -t "$IMAGE" -f "$REPO_ROOT/models/ramp/Dockerfile" "$REPO_ROOT" +fi + +GPU_ARGS=() +if [[ "$CPU_ONLY" != "1" ]]; then + GPU_ARGS=(--gpus all) +fi + +echo "Running container smoke tests..." +docker run --rm "${GPU_ARGS[@]}" \ + -v "$REPO_ROOT:/workspace" \ + "$IMAGE" \ + python /workspace/models/ramp/tests/inside_container_smoke_test.py \ + --dataset-root /workspace/data/sample + +echo "Done." diff --git a/models/ramp/tests/test_plan.yaml b/models/ramp/tests/test_plan.yaml new file mode 100644 index 00000000..b7bea389 --- /dev/null +++ b/models/ramp/tests/test_plan.yaml @@ -0,0 +1,58 @@ +# RAMP smoke tests use data/sample (train/oam + train/osm). +# The test auto-prepares input from data/sample layout to RAMP format (PNG + labels.geojson). + +tests: + - id: import_smoke + description: Verify all critical runtime packages import correctly inside container + checks: + - import tensorflow + - import segmentation_models (set_framework tf.keras) + - import ramp + - import hot_fair_utilities + - import osgeo.gdal + - import solaris + + - id: dataset_layout + description: Verify required input folder and files are present (or data/sample train/oam + train/osm) + required_paths: + - "train/oam (data/sample) or input/" + - "train/osm (data/sample) or input/labels.geojson" + required_patterns: + - "train/oam/OAM-*.tif (data/sample) or input/*.png" + + - id: preprocess + description: Run hot_fair_utilities.preprocess with multimasks=True; validate output structure + expected_outputs: + - preprocessed_test/chips/*.tif + - preprocessed_test/multimasks/*.mask.tif + invariants: + - chip_count == mask_count + + - id: val_split + description: Shuffle and move a fraction of chip+mask pairs to val directories + expected_outputs: + - preprocessed_test/val_chips/*.tif + - preprocessed_test/val_multimasks/*.mask.tif + invariants: + - at_least_one_pair_moved + + - id: train_smoke + description: > + Compile EfficientNetB0 U-Net, run 2-epoch training loop, save Keras SavedModel. + Validates that training does not crash and a SavedModel directory is created. + expected_outputs: + - preprocessed_test/checkpoints/smoke_.tf/ + + - id: inference_smoke + description: > + Load the SavedModel checkpoint and run chip-by-chip inference. + Each chip produces a single-band uint8 .pred.tif with 4-class values. + expected_outputs: + - prediction_test/output/*.pred.tif + + - id: polygonize_smoke + description: > + Use GDAL Polygonize via ramp.utils.mask_to_vec_utils to convert + predicted multimasks into per-chip GeoJSON building polygons. + expected_outputs: + - prediction_test/vectors/*.geojson From 65251b17742692a213eec4b51eac189190328446 Mon Sep 17 00:00:00 2001 From: Abdelrahman Katkat Date: Mon, 2 Mar 2026 10:26:57 +0200 Subject: [PATCH 02/30] feat(ramp): enhance pipeline with STAC integration and hyperparameter loading - Updated pipeline.py to load hyperparameters from STAC Item JSON, streamlining model configuration. - Modified training_pipeline to accept a path to the STAC Item, allowing for flexible hyperparameter management. - Revised README.md to reflect changes in hyperparameter handling and usage of STAC Item. - Enhanced stac-item.json with additional metadata and structure for better integration. - Removed outdated CODE_EXPLAINED.md and README.md from tests directory to clean up documentation. --- models/ramp/README.md | 9 +- models/ramp/pipeline.py | 126 +++++--- models/ramp/stac-item.json | 79 ++++- models/ramp/tests/CODE_EXPLAINED.md | 465 ---------------------------- models/ramp/tests/README.md | 137 -------- 5 files changed, 144 insertions(+), 672 deletions(-) delete mode 100644 models/ramp/tests/CODE_EXPLAINED.md delete mode 100644 models/ramp/tests/README.md diff --git a/models/ramp/README.md b/models/ramp/README.md index 95cd58c6..74c12a95 100644 --- a/models/ramp/README.md +++ b/models/ramp/README.md @@ -89,16 +89,13 @@ prediction/vectors/*.geojson (per-chip building footprints) from models.ramp.pipeline import training_pipeline, inference_pipeline # Full training run (use your dataset path) +# Hyperparameters are loaded from models/ramp/stac-item.json training_pipeline( input_path="data/sample/ramp_work/input", # or your dataset/input output_path="data/sample/ramp_work", # or your dataset - backbone="efficientnetb0", - num_epochs=100, - batch_size=16, - learning_rate=3e-4, - early_stopping_patience=35, - val_fraction=0.15, ) +# Use a different STAC item (e.g. versioned layout): +# training_pipeline(..., stac_item_path="models/ramp/1/stac-item.json") # Inference run (model_uri from STAC or local path) inference_pipeline( diff --git a/models/ramp/pipeline.py b/models/ramp/pipeline.py index 848e840b..1641f8c8 100644 --- a/models/ramp/pipeline.py +++ b/models/ramp/pipeline.py @@ -1,9 +1,9 @@ """ZenML pipeline for RAMP (EfficientNetB0 + U-Net) building semantic segmentation. -Entrypoints referenced by models/ramp/stac-item.json. +Entrypoints referenced by stac-item.json. Runtime: ramp-fair (TensorFlow/Keras), hot-fair-utilities (preprocessing only). -Implements the fAIr 3.0 contract (FAIr_3.0_Optimized_Pipeline.md): +Implements the fAIr entrypoints: - pre_processing_function → preprocess() - post_processing_function → postprocess() - mlm:entrypoint (training) → training_pipeline() @@ -16,7 +16,7 @@ - Postprocessing → ramp.utils.mask_to_vec_utils (GDAL polygonize) Model weights: Backend passes model_uri from STAC Item (assets.model.href). -Supports Google Drive, direct HTTP URLs, S3 (future), and local paths. +Supports Google Drive, direct HTTP URLs, S3, and local paths. Weights are downloaded on first use and cached locally. All heavy imports are lazy: this module is importable in the fAIr-models @@ -31,13 +31,11 @@ import shutil import zipfile from pathlib import Path -from typing import Annotated from urllib.request import urlretrieve -from annotated_types import Ge, Le from zenml import log_metadata, pipeline, step -# Cache directory for downloaded models (inside container, typically /workspace) +# Cache directory for downloaded models _DEFAULT_MODEL_CACHE = Path("/workspace/.ramp_model_cache") # Google Drive folder URL pattern @@ -288,20 +286,51 @@ def _make_val_split( ) +def _load_hyperparams_from_stac(stac_item_path: str) -> dict: + """Load mlm:hyperparameters from a STAC Item JSON file. + + Relative paths are resolved against /workspace (container) or cwd (local). + """ + import json + + path = Path(stac_item_path) + if not path.is_absolute(): + for base in (Path("/workspace"), Path.cwd()): + candidate = base / path + if candidate.is_file(): + path = candidate + break + else: + path = Path("/workspace") / path + with open(path, encoding="utf-8") as f: + item = json.load(f) + return dict(item.get("properties", {}).get("mlm:hyperparameters", {})) + + def _build_train_config( chips_subdir: str, masks_subdir: str, val_chips_subdir: str, val_masks_subdir: str, checkpts_subdir: str, - backbone: str, - num_epochs: int, - batch_size: int, - learning_rate: float, - early_stopping_patience: int, + hyperparams: dict, timestamp: str, ) -> dict: - """Build a RAMP JSON training config dict from pipeline parameters.""" + """Build a RAMP training config dict from STAC mlm:hyperparameters. + + hyperparams: from STAC Item properties.mlm:hyperparameters. + Path params (chips_subdir, etc.) are runtime-derived and passed separately. + """ + # Map STAC keys to RAMP config keys (STAC may use "epochs", RAMP uses "num_epochs") + num_epochs = hyperparams.get("num_epochs", hyperparams.get("epochs", 100)) + batch_size = hyperparams.get("batch_size", 16) + learning_rate = hyperparams.get("learning_rate", 3e-4) + early_stopping_patience = hyperparams.get("early_stopping_patience", 35) + backbone = hyperparams.get("backbone", "efficientnetb0") + input_img_shape = hyperparams.get("input_img_shape", [256, 256]) + output_img_shape = hyperparams.get("output_img_shape", [256, 256]) + use_aug = hyperparams.get("augmentation", hyperparams.get("use_aug", False)) + return { "experiment_name": "RAMP EffUnet multimask training", "discard_experiment": False, @@ -312,11 +341,11 @@ def _build_train_config( "val_img_dir": val_chips_subdir, "val_mask_dir": val_masks_subdir, }, - "num_classes": 4, + "num_classes": hyperparams.get("num_classes", 4), "num_epochs": num_epochs, "batch_size": batch_size, - "input_img_shape": [256, 256], - "output_img_shape": [256, 256], + "input_img_shape": input_img_shape, + "output_img_shape": output_img_shape, "loss": { "get_loss_fn_name": "get_sparse_categorical_crossentropy_fn", "loss_fn_parms": {}, @@ -338,7 +367,7 @@ def _build_train_config( }, }, "saved_model": {"use_saved_model": False}, - "augmentation": {"use_aug": False}, + "augmentation": {"use_aug": use_aug}, "early_stopping": { "use_early_stopping": True, "early_stopping_parms": { @@ -384,20 +413,18 @@ def run_preprocessing( def train_model( data_base_path: str, preprocessed_path: str, - backbone: str = "efficientnetb0", - num_epochs: int = 100, - batch_size: int = 16, - learning_rate: float = 3e-4, - early_stopping_patience: int = 35, - val_fraction: float = 0.15, + stac_item_path: str = "models/ramp/stac-item.json", + val_fraction: float | None = None, ) -> str: """Fine-tune EfficientNetB0 + U-Net on 4-class multimask chips. - 1. Splits preprocessed chips/masks into train and val sets. - 2. Builds the EfficientNet-B0 U-Net from segmentation_models. - 3. Trains with sparse categorical crossentropy loss. - 4. Returns the path to the best Keras SavedModel directory. + Hyperparameters are loaded from STAC Item (properties.mlm:hyperparameters). + stac_item_path: path to STAC Item JSON (relative to /workspace or cwd, or absolute). + Default matches current flat layout; for FAIr 3.0 versioned layout use + e.g. "models/ramp/1/stac-item.json". + val_fraction: optional override for validation split (not in STAC by default). + Returns the path to the best Keras SavedModel directory. val_sparse_categorical_accuracy is logged as ZenML step metadata. """ import os @@ -421,6 +448,17 @@ def train_model( os.environ["RAMP_HOME"] = data_base_path + hyperparams = _load_hyperparams_from_stac(stac_item_path) + if val_fraction is not None: + hyperparams["val_fraction"] = val_fraction + + val_fraction_val = hyperparams.get("val_fraction", 0.15) + batch_size = hyperparams.get("batch_size", 16) + num_epochs = hyperparams.get("num_epochs", hyperparams.get("epochs", 100)) + + # Paths under preprocessed_path ( = output_path/preprocessed ). Not persistent + # in the container unless output_path is on a mounted volume; per FAIr 3.0 + # the backend/ZenML persists outputs (e.g. best checkpoint) to S3 after the run. pre_path = Path(preprocessed_path) chips_dir = pre_path / "chips" masks_dir = pre_path / "multimasks" @@ -429,7 +467,7 @@ def train_model( checkpts_dir = pre_path / "checkpoints" if not val_chips_dir.is_dir(): - _make_val_split(chips_dir, masks_dir, val_chips_dir, val_masks_dir, val_fraction) + _make_val_split(chips_dir, masks_dir, val_chips_dir, val_masks_dir, val_fraction_val) def _rel(d: Path) -> str: return str(d.relative_to(data_base_path)) @@ -441,11 +479,7 @@ def _rel(d: Path) -> str: val_chips_subdir=_rel(val_chips_dir), val_masks_subdir=_rel(val_masks_dir), checkpts_subdir=_rel(checkpts_dir), - backbone=backbone, - num_epochs=num_epochs, - batch_size=batch_size, - learning_rate=learning_rate, - early_stopping_patience=early_stopping_patience, + hyperparams=hyperparams, timestamp=timestamp, ) @@ -593,16 +627,19 @@ def run_postprocessing( def training_pipeline( input_path: str, output_path: str, - backbone: str = "efficientnetb0", - num_epochs: Annotated[int, Ge(1), Le(2000)] = 100, - batch_size: Annotated[int, Ge(1), Le(64)] = 16, - learning_rate: Annotated[float, Ge(1e-6), Le(1e-2)] = 3e-4, - early_stopping_patience: Annotated[int, Ge(5), Le(100)] = 35, - val_fraction: Annotated[float, Ge(0.05), Le(0.4)] = 0.15, - boundary_width: int = 3, - contact_spacing: int = 8, + stac_item_path: str = "models/ramp/stac-item.json", ) -> None: - """Full RAMP training: georeference + multimask → val split → EfficientNetB0 U-Net.""" + """Full RAMP training: georeference + multimask → val split → EfficientNetB0 U-Net. + + Hyperparameters (backbone, epochs, batch_size, boundary_width, contact_spacing, etc.) + are loaded from the STAC Item at stac_item_path. Default path matches the current + flat layout (models/ramp/stac-item.json); for FAIr 3.0 versioned layout use + e.g. stac_item_path="models/ramp/1/stac-item.json". + """ + hyperparams = _load_hyperparams_from_stac(stac_item_path) + boundary_width = hyperparams.get("boundary_width", 3) + contact_spacing = hyperparams.get("contact_spacing", 8) + preprocessed_path = run_preprocessing( input_path=input_path, output_path=f"{output_path}/preprocessed", @@ -612,12 +649,7 @@ def training_pipeline( train_model( data_base_path=output_path, preprocessed_path=preprocessed_path, - backbone=backbone, - num_epochs=num_epochs, - batch_size=batch_size, - learning_rate=learning_rate, - early_stopping_patience=early_stopping_patience, - val_fraction=val_fraction, + stac_item_path=stac_item_path, ) diff --git a/models/ramp/stac-item.json b/models/ramp/stac-item.json index bf1e201b..c98f4ab6 100644 --- a/models/ramp/stac-item.json +++ b/models/ramp/stac-item.json @@ -29,18 +29,64 @@ "mlm:tasks": [ "semantic-segmentation" ], - "mlm:framework": "tensorflow", + "mlm:framework": "TensorFlow", "mlm:framework_version": "2.9.3", "mlm:pretrained": false, "mlm:pretrained_source": null, "keywords": [ "building", - "semantic-segmentation", - "multimask", - "efficientnet", - "unet" + "semantic-segmentation" ], "version": "1", + "license": "Apache-2.0", + "status": "active", + "mlm:total_parameters": 7500000, + "mlm:memory_size": 536870912, + "mlm:accelerator": null, + "mlm:accelerator_constrained": false, + "licenses": { + "model": "Apache-2.0", + "data": "ODbL-1.0", + "code": "Apache-2.0" + }, + "inference_time_per_256x256": { + "value": 45, + "unit": "ms" + }, + "pipeline_input": { + "type": "directory", + "description": "Directory of georeferenced RGB GeoTIFF chips (256×256, EPSG:3857)", + "expected_files": "*.tif", + "required_params": ["input_path", "model_uri"] + }, + "pipeline_output": { + "type": "directory", + "description": "Directory of GeoJSON files with building polygons (one per chip)", + "expected_files": "*.geojson", + "geometry_types": ["Polygon", "MultiPolygon"] + }, + "evaluation_metrics": [ + { + "metric": "val_sparse_categorical_accuracy", + "definition": "Sparse categorical accuracy on validation set", + "value": 0.92 + }, + { + "metric": "val_loss", + "definition": "Sparse categorical crossentropy on validation set", + "value": 0.15 + } + ], + "zenml_entrypoints": { + "preprocessing": "models.ramp.pipeline:run_preprocessing", + "inference": "models.ramp.pipeline:inference_pipeline", + "postprocessing": "models.ramp.pipeline:run_postprocessing", + "training": "models.ramp.pipeline:train_model" + }, + "training_time": { + "value": 4.0, + "unit": "hours" + }, "mlm:input": [ { "name": "RGB GeoTIFF chips", @@ -134,20 +180,19 @@ "type": "text/html", "title": "RAMP model pack source", "roles": ["code"], - "mlm:entrypoint": "models.ramp.pipeline:training_pipeline", - "mlm:inference_entrypoint": "models.ramp.pipeline:inference_pipeline" + "mlm:entrypoint": "models.ramp.pipeline:training_pipeline" }, - "mlm:training": { - "href": "local", - "type": "text/plain", - "title": "Training runtime (Docker)", - "roles": ["mlm:training-runtime", "runtime"] + "training-runtime": { + "href": "ghcr.io/hotosm/fair-models/ramp:v1", + "type": "application/vnd.oci.image.index.v1+json", + "title": "Training Docker Image", + "roles": ["training-runtime", "runtime"] }, - "mlm:inference": { - "href": "local", - "type": "text/plain", - "title": "Inference runtime (Docker)", - "roles": ["mlm:inference-runtime", "runtime"] + "inference-runtime": { + "href": "ghcr.io/hotosm/fair-models/ramp:v1", + "type": "application/vnd.oci.image.index.v1+json", + "title": "Inference Docker Image", + "roles": ["inference-runtime", "runtime"] } }, "links": [] diff --git a/models/ramp/tests/CODE_EXPLAINED.md b/models/ramp/tests/CODE_EXPLAINED.md deleted file mode 100644 index 84cfb1b7..00000000 --- a/models/ramp/tests/CODE_EXPLAINED.md +++ /dev/null @@ -1,465 +0,0 @@ -# Line-by-Line Explanation of `inside_container_smoke_test.py` - -A beginner-friendly walkthrough of the RAMP smoke test script. - ---- - -## What is `# noqa: F401`? - -**F401** is a **Flake8** (Python linter) rule code. It means: - -- **F** = Pyflakes (the part of Flake8 that checks for logic/import issues) -- **401** = "Module imported but unused" - -So **F401** = "You imported a module but never use any name from it." - -When we write: - -```python -import tensorflow as tf # noqa: F401 -``` - -we are: - -1. **Importing** the `tensorflow` package so Python loads it (and we check it’s installed). -2. **Not using** `tf` later in a way the linter sees (we do use `tf.__version__` and `tf.keras` elsewhere, but on *this* line we only care that the import works). -3. **Silencing the linter** with `# noqa: F401` so it doesn’t report "imported but unused" for this line. - -So: **F401 = "imported but unused"**, and **noqa: F401** means "don’t warn about F401 on this line." - ---- - -## Top of the file (docstring and imports) - -```python -"""End-to-end smoke tests for models/ramp Docker runtime. -... -""" -``` - -- **Triple-quoted string** at the top of a file is the **docstring** for the module. It describes what the script does. Tools and `help()` can show it. - -```python -from __future__ import annotations -``` - -- Makes type hints (like `str`, `Path`) be treated as strings by the interpreter. Helps with forward references and cleaner type hints. - -```python -import argparse -import os -import shutil -from pathlib import Path -``` - -- **argparse**: read command-line arguments (e.g. `--dataset-root`, `--epochs`). -- **os**: environment variables, path checks (e.g. `os.environ`, `os.path`). -- **shutil**: copy/delete trees (e.g. `shutil.rmtree`, `shutil.copy`). -- **pathlib.Path**: object-oriented paths (`Path("a") / "b"` → `a/b`), `.is_dir()`, `.glob()`, etc. - ---- - -## Helper functions - -```python -def _assert(condition: bool, message: str) -> None: - if not condition: - raise RuntimeError(message) -``` - -- **def**: define a function. -- **condition: bool**: argument must be a boolean (True/False). -- **message: str**: argument must be a string. -- **-> None**: function returns nothing. -- If `condition` is False, we raise `RuntimeError(message)` so the test fails with a clear message. The leading `_` means "internal/private" by convention. - -```python -def _stage(name: str) -> None: - print(f"\n=== {name} ===") -``` - -- Prints a section header like `\n=== Test 1: Critical imports ===`. The **f-string** `f"..."` lets you embed `{name}` in the string. - -```python -def _count_files(path: Path, pattern: str) -> int: - return len(list(path.glob(pattern))) -``` - -- **path.glob(pattern)**: finds all files under `path` matching `pattern` (e.g. `"*.png"`). Returns an iterator. -- **list(...)**: turn that iterator into a list. -- **len(...)**: number of items. So this returns how many files match the pattern. - ---- - -## Parsing command-line arguments - -```python -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Run RAMP container smoke tests.") -``` - -- **ArgumentParser**: object that will define and parse CLI arguments. -- **argparse.Namespace**: type of the object you get back (e.g. `args.dataset_root`). - -```python - parser.add_argument( - "--dataset-root", - default="/workspace/data/sample", - help="...", - ) -``` - -- Adds an option `--dataset-root`. If the user doesn’t pass it, `args.dataset_root` is `"/workspace/data/sample"`. -- **help**: text shown when you run `python script.py --help`. - -```python - parser.add_argument("--epochs", type=int, default=2) -``` - -- **type=int**: value is converted to an integer. -- Same idea for **--batch-size** and **--backbone** (string). - -```python - return parser.parse_args() -``` - -- Reads `sys.argv` (the actual command line), fills in the values, and returns an object with attributes like `args.dataset_root`, `args.epochs`, etc. - ---- - -## Preparing data/sample layout - -```python -def _prepare_data_sample_layout(dataset_root: Path) -> Path: -``` - -- Takes a path to the dataset root, returns the path to the "work" root (where we’ll put prepared input). - -```python - oam_dir = dataset_root / "train" / "oam" - osm_dir = dataset_root / "train" / "osm" -``` - -- **Path / "train" / "oam"**: builds path like `dataset_root/train/oam`. `/` on `Path` joins path parts. - -```python - _assert(oam_dir.is_dir(), f"train/oam not found under {dataset_root}") -``` - -- Fails fast if `train/oam` (or `train/osm`) doesn’t exist. - -```python - work_root = dataset_root / "ramp_work" - input_dir = work_root / "input" - input_dir.mkdir(parents=True, exist_ok=True) -``` - -- **mkdir(parents=True, exist_ok=True)**: create the directory and any missing parents; don’t error if it already exists. - -```python - import numpy as np - import rasterio - from PIL import Image -``` - -- Imports used only in this function. **numpy**: arrays/math. **rasterio**: read/write rasters (e.g. GeoTIFF). **PIL.Image**: image I/O (e.g. save PNG). - -```python - tif_files = sorted(oam_dir.glob("OAM-*.tif")) -``` - -- **glob("OAM-*.tif")**: all files whose names match that pattern. **sorted()**: stable order for reproducibility. - -```python - for tif_path in tif_files: - png_path = input_dir / (tif_path.stem + ".png") -``` - -- **tif_path.stem**: filename without extension (e.g. `OAM-386695-220244-19`). So we get a path like `input_dir/OAM-386695-220244-19.png`. - -```python - with rasterio.open(tif_path) as src: - data = src.read() -``` - -- **with ... as src**: open the file and automatically close it when the block ends. **src.read()**: read all bands as a 3D array (bands, height, width). - -```python - if data.shape[0] >= 3: - rgb = np.transpose(data[:3], (1, 2, 0)) -``` - -- **data.shape[0]**: number of bands. We need at least 3 (R, G, B). -- **data[:3]**: first 3 bands. **np.transpose(..., (1, 2, 0))**: change layout from (bands, H, W) to (H, W, bands) for image-style arrays. - -```python - if rgb.max() <= 1.0: - rgb = (rgb * 255).astype(np.uint8) - Image.fromarray(rgb).save(png_path) -``` - -- If values are in 0–1, scale to 0–255 and convert to 8-bit integer. **Image.fromarray(rgb).save(png_path)**: write a PNG file. - -```python - geojson_files = sorted(osm_dir.glob("*.geojson")) - ... - gdfs = [gpd.read_file(p) for p in geojson_files] -``` - -- **gpd.read_file(p)**: read a GeoJSON into a GeoDataFrame. The list comprehension builds a list of one GeoDataFrame per file. - -```python - if len(gdfs) == 1: - merged = gdfs[0] - else: - import pandas as pd - crs = gdfs[0].crs or "EPSG:4326" - merged = gpd.GeoDataFrame( - pd.concat([g.to_crs(crs) for g in gdfs], ignore_index=True), - crs=crs, - ) -``` - -- One file → use it as-is. Multiple → convert all to the same CRS (**crs**), concatenate with **pd.concat**, wrap in **GeoDataFrame**, and assign **crs**. - -```python - labels_path = input_dir / "labels.geojson" - merged.to_file(labels_path, driver="GeoJSON") -``` - -- Write the (possibly merged) labels to **input/labels.geojson** so the rest of the pipeline sees a single labels file. - -```python - return work_root -``` - -- Return the path to the directory that now contains **input/** with PNGs and **labels.geojson**. - ---- - -## main() — entry point - -```python -def main() -> None: - args = parse_args() -``` - -- Parse CLI into **args**. - -```python - os.environ.setdefault("RAMP_HOME", "/workspace") -``` - -- Set **RAMP_HOME** only if it’s not already set. Some RAMP code uses this. - -```python - dataset_root = Path(args.dataset_root).resolve() -``` - -- **Path(...)**: turn the string into a Path. **.resolve()**: make it absolute and normalize (e.g. resolve `..`). - -```python - if (dataset_root / "train" / "oam").is_dir() and (dataset_root / "train" / "osm").is_dir(): - dataset_root = _prepare_data_sample_layout(dataset_root) -``` - -- If we see the **data/sample** layout (train/oam and train/osm), we prepare it and then **dataset_root** becomes the work root (**ramp_work**). - -```python - input_dir = dataset_root / "input" - preprocessed_dir = dataset_root / "preprocessed_test" - chips_dir = preprocessed_dir / "chips" - ... -``` - -- Define all the paths we’ll use for inputs and outputs (chips, masks, val split, checkpoints, prediction output, vectors). No files are created yet; these are just path variables. - ---- - -## Test 1: Critical imports - -```python - import segmentation_models as sm # noqa: F401 - import tensorflow as tf # noqa: F401 - from osgeo import gdal # noqa: F401 - import ramp # noqa: F401 - import hot_fair_utilities # noqa: F401 -``` - -- We import these **to check they are installed and loadable**. If any import fails, the script crashes and the test fails. We use **noqa: F401** because the linter would otherwise say "imported but unused" on some of these lines (we do use `sm`, `tf`, etc. later; the comment keeps the linter happy where it still complains). - -```python - sm.set_framework("tf.keras") -``` - -- Tell **segmentation_models** to use TensorFlow/Keras as the backend. - -```python - print(f"PASS: tensorflow {tf.__version__} ...") -``` - -- **tf.__version__**: the TensorFlow version string. Printing it confirms the import worked and helps with debugging. - ---- - -## Test 2: Dataset layout - -```python - _assert(dataset_root.is_dir(), ...) - _assert(input_dir.is_dir(), ...) - _assert((input_dir / "labels.geojson").is_file(), ...) -``` - -- Ensure the expected folders and **labels.geojson** exist; otherwise raise with a clear message. - -```python - n_png = _count_files(input_dir, "*.png") - _assert(n_png > 0, ...) -``` - -- Count PNGs in **input/** and require at least one. - ---- - -## Test 3: Preprocessing - -```python - shutil.rmtree(preprocessed_dir, ignore_errors=True) -``` - -- Remove any previous preprocessed output so we start clean. **ignore_errors=True**: don’t fail if the directory doesn’t exist. - -```python - from hot_fair_utilities import preprocess as _preprocess - _preprocess( - input_path=str(input_dir), - output_path=str(preprocessed_dir), - ... - ) -``` - -- **import ... as _preprocess**: we only call the function; the leading `_` suggests "used locally." -- **str(...)**: some APIs expect strings, not **Path**; we convert so it works everywhere. - -```python - n_chips = _count_files(chips_dir, "*.tif") - n_masks = _count_files(masks_dir, "*.mask.tif") - _assert(n_chips == n_masks, ...) -``` - -- After preprocessing we expect one chip and one mask per image; we check that the counts match. - ---- - -## Test 4: Train/val split - -```python - chip_files = sorted(chips_dir.glob("*.tif")) - n_val = max(1, int(len(chip_files) * 0.2)) -``` - -- **n_val**: 20% of chips for validation, but at least 1. - -```python - random.shuffle(chip_files) - val_chip_files = chip_files[:n_val] -``` - -- Shuffle and take the first **n_val** as validation; the rest stay as training. - -```python - for chip_path in val_chip_files: - mask_path = masks_dir / (chip_path.stem + ".mask.tif") - if mask_path.is_file(): - shutil.copy(str(chip_path), val_chips_dir / chip_path.name) - shutil.copy(str(mask_path), val_masks_dir / mask_path.name) - moved += 1 -``` - -- For each validation chip, find the matching mask by name (e.g. `chip.tif` → `chip.mask.tif`). If it exists, copy both chip and mask into the val directories and count. - ---- - -## Test 5: Training - -- **cfg**: a big dictionary that RAMP uses for training (epochs, batch size, loss, optimizer, callbacks, etc.). This is the "config" for the run. -- **loss_fn**, **optimizer**, **acc_metric**: built from RAMP’s constructors using **cfg**. -- **sm.Unet(...)**: build the U-Net model. **encoder_weights=None** avoids downloading pretrained weights (which can 404 in CI). -- **the_model.compile(...)**: attach optimizer, loss, and metric for training. -- **training_batches_from_gtiff_dirs** / **test_batches_from_gtiff_dirs**: RAMP functions that yield batches from the chip and mask directories. -- **the_model.fit(...)**: run training for **args.epochs** (e.g. 2) with the given batches and callbacks. -- **the_model.save(model_save_path)**: save the trained model so we can load it in the next stage. - ---- - -## Test 6: Inference - -```python - inference_model = tf.keras.models.load_model(model_save_path, compile=False) -``` - -- Load the SavedModel we just saved. **compile=False**: we don’t need the training graph, only forward pass. - -```python - for chip_file in chip_files[:3]: -``` - -- Run prediction on only the first 3 chips to keep the smoke test fast. - -```python - with rio.open(chip_file) as src: - dst_profile = src.profile.copy() - ... - img = to_channels_last(src.read()).astype("float32") -``` - -- **src.profile**: metadata (size, dtype, etc.) we reuse for the output. **to_channels_last**: convert from (C, H, W) to (H, W, C) if needed for the model. - -```python - predicted = get_mask_from_prediction(inference_model.predict(np.expand_dims(img, 0))) -``` - -- **np.expand_dims(img, 0)**: add a batch dimension (1, H, W, C). **predict(...)**: run the model. **get_mask_from_prediction**: turn model output into a class mask. Then we write the result as **.pred.tif**. - ---- - -## Test 7: Polygonization - -```python - ref_ds = gdal.Open(str(pred_tif)) - multimask = gdal_get_mask_tensor(str(pred_tif)) - bin_mask = binary_mask_from_multichannel_mask(multimask) - binary_mask_to_geojson(bin_mask, ref_ds, json_path) -``` - -- **gdal.Open**: open the predicted GeoTIFF (for georeference info). -- **gdal_get_mask_tensor**: read the mask array. -- **binary_mask_from_multichannel_mask**: reduce multi-class mask to a single binary mask (e.g. building vs non-building). -- **binary_mask_to_geojson**: raster-to-vector (polygonize) and write building footprints to GeoJSON. - ---- - -## End of script - -```python -if __name__ == "__main__": - main() -``` - -- **__name__**: special variable. When you run this file as a script, **__name__** is **"__main__"**. When the file is imported, **__name__** is the module name. -- So **main()** runs only when you execute the script (e.g. `python inside_container_smoke_test.py`), not when another file does `import inside_container_smoke_test`. - ---- - -## Quick reference - -| Concept | Meaning | -|--------|--------| -| **F401** | Flake8: "module imported but unused" | -| **noqa: F401** | "Don’t report F401 on this line" | -| **Path / "a" / "b"** | Path joining: `path/a/b` | -| **path.glob("*.png")** | All files matching pattern under **path** | -| **with open(...) as f:** | Use **f** and close the resource when the block ends | -| **f"text {var}"** | f-string: insert **var** into the string | -| **-> None** | This function returns nothing | -| **type=int** | Convert CLI argument to int | diff --git a/models/ramp/tests/README.md b/models/ramp/tests/README.md deleted file mode 100644 index d26b08a1..00000000 --- a/models/ramp/tests/README.md +++ /dev/null @@ -1,137 +0,0 @@ -# RAMP Smoke Tests — In-Depth Guide - -This folder contains end-to-end smoke tests that validate the RAMP Docker runtime. -Tests run **inside** the container after the repo is mounted at `/workspace`. - -## Test Files Overview - -| File | Purpose | -| --- | --- | -| `inside_container_smoke_test.py` | Main test script. Run inside the container. Validates 7 stages (imports → preprocessing → train → inference → polygonization). | -| `run_docker_tests.ps1` | PowerShell runner: builds image (optional), runs container, executes the smoke test. | -| `run_docker_tests.sh` | Bash runner: same as above for Linux/macOS. | -| `test_plan.yaml` | Declarative description of what each stage checks. Used for documentation and CI planning. | - -## Data Source: `data/sample` - -Tests use **`data/sample`** (fAIr-models shared sample) instead of model-specific `ramp-data`. - -**Layout:** - -```text -data/sample/ -├── train/oam/ # OAM GeoTIFF tiles (OAM-{x}-{y}-{z}.tif) -├── train/osm/ # OSM building labels (*.geojson) -└── predict/oam/ # (Optional) images for inference -``` - -**Adapter logic:** `hot_fair_utilities.preprocess` expects `input/*.png` and `input/labels.geojson`. -The test script detects the `data/sample` layout and: - -1. Creates `data/sample/ramp_work/input/` -2. Converts `train/oam/*.tif` → PNG (rasterio + PIL) -3. Merges `train/osm/*.geojson` → `labels.geojson` -4. Runs the pipeline on `ramp_work/input/` - -Outputs go to `data/sample/ramp_work/preprocessed_test/` and `prediction_test/`. - -## Seven Test Stages (inside_container_smoke_test.py) - -### 1. Critical Imports - -Verifies runtime packages: `tensorflow`, `segmentation_models`, `ramp`, `hot_fair_utilities`, `osgeo.gdal`, `solaris`. -Sets `segmentation_models` to use `tf.keras` backend. - -### 2. Input Dataset Layout - -- **data/sample**: Requires `train/oam/`, `train/osm/`, at least one `OAM-*.tif`, and one `*.geojson`. -- **Legacy**: Requires `input/`, `input/labels.geojson`, and `input/*.png`. - -### 3. Preprocessing - -Calls `hot_fair_utilities.preprocess()` with: - -- `georeference_images=True` — converts PNG to GeoTIFF (EPSG:3857) -- `rasterize=True`, `rasterize_options=["binary"]` -- `multimasks=True` — 4-class masks (background, building, boundary, contact) - -Checks: `chips/*.tif` and `multimasks/*.mask.tif` exist and counts match. - -### 4. Train / Val Split - -Shuffles chip/mask pairs, moves ~20% to `val_chips/` and `val_multimasks/`. -RAMP’s data generator needs separate train and validation directories. - -### 5. Training Smoke Run - -Builds EfficientNetB0 U-Net (`encoder_weights=None` to avoid 404 on pretrained weights). -Runs 2 epochs, saves SavedModel to `checkpoints/`. - -### 6. Inference - -Loads SavedModel, runs prediction on first 3 chips. -Produces `*.pred.tif` (4-class uint8 masks). - -### 7. Polygonization - -Uses `ramp.utils.mask_to_vec_utils` (GDAL Polygonize) to convert predicted masks to GeoJSON building footprints. -Checks for `*.geojson` in `prediction_test/vectors/`. - -## How to Run - -**Prerequisite:** Ensure `data/sample` exists with `train/oam/` and `train/osm/`. - -### PowerShell (Windows) - -```powershell -cd fAIr-models # repo root - -# Build image and run tests -.\models\ramp\tests\run_docker_tests.ps1 -BuildImage - -# Or run tests only (image must already exist) -.\models\ramp\tests\run_docker_tests.ps1 - -# CPU-only (no GPU) -.\models\ramp\tests\run_docker_tests.ps1 -BuildImage -CpuOnly -``` - -### Bash (Linux / macOS) - -```bash -cd fAIr-models - -# Build and run -BUILD_IMAGE=1 ./models/ramp/tests/run_docker_tests.sh - -# CPU-only -CPU_ONLY=1 BUILD_IMAGE=1 ./models/ramp/tests/run_docker_tests.sh -``` - -### Manually Inside Container - -```bash -docker run --rm -v /path/to/fAIr-models:/workspace ramp-v1:gpu \ - python /workspace/models/ramp/tests/inside_container_smoke_test.py \ - --dataset-root /workspace/data/sample -``` - -### With Custom Dataset (Legacy Layout) - -```bash -# Legacy layout: dataset/input/*.png and dataset/input/labels.geojson -docker run --rm -v /path/to/data:/workspace/data ramp-v1:gpu \ - python /workspace/models/ramp/tests/inside_container_smoke_test.py \ - --dataset-root /workspace/data/your_dataset -``` - -## What Gets Created - -Running tests creates: - -- `data/sample/ramp_work/input/` — prepared PNG chips + labels.geojson (data/sample only) -- `data/sample/ramp_work/preprocessed_test/` — chips, multimasks, val split, checkpoints -- `data/sample/ramp_work/prediction_test/output/` — `*.pred.tif` -- `data/sample/ramp_work/prediction_test/vectors/` — `*.geojson` building polygons - -These paths are in `.gitignore`. From 0c41a1a04bb8964395a792a4006fbd20b3f800f4 Mon Sep 17 00:00:00 2001 From: Abdelrahman Katkat Date: Tue, 7 Apr 2026 14:25:57 +0200 Subject: [PATCH 03/30] feat(ramp): update Dockerfile and pipeline for improved functionality and compatibility - Refactored Dockerfile to streamline the build process, using a base image from GHCR for both CPU and GPU. - Enhanced pipeline.py to support new model weight loading mechanisms and improved error handling for model paths. - Updated README.md to reflect changes in framework version and model usage, including new data directory structure. - Revised stac-item.json to include updated model weights source and additional metadata for better integration. - Improved smoke tests to validate the new pipeline functionality and ensure compatibility with the latest TensorFlow/Keras versions. - Added per-file ignores in Ruff for specific linting rules in pipeline.py. --- models/ramp/Dockerfile | 116 +-- models/ramp/README.md | 197 ++-- models/ramp/pipeline.py | 860 ++++++++---------- models/ramp/stac-item.json | 29 +- .../ramp/tests/inside_container_smoke_test.py | 353 +++---- models/ramp/tests/run_docker_tests.ps1 | 85 +- models/ramp/tests/run_docker_tests.sh | 18 +- models/ramp/tests/test_plan.yaml | 58 -- pyproject.toml | 6 + 9 files changed, 781 insertions(+), 941 deletions(-) delete mode 100644 models/ramp/tests/test_plan.yaml diff --git a/models/ramp/Dockerfile b/models/ramp/Dockerfile index 45bc49f6..d19d2457 100644 --- a/models/ramp/Dockerfile +++ b/models/ramp/Dockerfile @@ -1,103 +1,23 @@ -# Build instructions (from fAIr-models repo root): -# GPU: docker build -f models/ramp/Dockerfile --build-arg BUILD_TYPE=gpu -t ramp-v1:gpu . -# CPU: docker build -f models/ramp/Dockerfile --build-arg BUILD_TYPE=cpu -t ramp-v1:cpu . +# syntax=docker/dockerfile:1.7 -ARG PY_VER=3.10 -ARG TF_VER=2.9.3 -ARG BUILD_TYPE=gpu -ARG CUDA_TAG=11.8.0-cudnn8-runtime-ubuntu22.04 +# Base: ghcr.io/hotosm/fair-utilities-ramp:cpu-latest (or :gpu-latest via --build-arg BASE_IMAGE=...) +# Build from fAIr-models root: docker build -f models/ramp/Dockerfile -t ramp-v1:cpu . +ARG BASE_IMAGE=ghcr.io/hotosm/fair-utilities-ramp:cpu-latest +FROM ${BASE_IMAGE} -# ============================================================================== -# === CPU base image (minimal) ================================================= -FROM python:${PY_VER}-slim-bookworm AS cpu-base -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - git build-essential gcc g++ python3-dev python3-rtree \ - gdal-bin libgdal-dev python3-gdal python3-opencv libspatialindex-dev libgeos-dev \ - libgl1 libglib2.0-0 \ - && rm -rf /var/lib/apt/lists/* +ENV MPLBACKEND=Agg \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + RAMP_HOME=/workspace \ + SM_FRAMEWORK=tf.keras -# ============================================================================== -# === GPU base image (CUDA + runtime-only Python & GDAL) ======================= -FROM nvidia/cuda:${CUDA_TAG} AS gpu-base -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - git python3 python3-pip build-essential gcc g++ python3-dev python3-rtree python-is-python3 \ - gdal-bin libgdal-dev python3-gdal python3-opencv libspatialindex-dev libgeos-dev \ - libgl1 libglib2.0-0 \ - && rm -rf /var/lib/apt/lists/* && \ - python3 -m pip install --upgrade pip - -# ============================================================================== -# === Builder stage (installs everything) ====================================== -FROM ${BUILD_TYPE}-base AS builder -ENV DEBIAN_FRONTEND=noninteractive \ - TF_CPP_MIN_LOG_LEVEL=2 -ARG TF_VER - -ENV CPLUS_INCLUDE_PATH=/usr/include/gdal \ - C_INCLUDE_PATH=/usr/include/gdal - -# Use pip cache and install Python packages -RUN --mount=type=cache,target=/root/.cache/pip \ - pip install --no-cache-dir --upgrade pip more-itertools && \ - pip install --no-cache-dir "numpy>=1.22,<2.0" && \ - pip install --no-cache-dir "GDAL==$(gdal-config --version)" && \ - pip install --no-cache-dir tensorflow==${TF_VER} && \ - pip install --no-cache-dir \ - "efficientnet==1.0.0" \ - "image-classifiers==1.0.0" \ - "segmentation-models==1.0.1" && \ - pip install --no-cache-dir \ - "geopandas==0.10.2" \ - "rasterio>=1.3.10" \ - "Shapely>=1.8.1" \ - "pyproj==3.3.0" \ - "scikit-image==0.19.2" \ - "scikit-learn==1.0.2" \ - "scipy==1.8.0" \ - "tqdm==4.62.3" \ - "pandas==1.4.1" \ - "matplotlib==3.5.1" \ - "albumentations==1.0.3" \ - "pyyaml>=6.0" \ - "requests>=2.27.1" \ - "tinydb==4.7.0" \ - "gdown>=5.0" - -# Compatibility pin for geopandas/raster stack -RUN --mount=type=cache,target=/root/.cache/pip \ - pip install --no-cache-dir "fiona>=1.8.22,<1.10" - -# Install solaris from ramp-code-fair (equivalent to Dockerfile.pb local install) -RUN --mount=type=cache,target=/root/.cache/pip \ - pip install --no-cache-dir \ - git+https://github.com/hotosm/ramp-code-fair.git#subdirectory=solaris - -# Install scikit-fmm and ramp-fair -RUN --mount=type=cache,target=/root/.cache/pip \ - pip install --no-cache-dir scikit-fmm && \ - pip install --no-cache-dir "ramp-fair==0.1.2" - -# Install hot-fair-utilities for shared preprocessing path used by pipeline.py -RUN --mount=type=cache,target=/root/.cache/pip \ - pip install --no-cache-dir "opencv-python-headless<=4.7.0.68" && \ - pip install --no-cache-dir "hot-fair-utilities==2.0.12" && \ - pip install --no-cache-dir "numpy>=1.22,<2.0" - -# ============================================================================== -# === Final minimal runtime image ============================================== -FROM ${BUILD_TYPE}-base AS final -ENV DEBIAN_FRONTEND=noninteractive - -ENV CPLUS_INCLUDE_PATH=/usr/include/gdal \ - C_INCLUDE_PATH=/usr/include/gdal \ - RAMP_HOME=/workspace +WORKDIR /workspace -COPY --from=builder /usr/local /usr/local -COPY --from=builder /usr/lib/python*/ /usr/lib/python*/ -COPY --from=builder /usr/include/gdal /usr/include/gdal +RUN /app/.venv/bin/python -m pip install --no-cache-dir "fair-py-ops==0.0.6" -WORKDIR /workspace +# Smoke-test extras only (ZenML[server], STAC, fairpredictor); drop for production slim images. +RUN /app/.venv/bin/python -m pip install --no-cache-dir \ + "zenml[server]==0.93.3" \ + "pystac[validation]>=1.14.3" \ + "universal-pathlib" \ + "fairpredictor>=0.5.1" diff --git a/models/ramp/README.md b/models/ramp/README.md index 74c12a95..df1106b7 100644 --- a/models/ramp/README.md +++ b/models/ramp/README.md @@ -13,7 +13,7 @@ for pixel-wise 4-class building segmentation on 256 × 256 px aerial image chips | Classes | 0=background, 1=building, 2=boundary, 3=contact-point | | Loss | Sparse categorical crossentropy | | Metric | Sparse categorical accuracy (`val_sparse_categorical_accuracy`) | -| Framework | TensorFlow 2.9.3 / Keras | +| Framework | TensorFlow 2.15.1 / Keras | The boundary (class 2) and contact-point (class 3) channels help the model cleanly separate adjacent buildings at inference time, even when they share a wall. @@ -24,16 +24,40 @@ separate adjacent buildings at inference time, even when they share a wall. | --- | --- | --- | | Preprocessing | `hot_fair_utilities.preprocess` | same | | Training | `hot_fair_utilities.training.yolo_v8_*` (ultralytics) | `ramp.training.*` (TF/Keras) | -| Inference | `hot_fair_utilities.predict` (ultralytics) | `tf.keras.models.load_model` | -| Postprocessing | `hot_fair_utilities.polygonize` (AutoBFE) | `ramp.utils.mask_to_vec_utils` (GDAL) | +| Inference | `hot_fair_utilities.predict` (ultralytics) | `fairpredictor.predictor.prediction.run_prediction` | +| Postprocessing | `hot_fair_utilities.polygonize` (AutoBFE) | `geomltoolkits` vectorization + geometry validation (writes `predictions.geojson`) | ## Model pack contents | File | Purpose | | --- | --- | | `pipeline.py` | ZenML `@step` / `@pipeline` entrypoints (pre → train → infer → post); `resolve_model_href()` for URLs | -| `stac-item.json` | STAC MLM item — model weights (mlm:model href), entrypoints; weights from Google Drive/S3 | -| `Dockerfile` | Isolated runtime (CUDA 11.8 + TF 2.9.3 + ramp-fair + hot-fair-utilities + gdown) | +| `stac-item.json` | STAC MLM item — model weights (mlm:model href), entrypoints; weights from local path or HTTP(S) `.zip` | +| `Dockerfile` | Runtime on top of `ghcr.io/hotosm/fair-utilities-ramp` (CPU/GPU); see [Docker image composition](#docker-image-composition) | + +## Docker image composition + +The RAMP image is **layered**, not a single monolithic install: + +1. **Base (hot-fair-utilities on GHCR)** + TensorFlow, GDAL, `hot-fair-utilities` RAMP extras, fairpredictor, geomltoolkits, and related geospatial/ML stack. You do not build this locally; Docker pulls `cpu-latest` or `gpu-latest`. + +2. **`fair-py-ops` (pinned)** + Installed in this Dockerfile to match `fAIr-models/pyproject.toml` (`fair-py-ops==0.0.6`). This is the long-term registry/orchestration contract for the repo. + +3. **Temporary test-only packages (remove before production merge)** + The Dockerfile installs **`zenml[server]==0.93.3`** and **`pystac[validation]>=1.14.3`**, aligned with `pyproject.toml` (`zenml>=0.93.3`, `pystac[validation]>=1.14.3`). + **Why `[server]` on ZenML:** `pipeline.py` wraps logic in `@step`. Calling a step (e.g. `run_preprocessing(...)`) runs ZenML’s single-step pipeline, which initializes the default **SQL** zen store. That code path imports **`sqlalchemy_utils`** (and related SQL stack). Those are **not** included in a bare `pip install zenml==…` — they are part of ZenML’s **`server`** optional extra (same idea as the repo’s `[dependency-groups] local` → `zenml[server]`). Without `[server]`, you get `ModuleNotFoundError: No module named 'sqlalchemy_utils'`. + **`fAIr-utilities/docker/Dockerfile.ramp`** does not install ZenML, PySTAC, `fair-py-ops`, or this SQL stack; those are added only in this model Dockerfile layer. + + **Before you push production-oriented code**, delete the **second** `RUN pip install …` block in `models/ramp/Dockerfile` (the one that installs `zenml[server]` and `pystac`). Keep the `fair-py-ops` `RUN` unless your platform injects it another way. + After removal, in-container smoke tests that import `pipeline.py` will fail unless you refactor (e.g. split core vs ZenML) or run tests only on infrastructure where ZenML is pre-installed. + +**STAC in code vs `pystac` on PyPI:** +Training hyperparameters in this pack are read from **`stac-item.json` with plain `json`** — you do not need the `pystac` Python package for that path. The Dockerfile adds `pystac[validation]` **only for testing parity** with the repo’s declared dependencies; production can rely on STAC-as-JSON plus platform tooling. + +**`SM_FRAMEWORK=tf.keras`:** +The `segmentation_models` / `efficientnet` stack reads `SM_FRAMEWORK` when the package is **first imported**. The default path uses standalone `keras` and `efficientnet.keras`, which rely on removed Keras 2 APIs (`keras.utils.generic_utils`) and fail on TensorFlow 2.15+ (bundled Keras 3). The Dockerfile sets `SM_FRAMEWORK=tf.keras` so `efficientnet.tfkeras` is used. Do not unset this in the RAMP container unless you pin an older TensorFlow/Keras stack. ## Data directory layout @@ -49,17 +73,21 @@ dataset/ ├── input/ │ ├── *.png # OAM chips (PNG, no geo-reference) │ └── labels.geojson # combined building polygon labels -├── preprocessed/ # created by run_preprocessing +├── preprocessed/ # created by preprocess (hot_fair_utilities) │ ├── chips/ # georeferenced .tif chips (EPSG:3857) │ ├── labels/ # per-chip .geojson labels │ ├── multimasks/ # 4-class .mask.tif targets -│ ├── val_chips/ # validation split chips -│ ├── val_multimasks/ # validation split masks -│ └── checkpoints// # Keras SavedModel checkpoints +│ └── (no training outputs here) # training uses a separate work dir +├── ramp_training_work/ # created by train_ramp_model +│ ├── chips/ # training chips (copied from preprocessed) +│ ├── multimasks/ # training masks (copied from preprocessed) +│ ├── val-chips/ # validation split chips (hyphenated) +│ ├── val-multimasks/ # validation split masks (hyphenated) +│ └── model-checkpts/ # SavedModel checkpoints + best model selection └── prediction/ ├── input/ # chips for inference (GeoTIFFs; typically copy from preprocessed/chips/) - ├── output/ # .pred.tif predicted masks - └── vectors/ # per-chip .geojson building polygons + ├── output/ # fairpredictor outputs (includes georeference/*.tif) + └── vectors/ # merged GeoJSON: predictions.geojson (+ merged_prediction_mask.tif, tmp/) ``` ## Pipeline steps @@ -68,85 +96,146 @@ dataset/ input/ (PNG chips + labels.geojson) │ ▼ -[run_preprocessing] hot_fair_utilities.preprocess (georeference + multimask) +[run_preprocessing] hot_fair_utilities.preprocess (georeference + multimask; returns chip/mask arrays) │ preprocessed/chips/ + preprocessed/multimasks/ ▼ -[train_model] ramp.training.* (EfficientNetB0 U-Net, TF/Keras) - │ best checkpoint (.tf SavedModel dir) + val_sparse_categorical_accuracy +[train_model] hot_fair_utilities.training.ramp.* (EfficientNetB0 U-Net, TF/Keras) + │ best checkpoint (SavedModel) loaded as tf.keras.Model + val_sparse_categorical_accuracy ▼ -[run_inference] tf.keras.models.load_model (chip-by-chip predict) - │ prediction/output/*.pred.tif +[run_inference] fairpredictor.run_prediction (georeferenced prediction rasters) + │ prediction/output/georeference/*.tif ▼ -[run_postprocessing] ramp.utils.mask_to_vec_utils (GDAL Polygonize) +[run_postprocessing] geomltoolkits vectorization + validation (writes predictions.geojson; returns dict) │ ▼ -prediction/vectors/*.geojson (per-chip building footprints) +prediction/vectors/predictions.geojson (merged building footprints) ``` ## Running locally (outside ZenML) ```python -from models.ramp.pipeline import training_pipeline, inference_pipeline +from models.ramp.pipeline import infer_ramp_model, train_ramp_model -# Full training run (use your dataset path) -# Hyperparameters are loaded from models/ramp/stac-item.json -training_pipeline( - input_path="data/sample/ramp_work/input", # or your dataset/input - output_path="data/sample/ramp_work", # or your dataset +# Training (expects you've already run preprocessing to produce chips/ + multimasks/) +trained_model = train_ramp_model( + data_base_path="data/sample/ramp_work", + preprocessed_path="data/sample/ramp_work/preprocessed_test", + stac_item_path="models/ramp/stac-item.json", ) -# Use a different STAC item (e.g. versioned layout): -# training_pipeline(..., stac_item_path="models/ramp/1/stac-item.json") -# Inference run (model_uri from STAC or local path) -inference_pipeline( - model_uri="data/sample/ramp_work/preprocessed_test/checkpoints//smoke_.tf", +# Inference run (model_uri can be a local SavedModel dir, an HTTP(S) .zip, or a tf.keras.Model) +final_geojson = infer_ramp_model( + model_uri=trained_model, input_path="data/sample/ramp_work/preprocessed_test/chips", prediction_path="data/sample/ramp_work/prediction_test/output", output_dir="data/sample/ramp_work/prediction_test/vectors", ) -# model_uri can also be: Google Drive folder URL, HTTP URL to .zip +# final_geojson is a dict (FeatureCollection-like). predictions.geojson is also written under output_dir. ``` ## Building the Docker image +The `fAIr-utilities` team publishes pre-built base images to GitHub Container Registry (GHCR) on +every push to `master`. You **do not need to build or clone the `fAIr-utilities` repo yourself**. +Docker pulls the image automatically. + +| Flavour | GHCR image | +| --- | --- | +| CPU (default) | `ghcr.io/hotosm/fair-utilities-ramp:cpu-latest` | +| GPU (CUDA) | `ghcr.io/hotosm/fair-utilities-ramp:gpu-latest` | + +Each build adds (see Dockerfile comments): + +- `fair-py-ops==0.0.6` (from `pyproject.toml`) +- **Temporary:** `zenml[server]==0.93.3` and `pystac[validation]>=1.14.3` — the `[server]` extra pulls the SQL stack (`sqlalchemy-utils`, etc.) required when `@step` runs; remove the dedicated `RUN` before a production merge if slim images must not carry orchestration deps (see [Docker image composition](#docker-image-composition)). + ```bash -# GPU image (default) +# CPU image (base image pulled from GHCR automatically — no --build-arg needed) +docker build -t ramp-v1:cpu -f models/ramp/Dockerfile . + +# GPU image (override the ARG to select the GPU base) docker build -t ramp-v1:gpu \ - --build-arg BUILD_TYPE=gpu \ + --build-arg BASE_IMAGE=ghcr.io/hotosm/fair-utilities-ramp:gpu-latest \ -f models/ramp/Dockerfile . +``` -# CPU-only (development / CI) -docker build -t ramp-v1:cpu \ - --build-arg BUILD_TYPE=cpu \ - -f models/ramp/Dockerfile . +```powershell +# PowerShell (Windows) — do NOT use "\" for line continuation +docker build -t ramp-v1:cpu -f models/ramp/Dockerfile . +docker build -t ramp-v1:gpu --build-arg BASE_IMAGE=ghcr.io/hotosm/fair-utilities-ramp:gpu-latest -f models/ramp/Dockerfile . ``` +> **Testing unreleased `fAIr-utilities` code?** If GHCR doesn't yet contain a feature you need, +> add a `pip install` from git inside your Dockerfile layer: +> ```dockerfile +> RUN pip install --no-cache-dir \ +> "hot-fair-utilities @ git+https://github.com/hotosm/fAIr-utilities.git@" +> ``` +> You do not need to build the base image yourself. + ## Running the smoke tests +The in-container script `models/ramp/tests/inside_container_smoke_test.py` imports `pipeline.py`, +which loads **ZenML** at import time. For that to work inside Docker, the image must either include +the temporary **ZenML + PySTAC** `RUN` in the Dockerfile (current approach for local/CI testing) or +you must refactor tests / pipeline imports (see [Docker image composition](#docker-image-composition)). + ```powershell # PowerShell (Windows) -.\models\ramp\tests\run_docker_tests.ps1 -BuildImage +$env:BUILD_IMAGE = "1" +$env:CPU_ONLY = "1" +.\models\ramp\tests\run_docker_tests.ps1 ``` ```bash -# Bash (Linux / macOS) -BUILD_IMAGE=1 ./models/ramp/tests/run_docker_tests.sh +# Bash (Linux / macOS / Git Bash) +BUILD_IMAGE=1 CPU_ONLY=1 ./models/ramp/tests/run_docker_tests.sh ``` > **Note**: The smoke tests use `data/sample` (train/oam OAM tiles + train/osm labels). > Run from the fAIr-models repo root so `/workspace/data/sample` is available in the container. +### Running the smoke script directly (after building the image) + +If you already built the image (see “Building the Docker image”), you can run the smoke test script directly: +`models/ramp/tests/inside_container_smoke_test.py`. + +```bash +# CPU image +docker run --rm -v "$(pwd):/workspace" ramp-v1:cpu \ + python /workspace/models/ramp/tests/inside_container_smoke_test.py \ + --dataset-root /workspace/data/sample \ + --epochs 2 --batch-size 4 --backbone efficientnetb0 + +# GPU image +docker run --rm --gpus all -v "$(pwd):/workspace" ramp-v1:gpu \ + python /workspace/models/ramp/tests/inside_container_smoke_test.py \ + --dataset-root /workspace/data/sample \ + --epochs 2 --batch-size 4 --backbone efficientnetb0 +``` + +```powershell +# PowerShell (Windows) — same idea, no Bash "\" line continuation +docker run --rm -v "${PWD}:/workspace" ramp-v1:cpu python /workspace/models/ramp/tests/inside_container_smoke_test.py --dataset-root /workspace/data/sample --epochs 2 --batch-size 4 --backbone efficientnetb0 + +docker run --rm --gpus all -v "${PWD}:/workspace" ramp-v1:gpu python /workspace/models/ramp/tests/inside_container_smoke_test.py --dataset-root /workspace/data/sample --epochs 2 --batch-size 4 --backbone efficientnetb0 +``` + +If you want to test a different dataset layout, point `--dataset-root` at a directory that contains either: + +- `train/oam/*.tif` + `train/osm/*.geojson` (sample-style), or +- `input/*.png` + `input/labels.geojson` (legacy-style) + ## Model weights (STAC mlm:model asset) The STAC Item's `assets.model.href` points to pretrained weights. Supported sources: | Source | Example | | --- | --- | -| Local path | `/workspace/checkpoints/model.tf` | -| Google Drive folder | `https://drive.google.com/drive/folders/FOLDER_ID` | -| HTTP .zip | `https://example.com/ramp_model.zip` | +| Local SavedModel directory | `/workspace/ramp-data/baseline` | +| HTTP(S) `.zip` containing a SavedModel | `https://example.com/ramp_model.zip` | -**Google Drive**: Upload the **full** Keras SavedModel directory (saved_model.pb + variables/ + assets/). The pipeline downloads via gdown and caches to `/workspace/.ramp_model_cache/`. + For remote weights, publish an HTTP(S) `.zip` that contains a SavedModel directory with `saved_model.pb` and `variables/`. Downloaded zips are cached under `/workspace/.ramp_model_cache/`. ## Registering in the STAC catalog @@ -159,12 +248,12 @@ cm.register_model("models/ramp/stac-item.json") ## Dependencies from hot_fair_utilities -Only the **preprocessing** path of `hot-fair-utilities` is used in this pack: +This pack uses **both preprocessing and training** paths from `hot-fair-utilities`: | Used | Not used | | --- | --- | -| `hot_fair_utilities.preprocess` (+ `multimasks_from_polygons`) | `hot_fair_utilities.predict` (YOLO / ultralytics) | -| `ramp.utils.multimask_utils` via transitive import | `hot_fair_utilities.polygonize` (AutoBFE, YOLO output) | +| `hot_fair_utilities.preprocess` (georeference + multimasks) | `hot_fair_utilities.predict` (YOLO / ultralytics) | +| `hot_fair_utilities.training.ramp.*` (RAMP_CONFIG, split/train helpers) | `hot_fair_utilities.polygonize` (AutoBFE, YOLO output) | The `ultralytics` and `torch` packages are installed transitively by `hot-fair-utilities` but are never imported at runtime for RAMP. @@ -176,17 +265,11 @@ The `ultralytics` and `torch` packages are installed transitively by This pack is thin: it declares *how* to run the model (pipeline.py) and *what* it is (stac-item.json). Upgrading means bumping the `ramp-fair` pin. -**Why solaris from GitHub source?** -Solaris is a geospatial ML toolkit vendored inside `ramp-code-fair`. It is -not on PyPI. The Dockerfile installs it directly from the -`hotosm/ramp-code-fair` GitHub repository's `solaris/` subdirectory. +**Where do “ramp” + “solaris” come from?** +They are pulled in via the `hot-fair-utilities[ramp]` / `[ramp-gpu]` extras in the Dockerfile. The RAMP runtime is intentionally installed as a consistent bundle inside the image so local machines don’t need to compile the full stack. -**Why separate val split in pipeline.py?** -RAMP's `data_generator` requires explicitly separate `train_img_dir` and -`val_img_dir`. The pipeline handles this automatically by shuffling a -configurable fraction (`val_fraction`, default 15%) of chips out of the -training set after preprocessing. +**Why is validation split handled in training?** +RAMP training expects explicit train/val directories. `train_ramp_model` calls `split_training_2_validation`, creating `ramp_training_work/val-chips` and `ramp_training_work/val-multimasks` under a dedicated training work directory. **Why one Dockerfile per model?** -TF 2.9.3 required for RAMP is incompatible with the TF 2.13 needed by the -YOLO packs' base image. Per-model images prevent version conflicts. +The RAMP stack depends on TensorFlow + GDAL + geospatial libs with tight version coupling. Keeping a per-model image prevents version conflicts and makes the runtime reproducible. diff --git a/models/ramp/pipeline.py b/models/ramp/pipeline.py index 1641f8c8..c8a260f4 100644 --- a/models/ramp/pipeline.py +++ b/models/ramp/pipeline.py @@ -1,7 +1,7 @@ """ZenML pipeline for RAMP (EfficientNetB0 + U-Net) building semantic segmentation. Entrypoints referenced by stac-item.json. -Runtime: ramp-fair (TensorFlow/Keras), hot-fair-utilities (preprocessing only). +Runtime: ramp-fair (TensorFlow/Keras), hot-fair-utilities (preprocessing + training). Implements the fAIr entrypoints: - pre_processing_function → preprocess() @@ -9,169 +9,153 @@ - mlm:entrypoint (training) → training_pipeline() - inference (model from STAC mlm:model asset href) → inference_pipeline() -Key architecture difference from YOLO packs: - - Preprocessing → hot_fair_utilities.preprocess (shared) - - Training → ramp.training.* (TF/Keras, NOT ultralytics) - - Inference → tf.keras.models.load_model (TF SavedModel) - - Postprocessing → ramp.utils.mask_to_vec_utils (GDAL polygonize) - Model weights: Backend passes model_uri from STAC Item (assets.model.href). -Supports Google Drive, direct HTTP URLs, S3, and local paths. -Weights are downloaded on first use and cached locally. +Supports direct HTTP(S) URLs to .zip archives and local paths. +Google Drive is not supported; weights should be published to HTTP or staged locally. + +Inference/postprocess use the **fairpredictor** PyPI distribution; its importable top-level +package is ``predictor`` (``pip install fairpredictor`` → ``import predictor``), same as +``hot_fair_utilities.inference.predict``. All heavy imports are lazy: this module is importable in the fAIr-models host environment where tensorflow, ramp, and solaris are not installed. """ -from __future__ import annotations - -import datetime -import random import re -import shutil import zipfile from pathlib import Path +from shutil import copy2, rmtree +from typing import Annotated, Any, Dict, List, Optional, Tuple, Union from urllib.request import urlretrieve from zenml import log_metadata, pipeline, step -# Cache directory for downloaded models _DEFAULT_MODEL_CACHE = Path("/workspace/.ramp_model_cache") - -# Google Drive folder URL pattern -_GDRIVE_FOLDER_RE = re.compile( - r"https?://drive\.google\.com/drive/folders/([a-zA-Z0-9_-]+)", - re.IGNORECASE, +_DEFAULT_RAMP_BASELINE_URL = ( + "https://api-prod.fair.hotosm.org/api/v1/workspace/download/ramp/baseline.zip" ) -_GDRIVE_FILE_RE = re.compile( - r"https?://drive\.google\.com/file/d/([a-zA-Z0-9_-]+)", - re.IGNORECASE, + + +def _download_and_extract_zip(zip_url: str, dest_dir: Path) -> None: + """Download a ZIP URL and extract in dest_dir.""" + dest_dir.mkdir(parents=True, exist_ok=True) + zip_name = Path(zip_url.split("/")[-1]).name or "archive.zip" + zip_path = dest_dir / zip_name + urlretrieve(zip_url, zip_path) + with zipfile.ZipFile(zip_path, "r") as zf: + zf.extractall(dest_dir) + zip_path.unlink(missing_ok=True) + + +def _to_local_path(path_value: str, purpose: str) -> Path: + """Resolve a path with UPath and ensure local filesystem semantics.""" + from upath import UPath + + upath_obj = UPath(path_value) + protocol = getattr(upath_obj, "protocol", "") or "" + if protocol not in ("", "file"): + raise NotImplementedError( + f"{purpose} requires a local filesystem path. " + f"Received protocol={protocol!r} for {path_value!r}." + ) + return Path(str(upath_obj)) + + +def _ensure_ramp_baseline(data_base_path: str, baseline_rel_path: str) -> Path: + """Return the directory that contains baseline weights; download under data_base_path if missing. + + ``baseline_rel_path`` is e.g. ``ramp-data/baseline/checkpoint.tf`` (file relative to RAMP_HOME). + """ + rel = Path(baseline_rel_path) + local_dir = Path(data_base_path) / rel.parent + local_ck = local_dir / rel.name + if local_ck.is_file() or (local_dir / "saved_model.pb").exists(): + return local_dir + image_ck = Path("/app") / baseline_rel_path + if image_ck.is_file(): + return image_ck.parent + _download_and_extract_zip(_DEFAULT_RAMP_BASELINE_URL, local_dir) + return local_dir + + +_QUBVEL_EFFICIENTNET_RELEASE = ( + "https://github.com/qubvel/efficientnet/releases/download/v0.0.1/" ) +def _patch_keras_get_file_for_efficientnet_weights() -> None: + """Redirect broken Callidior EfficientNet weight URLs to qubvel's GitHub release assets. + + The ``efficientnet`` package (via ``segmentation_models``) downloads encoder weights from + ``github.com/Callidior/keras-applications/releases/...``; those assets now return **404**. + qubvel hosts compatible ``*_imagenet_1000_notop.h5`` files on the same model's releases. + """ + import tensorflow as tf + + ku = tf.keras.utils + if getattr(ku.get_file, "_ramp_efficientnet_mirror", False): + return + + _orig = ku.get_file + + def _get_file(fname, origin, *args, **kwargs): + if isinstance(origin, str) and "Callidior" in origin and isinstance(fname, str): + m = re.match( + r"^(efficientnet-b\d+)_weights_tf_dim_ordering_tf_kernels_autoaugment_notop\.h5$", + fname, + ) + if m: + alt = f"{m.group(1)}_imagenet_1000_notop.h5" + origin = f"{_QUBVEL_EFFICIENTNET_RELEASE}{alt}" + kwargs = dict(kwargs) + kwargs["file_hash"] = None + return _orig(fname, origin, *args, **kwargs) + + _get_file._ramp_efficientnet_mirror = True # type: ignore[attr-defined] + ku.get_file = _get_file + + def resolve_model_href( model_uri: str, - cache_dir: Path | None = None, + cache_dir: Optional[Path] = None, ) -> str: """Resolve model_uri to a local SavedModel directory path. Supports: - - Local path: returned as-is if it exists and contains saved_model.pb - - Google Drive folder URL: downloaded via gdown, cached + - Local path: returned as-is if it exists - Direct HTTP(S) URL to .zip: downloaded, extracted, cached - - S3 URLs (s3://...): placeholder for future; raise if not implemented - Returns the absolute path to the SavedModel directory (contains saved_model.pb). + Returns the absolute path to the SavedModel directory. """ + if not isinstance(model_uri, str): + raise TypeError("model_uri must be a string") + if cache_dir is not None and not isinstance(cache_dir, Path): + raise TypeError("cache_dir must be a pathlib.Path or None") + cache_dir = cache_dir or _DEFAULT_MODEL_CACHE - path = Path(model_uri) - - # Local path: must exist and look like a SavedModel dir - if not ( - model_uri.startswith("http://") - or model_uri.startswith("https://") - or model_uri.startswith("s3://") - ): - resolved = path.resolve() - if resolved.is_dir() and (resolved / "saved_model.pb").is_file(): - return str(resolved) + + if not (model_uri.startswith("http://") or model_uri.startswith("https://")): + resolved = _to_local_path(model_uri, "model_uri").resolve() if resolved.exists(): return str(resolved) - raise FileNotFoundError(f"Model path not found or invalid: {resolved}") + raise FileNotFoundError(f"Model path not found: {resolved}") - # S3: future support (backend could download before calling) - if model_uri.startswith("s3://"): - raise NotImplementedError( - "S3 model URIs require backend pre-download or boto3 in container. " - "Use Google Drive or HTTP URL for now." - ) - - # Google Drive folder - folder_match = _GDRIVE_FOLDER_RE.search(model_uri) - if folder_match: - folder_id = folder_match.group(1) - dest_dir = cache_dir / f"gdrive_{folder_id}" - dest_dir.mkdir(parents=True, exist_ok=True) - - # Check if already downloaded - if (dest_dir / "saved_model.pb").is_file(): - return str(dest_dir) - - try: - import gdown - - gdown.download_folder( - id=folder_id, - output=str(dest_dir), - quiet=True, - ) - except ImportError as e: - raise ImportError( - "gdown is required to download models from Google Drive. " - "Add 'gdown' to the RAMP Dockerfile." - ) from e - - if not (dest_dir / "saved_model.pb").is_file(): - # gdown may create a subfolder with the Drive folder name - subdirs = [d for d in dest_dir.iterdir() if d.is_dir()] - if len(subdirs) == 1 and (subdirs[0] / "saved_model.pb").is_file(): - return str(subdirs[0]) - raise RuntimeError( - f"Downloaded folder {dest_dir} does not contain saved_model.pb. " - "Ensure the Drive folder contains the full Keras SavedModel (saved_model.pb + variables/)." - ) - return str(dest_dir) - - # Google Drive single file (less common for SavedModel) - file_match = _GDRIVE_FILE_RE.search(model_uri) - if file_match: - file_id = file_match.group(1) - dest_dir = cache_dir / f"gdrive_file_{file_id}" - dest_dir.mkdir(parents=True, exist_ok=True) - out_file = dest_dir / "downloaded" - try: - import gdown - - gdown.download(id=file_id, output=str(out_file), quiet=True) - except ImportError as e: - raise ImportError("gdown required for Google Drive downloads.") from e - if out_file.suffix == ".zip": - with zipfile.ZipFile(out_file, "r") as zf: - zf.extractall(dest_dir) - out_file.unlink() - saved_pb = dest_dir / "saved_model.pb" - if saved_pb.is_file(): - return str(dest_dir) - for sub in dest_dir.rglob("saved_model.pb"): - return str(sub.parent) - raise RuntimeError(f"Downloaded file did not yield a valid SavedModel in {dest_dir}") - - # Direct HTTP(S) URL to .zip if model_uri.lower().endswith(".zip"): base_name = Path(model_uri.split("/")[-1]).stem dest_dir = cache_dir / base_name dest_dir.mkdir(parents=True, exist_ok=True) - zip_path = cache_dir / (base_name + ".zip") if not any(dest_dir.rglob("saved_model.pb")): - urlretrieve(model_uri, zip_path) - with zipfile.ZipFile(zip_path, "r") as zf: - zf.extractall(dest_dir) - zip_path.unlink(missing_ok=True) + _download_and_extract_zip(model_uri, dest_dir) for sub in dest_dir.rglob("saved_model.pb"): return str(sub.parent) raise RuntimeError(f"Zip from {model_uri} did not contain a valid SavedModel") raise ValueError( - f"Unsupported model_uri format: {model_uri}. " - "Use: local path, Google Drive folder URL, or HTTP(S) URL to a .zip file." + f"Unsupported model_uri: {model_uri}. " + "Use a local path or an HTTP(S) URL to a .zip file containing the SavedModel." ) -# --------------------------------------------------------------------------- -# STAC MLM processing-expression callables -# --------------------------------------------------------------------------- - - def preprocess( input_path: str, output_path: str, @@ -202,88 +186,66 @@ def preprocess( return output_path -def postprocess(prediction_masks_dir: str, output_dir: str) -> str: - """Convert RAMP multichannel predicted masks to per-chip GeoJSON polygons. - - For each .pred.tif in prediction_masks_dir: - 1. Reads the 4-class sparse multimask (uint8, channels-first, 1 band). - 2. Extracts a binary building footprint mask (class == 1, ignores boundary/contact). - 3. Polygonizes via GDAL and writes a matching .geojson file. +def postprocess(prediction_masks_dir: str, output_dir: str) -> dict: + """Run fairpredictor-style postprocessing and return merged prediction GeoJSON content.""" + import json - Returns the output_dir path containing per-chip GeoJSONs. - """ - from osgeo import gdal + from geomltoolkits.regularizer import VectorizeMasks + from geomltoolkits.utils import merge_rasters, validate_polygon_geometries + from predictor.utils import morphological_cleaning - from ramp.utils.img_utils import gdal_get_mask_tensor - from ramp.utils.mask_to_vec_utils import ( - binary_mask_from_multichannel_mask, - binary_mask_to_geojson, - ) - - pred_dir = Path(prediction_masks_dir) - out_dir = Path(output_dir) + pred_dir = _to_local_path(prediction_masks_dir, "prediction_masks_dir") + out_dir = _to_local_path(output_dir, "output_dir") out_dir.mkdir(parents=True, exist_ok=True) - pred_tifs = sorted(pred_dir.glob("*.pred.tif")) + pred_tifs = sorted(pred_dir.glob("*.tif")) if not pred_tifs: - raise RuntimeError(f"No *.pred.tif files found in {pred_dir}") - - for mask_path in pred_tifs: - json_name = mask_path.stem.replace(".pred", "") + ".geojson" - json_path = str(out_dir / json_name) - - ref_ds = gdal.Open(str(mask_path)) - if ref_ds is None: - raise RuntimeError(f"GDAL could not open {mask_path}") - - multimask = gdal_get_mask_tensor(str(mask_path)) - bin_mask = binary_mask_from_multichannel_mask(multimask) - binary_mask_to_geojson(bin_mask, ref_ds, json_path) - - return output_dir - - -# --------------------------------------------------------------------------- -# Private helpers -# --------------------------------------------------------------------------- - - -def _make_val_split( - chips_dir: Path, - masks_dir: Path, - val_chips_dir: Path, - val_masks_dir: Path, - val_fraction: float = 0.15, -) -> None: - """Move val_fraction of chip+mask pairs to validation directories. - - RAMP mask filenames follow the convention .mask.tif so we - match chips to their masks by stem before moving. - """ - chip_files = sorted(chips_dir.glob("*.tif")) - if not chip_files: - raise RuntimeError(f"No .tif chips found in {chips_dir}") - - n_val = max(1, int(len(chip_files) * val_fraction)) - random.shuffle(chip_files) - val_chips = chip_files[:n_val] - - val_chips_dir.mkdir(parents=True, exist_ok=True) - val_masks_dir.mkdir(parents=True, exist_ok=True) - - moved = 0 - for chip_path in val_chips: - mask_path = masks_dir / (chip_path.stem + ".mask.tif") - if mask_path.is_file(): - shutil.move(str(chip_path), val_chips_dir / chip_path.name) - shutil.move(str(mask_path), val_masks_dir / mask_path.name) - moved += 1 + raise RuntimeError(f"No *.tif files found in {pred_dir}") + + merged_mask_path = out_dir / "merged_prediction_mask.tif" + merged_geojson_path = out_dir / "predictions.geojson" + tmp_dir = out_dir / "tmp" + tmp_dir.mkdir(parents=True, exist_ok=True) + + merge_rasters(str(pred_dir), str(merged_mask_path)) + morphological_cleaning(str(merged_mask_path)) + gdf = VectorizeMasks( + simplify_tolerance=0.5, + min_area=3, + orthogonalize=True, + tmp_dir=str(tmp_dir), + ortho_skew_tolerance_deg=15, + ortho_max_angle_change_deg=15, + ).convert(str(merged_mask_path), str(merged_geojson_path)) + + if gdf.crs and gdf.crs != "EPSG:4326": + gdf = gdf.to_crs("EPSG:4326") + elif not gdf.crs: + gdf.set_crs("EPSG:3857", inplace=True) + gdf = gdf.to_crs("EPSG:4326") + # Pandas extension dtypes break some Fiona GeoJSON writes; coerce to object. + import pandas as pd + + for col in gdf.columns: + if col == "geometry": + continue + if pd.api.types.is_extension_array_dtype(gdf[col].dtype): + gdf[col] = gdf[col].astype(object).where(gdf[col].notna(), None) + gdf.to_file(merged_geojson_path, driver="GeoJSON") + + validated_geojson = validate_polygon_geometries( + json.loads(gdf.to_json()), + output_path=str(merged_geojson_path), + ) + if isinstance(validated_geojson, str) and Path(validated_geojson).is_file(): + if Path(validated_geojson) != merged_geojson_path: + copy2(validated_geojson, merged_geojson_path) + with merged_geojson_path.open("r", encoding="utf-8") as file_handle: + return json.load(file_handle) - if moved == 0: - raise RuntimeError( - f"Val split produced 0 pairs. " - f"Check that chips in {chips_dir} match masks in {masks_dir}." - ) + if isinstance(validated_geojson, dict): + return validated_geojson + return json.loads(gdf.to_json()) def _load_hyperparams_from_stac(stac_item_path: str) -> dict: @@ -293,7 +255,7 @@ def _load_hyperparams_from_stac(stac_item_path: str) -> dict: """ import json - path = Path(stac_item_path) + path = _to_local_path(stac_item_path, "stac_item_path") if not path.is_absolute(): for base in (Path("/workspace"), Path.cwd()): candidate = base / path @@ -307,320 +269,279 @@ def _load_hyperparams_from_stac(stac_item_path: str) -> dict: return dict(item.get("properties", {}).get("mlm:hyperparameters", {})) -def _build_train_config( - chips_subdir: str, - masks_subdir: str, - val_chips_subdir: str, - val_masks_subdir: str, - checkpts_subdir: str, - hyperparams: dict, - timestamp: str, -) -> dict: - """Build a RAMP training config dict from STAC mlm:hyperparameters. - - hyperparams: from STAC Item properties.mlm:hyperparameters. - Path params (chips_subdir, etc.) are runtime-derived and passed separately. - """ - # Map STAC keys to RAMP config keys (STAC may use "epochs", RAMP uses "num_epochs") - num_epochs = hyperparams.get("num_epochs", hyperparams.get("epochs", 100)) - batch_size = hyperparams.get("batch_size", 16) - learning_rate = hyperparams.get("learning_rate", 3e-4) - early_stopping_patience = hyperparams.get("early_stopping_patience", 35) - backbone = hyperparams.get("backbone", "efficientnetb0") - input_img_shape = hyperparams.get("input_img_shape", [256, 256]) - output_img_shape = hyperparams.get("output_img_shape", [256, 256]) - use_aug = hyperparams.get("augmentation", hyperparams.get("use_aug", False)) - - return { - "experiment_name": "RAMP EffUnet multimask training", - "discard_experiment": False, - "logging": {"log_experiment": False}, - "datasets": { - "train_img_dir": chips_subdir, - "train_mask_dir": masks_subdir, - "val_img_dir": val_chips_subdir, - "val_mask_dir": val_masks_subdir, - }, - "num_classes": hyperparams.get("num_classes", 4), - "num_epochs": num_epochs, - "batch_size": batch_size, - "input_img_shape": input_img_shape, - "output_img_shape": output_img_shape, - "loss": { - "get_loss_fn_name": "get_sparse_categorical_crossentropy_fn", - "loss_fn_parms": {}, - }, - "metrics": { - "use_metrics": True, - "get_metrics_fn_names": ["get_sparse_categorical_accuracy_fn"], - "metrics_fn_parms": [{}], - }, - "optimizer": { - "get_optimizer_fn_name": "get_adam_optimizer", - "optimizer_fn_parms": {"learning_rate": learning_rate}, - }, - "model": { - "get_model_fn_name": "get_effunet_model", - "model_fn_parms": { - "backbone": backbone, - "classes": ["background", "building", "boundary", "contact"], - }, - }, - "saved_model": {"use_saved_model": False}, - "augmentation": {"use_aug": use_aug}, - "early_stopping": { - "use_early_stopping": True, - "early_stopping_parms": { - "monitor": "val_loss", - "min_delta": 0.005, - "patience": early_stopping_patience, - "verbose": 1, - "mode": "auto", - "restore_best_weights": True, - }, - }, - "cyclic_learning_scheduler": {"use_clr": False}, - "tensorboard": {"use_tb": False}, - "prediction_logging": {"use_prediction_logging": False}, - "model_checkpts": { - "use_model_checkpts": True, - "model_checkpts_dir": checkpts_subdir, - "get_model_checkpt_callback_fn_name": "get_model_checkpt_callback_fn", - "model_checkpt_callback_parms": {"mode": "max", "save_best_only": True}, - }, - "random_seed": 20220523, - "timestamp": timestamp, - } - - -# --------------------------------------------------------------------------- -# ZenML steps -# --------------------------------------------------------------------------- - - @step def run_preprocessing( input_path: str, output_path: str, boundary_width: int = 3, contact_spacing: int = 8, -) -> str: - """Georeference OAM chips and generate 4-class multimasks. Returns preprocessed dir.""" - return preprocess(input_path, output_path, boundary_width, contact_spacing) +) -> List[Tuple[Any, Any]]: + """Georeference OAM chips and return chip/mask data arrays.""" + import rasterio + + preprocessed = Path(preprocess(input_path, output_path, boundary_width, contact_spacing)) + chips_dir = preprocessed / "chips" + masks_dir = preprocessed / "multimasks" + data_loader: list[tuple[Any, Any]] = [] + for chip in sorted(chips_dir.glob("*.tif")): + mask = masks_dir / f"{chip.stem}.mask.tif" + if mask.is_file(): + with rasterio.open(chip) as chip_src: + chip_data = chip_src.read() + with rasterio.open(mask) as mask_src: + mask_data = mask_src.read() + data_loader.append((chip_data, mask_data)) + return data_loader @step def train_model( data_base_path: str, + data_loader: List[Tuple[Any, Any]], preprocessed_path: str, stac_item_path: str = "models/ramp/stac-item.json", - val_fraction: float | None = None, -) -> str: + val_fraction: Optional[float] = None, +) -> Any: + """ZenML step wrapper for RAMP training; returns the best Keras SavedModel checkpoint.""" + if not data_loader: + raise RuntimeError("Preprocessing returned an empty dataloader; no chip/mask pairs found.") + return train_ramp_model( + data_base_path=data_base_path, + preprocessed_path=preprocessed_path, + stac_item_path=stac_item_path, + val_fraction=val_fraction, + log_zenml_step_metadata=True, + ) + + +def train_ramp_model( + data_base_path: str, + preprocessed_path: str, + stac_item_path: str = "models/ramp/stac-item.json", + val_fraction: Annotated[Optional[float], "0.0 <= val_fraction <= 0.5"] = None, + num_epochs: Annotated[Optional[int], "1 <= num_epochs <= 20"] = None, + batch_size: Annotated[Optional[int], "1 <= batch_size <= 8"] = None, + backbone: Optional[str] = None, + early_stopping_patience: Annotated[Optional[int], "1 <= early_stopping_patience <= 20"] = None, + log_zenml_step_metadata: bool = False, +) -> Any: """Fine-tune EfficientNetB0 + U-Net on 4-class multimask chips. - Hyperparameters are loaded from STAC Item (properties.mlm:hyperparameters). - stac_item_path: path to STAC Item JSON (relative to /workspace or cwd, or absolute). - Default matches current flat layout; for FAIr 3.0 versioned layout use - e.g. "models/ramp/1/stac-item.json". - val_fraction: optional override for validation split (not in STAC by default). + Uses hot_fair_utilities.training.ramp for training orchestration. + RAMP_CONFIG is used as the base configuration; hyperparameters from the + STAC Item and keyword arguments selectively override the base. + + Val split is handled internally by split_training_2_validation: + preprocessed_path → ramp_training_work/ (train + val-chips + val-multimasks). + + Sets ``RAMP_HOME`` before importing training helpers: ``run_training`` caches + ``working_ramp_home`` at import time, so ``/app`` is used when the GHCR baseline exists there. - Returns the path to the best Keras SavedModel directory. - val_sparse_categorical_accuracy is logged as ZenML step metadata. + If the RAMP baseline exists under the hot-fair-utilities image (``/app/ramp-data/baseline/``) + it is used for fine-tuning. Otherwise weights are resolved under ``data_base_path`` or downloaded. + + Returns the best checkpoint as a loaded ``tf.keras.Model`` (SavedModel on disk is loaded with compile=False). """ import os - import segmentation_models as sm - - sm.set_framework("tf.keras") + # run_training.run_training sets ``working_ramp_home = os.environ["RAMP_HOME"]`` at *import* time. + # That value is used for ``Path(working_ramp_home) / saved_model_path`` when loading the baseline. + # Dataset paths in cfg are absolute (from manage_fine_tuning_config), so they still resolve correctly. + # Docker + GHCR base: baseline lives under /app; do not rely on /workspace/ramp-data (bind mount hides it). + resolved_base = str(Path(data_base_path).resolve()) + image_baseline_ck = Path("/app/ramp-data/baseline/checkpoint.tf") + if image_baseline_ck.is_file(): + os.environ["RAMP_HOME"] = "/app" + else: + os.environ["RAMP_HOME"] = resolved_base + + # segmentation_models configures efficientnet at import time via SM_FRAMEWORK. + os.environ.setdefault("SM_FRAMEWORK", "tf.keras") + import tensorflow as tf - from ramp.data_mgmt.data_generator import ( - test_batches_from_gtiff_dirs, - training_batches_from_gtiff_dirs, - ) - from ramp.training import ( - callback_constructors, - loss_constructors, - metric_constructors, - model_constructors, - optimizer_constructors, + _patch_keras_get_file_for_efficientnet_weights() + import segmentation_models as sm + from hot_fair_utilities.training.ramp.cleanup import extract_highest_accuracy_model + from hot_fair_utilities.training.ramp.config import RAMP_CONFIG + from hot_fair_utilities.training.ramp.prepare_data import split_training_2_validation + from hot_fair_utilities.training.ramp.run_training import ( + manage_fine_tuning_config, + run_main_train_code, ) - from ramp.utils.model_utils import get_best_model_value_and_epoch - os.environ["RAMP_HOME"] = data_base_path + sm.set_framework("tf.keras") + # Load STAC hyperparams and apply call-site overrides hyperparams = _load_hyperparams_from_stac(stac_item_path) if val_fraction is not None: + if not 0.0 <= val_fraction <= 0.5: + raise ValueError("val_fraction must be in [0.0, 0.5]") hyperparams["val_fraction"] = val_fraction - - val_fraction_val = hyperparams.get("val_fraction", 0.15) - batch_size = hyperparams.get("batch_size", 16) - num_epochs = hyperparams.get("num_epochs", hyperparams.get("epochs", 100)) - - # Paths under preprocessed_path ( = output_path/preprocessed ). Not persistent - # in the container unless output_path is on a mounted volume; per FAIr 3.0 - # the backend/ZenML persists outputs (e.g. best checkpoint) to S3 after the run. - pre_path = Path(preprocessed_path) - chips_dir = pre_path / "chips" - masks_dir = pre_path / "multimasks" - val_chips_dir = pre_path / "val_chips" - val_masks_dir = pre_path / "val_multimasks" - checkpts_dir = pre_path / "checkpoints" - - if not val_chips_dir.is_dir(): - _make_val_split(chips_dir, masks_dir, val_chips_dir, val_masks_dir, val_fraction_val) - - def _rel(d: Path) -> str: - return str(d.relative_to(data_base_path)) - - timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") - cfg = _build_train_config( - chips_subdir=_rel(chips_dir), - masks_subdir=_rel(masks_dir), - val_chips_subdir=_rel(val_chips_dir), - val_masks_subdir=_rel(val_masks_dir), - checkpts_subdir=_rel(checkpts_dir), - hyperparams=hyperparams, - timestamp=timestamp, - ) - - loss_fn = loss_constructors.get_sparse_categorical_crossentropy_fn(cfg) - optimizer = optimizer_constructors.get_adam_optimizer(cfg) - accuracy_metric = metric_constructors.get_sparse_categorical_accuracy_fn({}) - the_model = model_constructors.get_effunet_model(cfg) - the_model.compile(optimizer=optimizer, loss=loss_fn, metrics=[accuracy_metric]) - - n_train = len(list(chips_dir.glob("*.tif"))) - n_val = len(list(val_chips_dir.glob("*.tif"))) - steps_per_epoch = max(1, n_train // batch_size) - validation_steps = max(1, n_val // batch_size) - cfg["runtime"] = { - "n_training": n_train, - "n_val": n_val, - "steps_per_epoch": steps_per_epoch, - "validation_steps": validation_steps, - } - - img_shape = cfg["input_img_shape"] - mask_shape = cfg["output_img_shape"] - - train_batches = training_batches_from_gtiff_dirs( - chips_dir, masks_dir, batch_size, img_shape, mask_shape + if num_epochs is not None: + if not 1 <= num_epochs <= 20: + raise ValueError("num_epochs must be in [1, 20] for RAMP runtime limits") + hyperparams["num_epochs"] = num_epochs + if batch_size is not None: + if not 1 <= batch_size <= 8: + raise ValueError("batch_size must be in [1, 8] for RAMP runtime limits") + hyperparams["batch_size"] = batch_size + if backbone is not None: + hyperparams["backbone"] = backbone + if early_stopping_patience is not None: + if not 1 <= early_stopping_patience <= 20: + raise ValueError("early_stopping_patience must be in [1, 20]") + hyperparams["early_stopping_patience"] = early_stopping_patience + + # Resolve effective values from STAC overrides or RAMP_CONFIG defaults + eff_epochs = hyperparams.get("num_epochs", hyperparams.get("epochs", RAMP_CONFIG["num_epochs"])) + eff_batch = hyperparams.get("batch_size", RAMP_CONFIG["batch_size"]) + eff_backbone = hyperparams.get("backbone", RAMP_CONFIG["model"]["model_fn_parms"]["backbone"]) + eff_lr = hyperparams.get("learning_rate", RAMP_CONFIG["optimizer"]["optimizer_fn_parms"]["learning_rate"]) + eff_patience = hyperparams.get( + "early_stopping_patience", + RAMP_CONFIG["early_stopping"]["early_stopping_parms"]["patience"], ) - val_batches = test_batches_from_gtiff_dirs( - val_chips_dir, val_masks_dir, batch_size, img_shape, mask_shape + if not 1 <= int(eff_epochs) <= 20: + raise ValueError(f"Resolved num_epochs={eff_epochs} is outside [1, 20]") + if not 1 <= int(eff_batch) <= 8: + raise ValueError(f"Resolved batch_size={eff_batch} is outside [1, 8]") + if not 1 <= int(eff_patience) <= 20: + raise ValueError(f"Resolved early_stopping_patience={eff_patience} is outside [1, 20]") + + # Split preprocessed_path into train + val dirs under a dedicated work dir + # (split_training_2_validation requires src != dst) + ramp_train_dir_path = _to_local_path( + str(Path(data_base_path) / "ramp_training_work"), + "ramp_training_work", ) - - callbacks = [ - callback_constructors.get_early_stopping_callback_fn(cfg), - callback_constructors.get_model_checkpt_callback_fn(cfg), - ] - - history = the_model.fit( - train_batches, - epochs=num_epochs, - steps_per_epoch=steps_per_epoch, - validation_data=val_batches, - validation_steps=validation_steps, - callbacks=callbacks, + if ramp_train_dir_path.exists(): + rmtree(ramp_train_dir_path) + ramp_train_dir = str(ramp_train_dir_path) + split_training_2_validation( + str(_to_local_path(preprocessed_path, "preprocessed_path")), + ramp_train_dir, + multimasks=True, ) - best_epoch, best_val_acc = get_best_model_value_and_epoch(history) - log_metadata( - metadata={ - "best_val_sparse_categorical_accuracy": float(best_val_acc), - "best_epoch": int(best_epoch), - } - ) + # Build config from RAMP_CONFIG via the utilities helper, then apply overrides + cfg = manage_fine_tuning_config(ramp_train_dir, eff_epochs, eff_batch, freeze_layers=False, multimasks=True) + cfg["model"]["model_fn_parms"]["backbone"] = eff_backbone + cfg["optimizer"]["optimizer_fn_parms"]["learning_rate"] = eff_lr + cfg["early_stopping"]["early_stopping_parms"]["patience"] = eff_patience + + # Use baseline checkpoint for fine-tuning if present, else train from scratch + saved_rel = cfg["saved_model"]["saved_model_path"] + baseline_dir: Path + if cfg["saved_model"]["use_saved_model"]: + baseline_dir = _ensure_ramp_baseline( + data_base_path=data_base_path, + baseline_rel_path=saved_rel, + ) + ck = baseline_dir / Path(saved_rel).name + cfg["saved_model"]["use_saved_model"] = ck.is_file() or (baseline_dir / "saved_model.pb").exists() + + run_main_train_code(cfg) + final_accuracy, final_model_path = extract_highest_accuracy_model(ramp_train_dir) + + if log_zenml_step_metadata: + log_metadata( + metadata={ + "best_val_accuracy": float(final_accuracy), + "best_model_path": str(final_model_path), + } + ) - checkpts_ts_dir = checkpts_dir / timestamp - checkpoints = sorted(checkpts_ts_dir.glob("*.tf")) - if not checkpoints: - raise RuntimeError(f"No .tf checkpoint found in {checkpts_ts_dir}") - return str(checkpoints[-1]) + return tf.keras.models.load_model(str(final_model_path), compile=False) @step def run_inference( - model_uri: str, + model_uri: Union[str, Path, Any], input_path: str, prediction_path: str, - model_cache_dir: str | None = None, -) -> str: - """Run RAMP EfficientNetB0 U-Net inference on georeferenced chips. + output_dir: str, + model_cache_dir: Optional[str] = None, +) -> Dict[str, Any]: + """ZenML step wrapper for RAMP inference returning final GeoJSON content. - model_uri: From STAC Item (assets.model.href). Can be: - - Local path to SavedModel directory - - Google Drive folder URL - - HTTP(S) URL to .zip containing SavedModel - Resolves URLs to local path (downloads and caches if needed). + model_uri may be a STAC/local path, HTTP(S) .zip URL, or a ``tf.keras.Model`` from training. + """ + return infer_ramp_model( + model_uri=model_uri, + input_path=input_path, + prediction_path=prediction_path, + output_dir=output_dir, + model_cache_dir=model_cache_dir, + ) - Loads Keras SavedModel, runs prediction chip-by-chip, and writes - one .pred.tif (single-band uint8 sparse mask) per input chip. - Returns prediction_path containing the .pred.tif files. +def infer_ramp_model( + model_uri: Union[str, Path, Any], + input_path: str, + prediction_path: str, + output_dir: str, + model_cache_dir: Optional[str] = None, + max_chips: Optional[int] = None, +) -> Dict[str, Any]: + """Run fairpredictor inference and return final merged GeoJSON content. + + model_uri: local path, HTTP(S) URL to a .zip SavedModel, or a ``tf.keras.Model`` from ``train_ramp_model``. """ - import numpy as np - import rasterio as rio - import tensorflow as tf - from tqdm import tqdm + import tempfile - from ramp.data_mgmt.display_data import get_mask_from_prediction - from ramp.utils.file_utils import get_basename - from ramp.utils.img_utils import to_channels_first, to_channels_last + import tensorflow as tf + from predictor.prediction import run_prediction cache = Path(model_cache_dir) if model_cache_dir else None - model_dir = resolve_model_href(model_uri, cache_dir=cache) - model = tf.keras.models.load_model(model_dir, compile=False) - out_dir = Path(prediction_path) + if isinstance(model_uri, (str, Path)): + model_dir = resolve_model_href(str(model_uri), cache_dir=cache) + elif isinstance(model_uri, tf.keras.Model): + tmp = Path(tempfile.mkdtemp(prefix="ramp_infer_savedmodel_")) + model_uri.save(str(tmp)) + model_dir = str(tmp) + else: + raise TypeError( + "model_uri must be a str, pathlib.Path, or a tf.keras.Model from training (compile=False load)." + ) + + input_dir = _to_local_path(input_path, "input_path") + out_dir = _to_local_path(prediction_path, "prediction_path") out_dir.mkdir(parents=True, exist_ok=True) - chip_files = sorted(Path(input_path).glob("**/*.tif")) + chip_files = sorted(input_dir.glob("**/*.tif")) if not chip_files: - png_files = sorted(Path(input_path).glob("**/*.png")) - if png_files: - raise RuntimeError( - "No GeoTIFF chips (*.tif) found for inference. " - f"Found {len(png_files)} PNG(s) in {input_path}. " - "RAMP inference expects georeferenced RGB GeoTIFF chips (typically the output of " - "`preprocess(...)` under `/chips/`). " - "Run preprocessing and point `input_path` to the resulting `chips/` directory." - ) raise RuntimeError( - f"No GeoTIFF chips (*.tif) found in {input_path}. " - "RAMP inference expects georeferenced RGB GeoTIFF chips (typically under `/chips/`)." + f"No GeoTIFF chips (*.tif) found in {input_dir}. " + "RAMP inference expects georeferenced chips." ) - for chip_file in tqdm(chip_files, desc="RAMP inference"): - bname = get_basename(str(chip_file)) - mask_name = bname + ".pred.tif" - with rio.open(chip_file) as src: - dst_profile = src.profile.copy() - dst_profile["count"] = 1 - dst_profile["dtype"] = "uint8" - img = to_channels_last(src.read()).astype("float32") - max_val = float(img.max()) - if max_val > 0: - img = img / max_val - predicted = get_mask_from_prediction(model.predict(np.expand_dims(img, 0))) - predicted = np.squeeze(predicted, axis=0) - with rio.open(out_dir / mask_name, "w", **dst_profile) as dst: - dst.write(to_channels_first(predicted)) - - return prediction_path + run_input_dir = input_dir + if max_chips is not None and max_chips > 0: + subset_dir = out_dir / "subset_input" + if subset_dir.exists(): + rmtree(subset_dir) + subset_dir.mkdir(parents=True, exist_ok=True) + for chip_file in chip_files[:max_chips]: + copy2(chip_file, subset_dir / chip_file.name) + run_input_dir = subset_dir + + georef_dir = run_prediction( + checkpoint_path=model_dir, + input_path=str(run_input_dir), + prediction_path=str(out_dir), + confidence=0.5, + crs="3857", + ) + return postprocess(str(georef_dir), output_dir) @step def run_postprocessing( - prediction_path: str, + prediction_path: Union[Path, str], output_dir: str, -) -> str: - """Polygonize RAMP predicted masks into per-chip building GeoJSONs.""" - return postprocess(prediction_path, output_dir) - - -# --------------------------------------------------------------------------- -# ZenML pipelines -# --------------------------------------------------------------------------- +) -> Dict[str, Any]: + """Run fairpredictor-style postprocessing and return merged GeoJSON content.""" + return postprocess(str(prediction_path), output_dir) @pipeline @@ -632,15 +553,13 @@ def training_pipeline( """Full RAMP training: georeference + multimask → val split → EfficientNetB0 U-Net. Hyperparameters (backbone, epochs, batch_size, boundary_width, contact_spacing, etc.) - are loaded from the STAC Item at stac_item_path. Default path matches the current - flat layout (models/ramp/stac-item.json); for FAIr 3.0 versioned layout use - e.g. stac_item_path="models/ramp/1/stac-item.json". + are loaded from the STAC Item at stac_item_path. """ hyperparams = _load_hyperparams_from_stac(stac_item_path) boundary_width = hyperparams.get("boundary_width", 3) contact_spacing = hyperparams.get("contact_spacing", 8) - preprocessed_path = run_preprocessing( + data_loader = run_preprocessing( input_path=input_path, output_path=f"{output_path}/preprocessed", boundary_width=boundary_width, @@ -648,30 +567,29 @@ def training_pipeline( ) train_model( data_base_path=output_path, - preprocessed_path=preprocessed_path, + data_loader=data_loader, + preprocessed_path=f"{output_path}/preprocessed", stac_item_path=stac_item_path, ) @pipeline def inference_pipeline( - model_uri: str, + model_uri: Union[str, Path, Any], input_path: str, prediction_path: str, output_dir: str, - model_cache_dir: str | None = None, -) -> None: - """RAMP inference: load model (from STAC) → predict → polygonize to GeoJSONs. + model_cache_dir: Optional[str] = None, +) -> Dict[str, Any]: + """RAMP inference: load model → predict → postprocess → final GeoJSON. - model_uri: From STAC Item assets.model.href. Supports: - - Local path (e.g. /workspace/checkpoints/model.tf) - - Google Drive folder URL - - HTTP URL to .zip containing SavedModel + model_uri: local path, HTTP(S) URL to a .zip SavedModel, or a ``tf.keras.Model`` from training. """ - pred_path = run_inference( + final_geojson = run_inference( model_uri=model_uri, input_path=input_path, prediction_path=prediction_path, + output_dir=output_dir, model_cache_dir=model_cache_dir, ) - run_postprocessing(prediction_path=pred_path, output_dir=output_dir) + return final_geojson diff --git a/models/ramp/stac-item.json b/models/ramp/stac-item.json index c98f4ab6..987c70d1 100644 --- a/models/ramp/stac-item.json +++ b/models/ramp/stac-item.json @@ -30,12 +30,13 @@ "semantic-segmentation" ], "mlm:framework": "TensorFlow", - "mlm:framework_version": "2.9.3", - "mlm:pretrained": false, - "mlm:pretrained_source": null, + "mlm:framework_version": "2.15.1", + "mlm:pretrained": true, + "mlm:pretrained_source": "https://api-prod.fair.hotosm.org/api/v1/workspace/download/ramp/baseline.zip", "keywords": [ "building", - "semantic-segmentation" + "semantic-segmentation", + "polygon" ], "version": "1", "license": "Apache-2.0", @@ -61,8 +62,8 @@ }, "pipeline_output": { "type": "directory", - "description": "Directory of GeoJSON files with building polygons (one per chip)", - "expected_files": "*.geojson", + "description": "Directory containing merged prediction GeoJSON (`predictions.geojson`) built from fairpredictor-style postprocessing", + "expected_files": "predictions.geojson", "geometry_types": ["Polygon", "MultiPolygon"] }, "evaluation_metrics": [ @@ -148,8 +149,8 @@ "mlm:hyperparameters": { "backbone": "efficientnetb0", "num_classes": 4, - "epochs": 100, - "batch_size": 16, + "epochs": 20, + "batch_size": 8, "learning_rate": 0.0003, "loss": "sparse_categorical_crossentropy", "optimizer": "adam", @@ -158,13 +159,13 @@ "boundary_width": 3, "contact_spacing": 8, "val_fraction": 0.15, - "early_stopping_patience": 35, + "early_stopping_patience": 10, "augmentation": false } }, "assets": { "model": { - "href": "https://drive.google.com/drive/folders/1VHFPkAWVtI5UEyzbD6e1uZlqe80YtQpn", + "href": "https://api-prod.fair.hotosm.org/api/v1/workspace/download/ramp/baseline.zip", "type": "application/octet-stream; framework=tensorflow", "title": "RAMP EfficientNetB0 U-Net weights (Keras SavedModel)", "roles": ["mlm:model"], @@ -182,17 +183,17 @@ "roles": ["code"], "mlm:entrypoint": "models.ramp.pipeline:training_pipeline" }, - "training-runtime": { + "mlm:training": { "href": "ghcr.io/hotosm/fair-models/ramp:v1", "type": "application/vnd.oci.image.index.v1+json", "title": "Training Docker Image", - "roles": ["training-runtime", "runtime"] + "roles": ["mlm:training-runtime"] }, - "inference-runtime": { + "mlm:inference": { "href": "ghcr.io/hotosm/fair-models/ramp:v1", "type": "application/vnd.oci.image.index.v1+json", "title": "Inference Docker Image", - "roles": ["inference-runtime", "runtime"] + "roles": ["mlm:inference-runtime"] } }, "links": [] diff --git a/models/ramp/tests/inside_container_smoke_test.py b/models/ramp/tests/inside_container_smoke_test.py index d4270f57..f3097b38 100644 --- a/models/ramp/tests/inside_container_smoke_test.py +++ b/models/ramp/tests/inside_container_smoke_test.py @@ -2,11 +2,17 @@ Run this script INSIDE the container. It validates: 1) Critical imports (tensorflow, ramp, segmentation_models, osgeo.gdal, solaris) -2) hot-fair-utilities preprocessing (georeference + multimask generation) -3) Train/val split -4) Short training run (2 epochs, checkpoint creation) -5) Inference output generation (.pred.tif per chip) -6) Polygonization (per-chip GeoJSON from predicted masks) +2) ``pipeline.preprocess`` path (georeference + multimask generation) +3) ``pipeline.run_preprocessing`` contract: chip/mask arrays + on-disk counts +4) Training wrappers: ``pipeline.train_model`` and ``pipeline.train_ramp_model`` + both return loaded ``tf.keras.Model`` +5) ``pipeline.resolve_model_href`` for local SavedModel directories +6) ``pipeline.run_inference`` returns merged GeoJSON dict content +7) ``pipeline.run_postprocessing`` wrapper returns dict and writes output +8) Inference intermediate georeferenced rasters for debug visibility + +All training, inference, and postprocessing steps delegate to pipeline.py +helpers so the smoke test exercises the same code paths as production. Data layouts supported: - data/sample layout: --dataset-root /workspace/data/sample @@ -26,6 +32,7 @@ import os import shutil from pathlib import Path +from typing import Any def _assert(condition: bool, message: str) -> None: @@ -41,6 +48,18 @@ def _count_files(path: Path, pattern: str) -> int: return len(list(path.glob(pattern))) +def _load_ramp_pipeline_module(): + """Import models/ramp/pipeline.py directly so smoke tests reuse production helpers.""" + import sys + + module_dir = Path("/workspace/models/ramp") + if str(module_dir) not in sys.path: + sys.path.insert(0, str(module_dir)) + import pipeline as ramp_pipeline + + return ramp_pipeline + + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Run RAMP container smoke tests.") parser.add_argument( @@ -92,18 +111,25 @@ def _prepare_data_sample_layout(dataset_root: Path) -> Path: _assert(len(geojson_files) > 0, f"No .geojson files in {osm_dir}") import geopandas as gpd + import pandas as pd gdfs = [gpd.read_file(p) for p in geojson_files] if len(gdfs) == 1: merged = gdfs[0] else: - import pandas as pd - crs = gdfs[0].crs or "EPSG:4326" merged = gpd.GeoDataFrame( pd.concat([g.to_crs(crs) for g in gdfs], ignore_index=True), crs=crs, ) + # Fiona/GeoJSON writers cannot reliably handle pandas extension dtypes + # (for example string[python], Int64, boolean). Normalize to plain Python + # objects with None for missing values before writing. + for col in merged.columns: + if col == "geometry": + continue + if pd.api.types.is_extension_array_dtype(merged[col].dtype): + merged[col] = merged[col].astype(object).where(merged[col].notna(), None) labels_path = input_dir / "labels.geojson" merged.to_file(labels_path, driver="GeoJSON") print(f"PASS: prepared {len(tif_files)} PNG chip(s) + labels.geojson from data/sample") @@ -115,6 +141,9 @@ def main() -> None: args = parse_args() os.environ.setdefault("RAMP_HOME", "/workspace") + # segmentation_models picks the backend when the package is first imported. + # Must be tf.keras for TF 2.15+ (Keras 3); calling set_framework() after import is too late. + os.environ.setdefault("SM_FRAMEWORK", "tf.keras") dataset_root = Path(args.dataset_root).resolve() @@ -126,9 +155,6 @@ def main() -> None: preprocessed_dir = dataset_root / "preprocessed_test" chips_dir = preprocessed_dir / "chips" masks_dir = preprocessed_dir / "multimasks" - val_chips_dir = preprocessed_dir / "val_chips" - val_masks_dir = preprocessed_dir / "val_multimasks" - checkpts_dir = preprocessed_dir / "checkpoints" pred_input_dir = preprocessed_dir / "chips" pred_output_dir = dataset_root / "prediction_test" / "output" vectors_dir = dataset_root / "prediction_test" / "vectors" @@ -136,16 +162,16 @@ def main() -> None: # ------------------------------------------------------------------------- _stage("Test 1: Critical imports") - import segmentation_models as sm # noqa: F401 - import tensorflow as tf # noqa: F401 - from osgeo import gdal # noqa: F401 - import ramp # noqa: F401 import hot_fair_utilities # noqa: F401 + import ramp # noqa: F401 + import segmentation_models as sm + import solaris + import tensorflow as tf + from osgeo import gdal - sm.set_framework("tf.keras") + sm.set_framework("tf.keras") # redundant if SM_FRAMEWORK already set; keeps intent explicit print(f"PASS: tensorflow {tf.__version__} / segmentation_models / ramp / gdal imported") - - import solaris # noqa: F401 + print(f"PASS: GDAL runtime version {gdal.VersionInfo('--version')}") print(f"PASS: solaris {solaris.__version__} imported") # ------------------------------------------------------------------------- @@ -161,23 +187,19 @@ def main() -> None: _assert(n_png > 0, f"No PNG chips found in {input_dir}. Add OAM PNG chips to run this test.") print(f"PASS: found {n_png} PNG chip(s) + labels.geojson") + ramp_pipeline = _load_ramp_pipeline_module() + # ------------------------------------------------------------------------- - _stage("Test 3: Preprocessing (georeference + multimask generation)") + _stage("Test 3: Preprocessing via pipeline.preprocess + run_preprocessing contract") shutil.rmtree(preprocessed_dir, ignore_errors=True) - - from hot_fair_utilities import preprocess as _preprocess - - _preprocess( + out_path = ramp_pipeline.preprocess( input_path=str(input_dir), output_path=str(preprocessed_dir), - rasterize=True, - rasterize_options=["binary"], - georeference_images=True, - multimasks=True, - input_boundary_width=3, - input_contact_spacing=8, + boundary_width=3, + contact_spacing=8, ) + _assert(Path(out_path).resolve() == preprocessed_dir.resolve(), f"Unexpected preprocess output: {out_path}") n_chips = _count_files(chips_dir, "*.tif") n_masks = _count_files(masks_dir, "*.mask.tif") @@ -186,214 +208,111 @@ def main() -> None: _assert(n_chips == n_masks, f"Chip count ({n_chips}) != mask count ({n_masks})") print(f"PASS: preprocessing produced {n_chips} chip(s) + {n_masks} multimask(s)") - # ------------------------------------------------------------------------- - _stage("Test 4: Train / val split") - - import random - - chip_files = sorted(chips_dir.glob("*.tif")) - n_val = max(1, int(len(chip_files) * 0.2)) - random.shuffle(chip_files) - val_chip_files = chip_files[:n_val] - - val_chips_dir.mkdir(parents=True, exist_ok=True) - val_masks_dir.mkdir(parents=True, exist_ok=True) - - moved = 0 - for chip_path in val_chip_files: - mask_path = masks_dir / (chip_path.stem + ".mask.tif") - if mask_path.is_file(): - shutil.copy(str(chip_path), val_chips_dir / chip_path.name) - shutil.copy(str(mask_path), val_masks_dir / mask_path.name) - moved += 1 - - _assert(moved > 0, "Val split produced 0 pairs — check chip/mask naming") - print(f"PASS: val split created {moved} chip+mask pair(s)") - - # ------------------------------------------------------------------------- - _stage("Test 5: Training smoke run (2 epochs)") - - checkpts_dir.mkdir(parents=True, exist_ok=True) - - import datetime - - import segmentation_models as sm - - sm.set_framework("tf.keras") + # Use pipeline.run_preprocessing directly so this smoke test follows pipeline.py behavior. + import numpy as np - from ramp.data_mgmt.data_generator import ( - test_batches_from_gtiff_dirs, - training_batches_from_gtiff_dirs, - ) - from ramp.training import ( - callback_constructors, - loss_constructors, - metric_constructors, - optimizer_constructors, + data_loader_contract: list[tuple[Any, Any]] = ramp_pipeline.run_preprocessing( + input_path=str(input_dir), + output_path=str(preprocessed_dir), + boundary_width=3, + contact_spacing=8, ) - - timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") - cfg = { - "experiment_name": "smoke_test", - "num_classes": 4, - "num_epochs": args.epochs, - "batch_size": args.batch_size, - "input_img_shape": [256, 256], - "output_img_shape": [256, 256], - "loss": {"get_loss_fn_name": "get_sparse_categorical_crossentropy_fn", "loss_fn_parms": {}}, - "metrics": { - "use_metrics": True, - "get_metrics_fn_names": ["get_sparse_categorical_accuracy_fn"], - "metrics_fn_parms": [{}], - }, - "optimizer": { - "get_optimizer_fn_name": "get_adam_optimizer", - "optimizer_fn_parms": {"learning_rate": 3e-4}, - }, - "model": { - "get_model_fn_name": "get_effunet_model", - "model_fn_parms": { - "backbone": args.backbone, - "classes": ["background", "building", "boundary", "contact"], - }, - }, - "saved_model": {"use_saved_model": False}, - "augmentation": {"use_aug": False}, - "early_stopping": { - "use_early_stopping": True, - "early_stopping_parms": { - "monitor": "val_loss", - "min_delta": 0.005, - "patience": 2, - "verbose": 0, - "mode": "auto", - "restore_best_weights": True, - }, - }, - "cyclic_learning_scheduler": {"use_clr": False}, - "tensorboard": {"use_tb": False}, - "prediction_logging": {"use_prediction_logging": False}, - "model_checkpts": { - "use_model_checkpts": True, - "model_checkpts_dir": str(checkpts_dir), - "get_model_checkpt_callback_fn_name": "get_model_checkpt_callback_fn", - "model_checkpt_callback_parms": {"mode": "max", "save_best_only": True}, - }, - "random_seed": 42, - "timestamp": timestamp, - } - - loss_fn = loss_constructors.get_sparse_categorical_crossentropy_fn(cfg) - optimizer = optimizer_constructors.get_adam_optimizer(cfg) - acc_metric = metric_constructors.get_sparse_categorical_accuracy_fn({}) - # NOTE: For the container smoke test we intentionally disable any pretrained - # EfficientNet weight downloads (the old Keras Applications URL can 404). - # This keeps the test self-contained while still validating the end-to-end - # training + checkpoint + inference + polygonization flow. - the_model = sm.Unet( - backbone_name=args.backbone, - encoder_weights=None, - classes=4, - activation="softmax", + _assert(len(data_loader_contract) > 0, "Expected at least one chip/mask array pair.") + _assert( + isinstance(data_loader_contract[0][0], np.ndarray), + "Chip data must be a numpy ndarray (pipeline.run_preprocessing contract).", ) - the_model.compile(optimizer=optimizer, loss=loss_fn, metrics=[acc_metric]) - - n_train = _count_files(chips_dir, "*.tif") - n_val_c = _count_files(val_chips_dir, "*.tif") - steps_per_epoch = max(1, n_train // args.batch_size) - validation_steps = max(1, n_val_c // args.batch_size) - cfg["runtime"] = { - "n_training": n_train, - "n_val": n_val_c, - "steps_per_epoch": steps_per_epoch, - "validation_steps": validation_steps, - } - - train_batches = training_batches_from_gtiff_dirs( - chips_dir, masks_dir, args.batch_size, [256, 256], [256, 256] + _assert( + isinstance(data_loader_contract[0][1], np.ndarray), + "Mask data must be a numpy ndarray (pipeline.run_preprocessing contract).", ) - val_batches = test_batches_from_gtiff_dirs( - val_chips_dir, val_masks_dir, args.batch_size, [256, 256], [256, 256] + print( + f"PASS: dataloader contract — {len(data_loader_contract)} (chip, mask) ndarray pair(s), " + "matching pipeline.run_preprocessing" ) - callbacks = [callback_constructors.get_early_stopping_callback_fn(cfg)] - - the_model.fit( - train_batches, - epochs=args.epochs, - steps_per_epoch=steps_per_epoch, - validation_data=val_batches, - validation_steps=validation_steps, - callbacks=callbacks, + # ------------------------------------------------------------------------- + _stage("Test 4: Training smoke run via train_model/train_ramp_model wrappers") + # train_model wraps train_ramp_model and validates preprocessing output. + # train_ramp_model handles val split internally and returns loaded tf.keras.Model. + + import tensorflow as tf + + trained_model_step = ramp_pipeline.train_model( + data_base_path=str(dataset_root), + data_loader=data_loader_contract, + preprocessed_path=str(preprocessed_dir), + stac_item_path="/workspace/models/ramp/stac-item.json", + val_fraction=0.15, ) - - model_save_path = str(checkpts_dir / f"smoke_{timestamp}.tf") - the_model.save(model_save_path) - _assert(Path(model_save_path).is_dir(), f"Saved model not found at {model_save_path}") - print(f"PASS: 2-epoch training completed; model saved to {model_save_path}") + _assert(isinstance(trained_model_step, tf.keras.Model), "train_model did not return a Keras model checkpoint.") + + trained_model = ramp_pipeline.train_ramp_model( + data_base_path=str(dataset_root), + preprocessed_path=str(preprocessed_dir), + stac_item_path="/workspace/models/ramp/stac-item.json", + num_epochs=args.epochs, + batch_size=args.batch_size, + backbone=args.backbone, + early_stopping_patience=2, + log_zenml_step_metadata=False, + ) + _assert(isinstance(trained_model, tf.keras.Model), "train_ramp_model did not return a Keras model checkpoint.") + print(f"PASS: {args.epochs}-epoch training completed; train wrappers returned tf.keras.Model") # ------------------------------------------------------------------------- - _stage("Test 6: Inference smoke run") + _stage("Test 5: resolve_model_href local SavedModel resolution") - import numpy as np - import rasterio as rio + local_saved_model_dir = dataset_root / "local_smoke_savedmodel" + shutil.rmtree(local_saved_model_dir, ignore_errors=True) + trained_model.save(str(local_saved_model_dir)) + resolved_model_dir = ramp_pipeline.resolve_model_href(str(local_saved_model_dir)) + _assert( + (Path(resolved_model_dir) / "saved_model.pb").is_file(), + f"resolve_model_href did not return a SavedModel dir: {resolved_model_dir}", + ) + print("PASS: resolve_model_href resolved a valid local SavedModel directory") - from ramp.data_mgmt.display_data import get_mask_from_prediction - from ramp.utils.file_utils import get_basename - from ramp.utils.img_utils import to_channels_first, to_channels_last + # ------------------------------------------------------------------------- + _stage("Test 6: Inference smoke run via run_inference") - pred_output_dir.mkdir(parents=True, exist_ok=True) shutil.rmtree(pred_output_dir, ignore_errors=True) pred_output_dir.mkdir(parents=True, exist_ok=True) - - inference_model = tf.keras.models.load_model(model_save_path, compile=False) - - chip_files = sorted(pred_input_dir.glob("*.tif")) - for chip_file in chip_files[:3]: - bname = get_basename(str(chip_file)) - mask_name = bname + ".pred.tif" - with rio.open(chip_file) as src: - dst_profile = src.profile.copy() - dst_profile["count"] = 1 - dst_profile["dtype"] = "uint8" - img = to_channels_last(src.read()).astype("float32") - max_val = float(img.max()) - if max_val > 0: - img = img / max_val - predicted = get_mask_from_prediction(inference_model.predict(np.expand_dims(img, 0))) - predicted = np.squeeze(predicted, axis=0) - with rio.open(pred_output_dir / mask_name, "w", **dst_profile) as dst: - dst.write(to_channels_first(predicted)) - - n_pred = _count_files(pred_output_dir, "*.pred.tif") - _assert(n_pred > 0, f"No .pred.tif files produced in {pred_output_dir}") - print(f"PASS: inference produced {n_pred} .pred.tif file(s)") + final_geojson = ramp_pipeline.run_inference( + model_uri=resolved_model_dir, + input_path=str(pred_input_dir), + prediction_path=str(pred_output_dir), + output_dir=str(vectors_dir), + ) + _assert(isinstance(final_geojson, dict), "Inference did not return GeoJSON dict content.") + _assert(final_geojson.get("type") == "FeatureCollection", "GeoJSON response missing FeatureCollection type.") + print(f"PASS: run_inference returned GeoJSON content with {len(final_geojson.get('features', []))} feature(s)") # ------------------------------------------------------------------------- - _stage("Test 7: Polygonization") - - from osgeo import gdal - - from ramp.utils.img_utils import gdal_get_mask_tensor - from ramp.utils.mask_to_vec_utils import ( - binary_mask_from_multichannel_mask, - binary_mask_to_geojson, + _stage("Test 7: Postprocessing via run_postprocessing wrapper") + + georef_dir = pred_output_dir / "georeference" + postprocess_out_dir = dataset_root / "prediction_test" / "vectors_postprocess_wrapper" + shutil.rmtree(postprocess_out_dir, ignore_errors=True) + postprocess_out_dir.mkdir(parents=True, exist_ok=True) + wrapped_geojson = ramp_pipeline.run_postprocessing( + prediction_path=str(georef_dir), + output_dir=str(postprocess_out_dir), ) + _assert(isinstance(wrapped_geojson, dict), "run_postprocessing did not return GeoJSON dict content.") + _assert( + (postprocess_out_dir / "predictions.geojson").is_file(), + "run_postprocessing did not write predictions.geojson", + ) + print("PASS: run_postprocessing returned dict and wrote predictions.geojson") - vectors_dir.mkdir(parents=True, exist_ok=True) - - for pred_tif in sorted(pred_output_dir.glob("*.pred.tif")): - json_name = pred_tif.stem.replace(".pred", "") + ".geojson" - json_path = str(vectors_dir / json_name) - ref_ds = gdal.Open(str(pred_tif)) - _assert(ref_ds is not None, f"GDAL could not open {pred_tif}") - multimask = gdal_get_mask_tensor(str(pred_tif)) - bin_mask = binary_mask_from_multichannel_mask(multimask) - binary_mask_to_geojson(bin_mask, ref_ds, json_path) + # ------------------------------------------------------------------------- + _stage("Test 8: Inference intermediate artifacts") - n_geojson = _count_files(vectors_dir, "*.geojson") - _assert(n_geojson > 0, f"No GeoJSON files produced in {vectors_dir}") - print(f"PASS: polygonization produced {n_geojson} GeoJSON file(s)") + _assert(georef_dir.is_dir(), f"Georeferenced output directory not found: {georef_dir}") + n_pred = _count_files(georef_dir, "*.tif") + _assert(n_pred > 0, f"No georeferenced prediction .tif files produced in {georef_dir}") + print(f"PASS: fairpredictor produced {n_pred} georeferenced prediction .tif file(s)") # ------------------------------------------------------------------------- _stage("ALL TESTS PASSED") diff --git a/models/ramp/tests/run_docker_tests.ps1 b/models/ramp/tests/run_docker_tests.ps1 index 25189917..7f08a78c 100644 --- a/models/ramp/tests/run_docker_tests.ps1 +++ b/models/ramp/tests/run_docker_tests.ps1 @@ -1,34 +1,77 @@ +# RAMP Docker smoke test — PowerShell (Windows). +# Same behavior as run_docker_tests.sh. +# +# Base images are pulled from GHCR (no local build needed): +# CPU: ghcr.io/hotosm/fair-utilities-ramp:cpu-latest (default) +# GPU: ghcr.io/hotosm/fair-utilities-ramp:gpu-latest +# +# Bash syntax "VAR=1 ./script.sh" does not work in PowerShell. Use either: +# - This script, or +# - $env:BUILD_IMAGE = "1"; $env:CPU_ONLY = "1"; .\models\ramp\tests\run_docker_tests.ps1 +# Optional smoke args: +# - $env:SMOKE_DATASET_ROOT (default: /workspace/data/sample) +# - $env:SMOKE_EPOCHS (default: 2) +# - $env:SMOKE_BATCH_SIZE (default: 4) +# - $env:SMOKE_BACKBONE (default: efficientnetb0) +# +# Examples (run from fAIr-models repo root): +# .\models\ramp\tests\run_docker_tests.ps1 +# $env:BUILD_IMAGE = "1"; $env:CPU_ONLY = "1"; .\models\ramp\tests\run_docker_tests.ps1 +# .\models\ramp\tests\run_docker_tests.ps1 -RepoRoot "E:\path\to\fAIr-models" -Image "ramp-v1:cpu" + +#Requires -Version 5.1 param( - [string]$RepoRoot = "E:\On Going Projects\Work Growth\HOTOSM\fAIr_Repos\fAIr-models", - [string]$Image = "ramp-v1:gpu", - [switch]$BuildImage, - [switch]$CpuOnly + [string] $RepoRoot = (Get-Location).Path, + [string] $Image = "" ) $ErrorActionPreference = "Stop" -if ($BuildImage) { - $buildArg = "" - if ($CpuOnly) { - $buildArg = "--build-arg BUILD_TYPE=cpu" +function Invoke-NativeOrThrow { + param([scriptblock] $Command) + & $Command + if ($LASTEXITCODE -ne 0) { + throw "Command failed with exit code $LASTEXITCODE." + } +} + +$buildImage = if ($null -ne $env:BUILD_IMAGE) { $env:BUILD_IMAGE } else { "0" } +$cpuOnly = if ($null -ne $env:CPU_ONLY) { $env:CPU_ONLY } else { "0" } +$smokeDatasetRoot = if ($null -ne $env:SMOKE_DATASET_ROOT) { $env:SMOKE_DATASET_ROOT } else { "/workspace/data/sample" } +$smokeEpochs = if ($null -ne $env:SMOKE_EPOCHS) { $env:SMOKE_EPOCHS } else { "2" } +$smokeBatchSize = if ($null -ne $env:SMOKE_BATCH_SIZE) { $env:SMOKE_BATCH_SIZE } else { "4" } +$smokeBackbone = if ($null -ne $env:SMOKE_BACKBONE) { $env:SMOKE_BACKBONE } else { "efficientnetb0" } + +if (-not $Image) { + if ($cpuOnly -eq "1") { $Image = "ramp-v1:cpu" - } else { - $buildArg = "--build-arg BUILD_TYPE=gpu" } - Write-Host "Building image $Image ..." - Invoke-Expression "docker build $buildArg -t $Image -f `"$RepoRoot\models\ramp\Dockerfile`" `"$RepoRoot`"" + else { + $Image = "ramp-v1:gpu" + } } -$gpuArgs = @() -if (-not $CpuOnly) { - $gpuArgs = @("--gpus", "all") +if ($buildImage -eq "1") { + if ($cpuOnly -eq "1") { + $Image = "ramp-v1:cpu" + Write-Host "Building image $Image (base: ghcr.io/hotosm/fair-utilities-ramp:cpu-latest) ..." + # No --build-arg needed — the Dockerfile default already points at the GHCR CPU image. + Invoke-NativeOrThrow { docker build -t $Image -f "$RepoRoot/models/ramp/Dockerfile" $RepoRoot } + } + else { + $Image = "ramp-v1:gpu" + Write-Host "Building image $Image (base: ghcr.io/hotosm/fair-utilities-ramp:gpu-latest) ..." + Invoke-NativeOrThrow { docker build --build-arg BASE_IMAGE=ghcr.io/hotosm/fair-utilities-ramp:gpu-latest -t $Image -f "$RepoRoot/models/ramp/Dockerfile" $RepoRoot } + } } -Write-Host "Running container smoke tests..." -docker run --rm @gpuArgs ` - -v "${RepoRoot}:/workspace" ` - $Image ` - python /workspace/models/ramp/tests/inside_container_smoke_test.py ` - --dataset-root /workspace/data/sample +if ($cpuOnly -ne "1") { + Write-Host "Running container smoke tests..." + Invoke-NativeOrThrow { docker run --rm --gpus all -v "${RepoRoot}:/workspace" $Image python /workspace/models/ramp/tests/inside_container_smoke_test.py --dataset-root $smokeDatasetRoot --epochs $smokeEpochs --batch-size $smokeBatchSize --backbone $smokeBackbone } +} +else { + Write-Host "Running container smoke tests..." + Invoke-NativeOrThrow { docker run --rm -v "${RepoRoot}:/workspace" $Image python /workspace/models/ramp/tests/inside_container_smoke_test.py --dataset-root $smokeDatasetRoot --epochs $smokeEpochs --batch-size $smokeBatchSize --backbone $smokeBackbone } +} Write-Host "Done." diff --git a/models/ramp/tests/run_docker_tests.sh b/models/ramp/tests/run_docker_tests.sh index b08498a9..185e30c6 100644 --- a/models/ramp/tests/run_docker_tests.sh +++ b/models/ramp/tests/run_docker_tests.sh @@ -1,4 +1,11 @@ #!/usr/bin/env bash +# Run RAMP in-container smoke tests. Default: /workspace/data/sample, script defaults for epochs/batch. +# Base images come from GHCR (no local build of fAIr-utilities needed): +# CPU: ghcr.io/hotosm/fair-utilities-ramp:cpu-latest (default) +# GPU: ghcr.io/hotosm/fair-utilities-ramp:gpu-latest +# Windows PowerShell: use run_docker_tests.ps1 (Bash-style VAR=1 ./script.sh does not work in PowerShell). +# To pass --epochs / --batch-size / --backbone / a custom --dataset-root, use docker run manually +# (see research/fAIr_3.0/ramp_model/03_ramp_test_explained.md). set -euo pipefail REPO_ROOT="${1:-$(pwd)}" @@ -7,15 +14,16 @@ BUILD_IMAGE="${BUILD_IMAGE:-0}" CPU_ONLY="${CPU_ONLY:-0}" if [[ "$BUILD_IMAGE" == "1" ]]; then - BUILD_ARGS="" if [[ "$CPU_ONLY" == "1" ]]; then - BUILD_ARGS="--build-arg BUILD_TYPE=cpu" IMAGE="ramp-v1:cpu" + # No --build-arg needed; Dockerfile default is ghcr.io/hotosm/fair-utilities-ramp:cpu-latest + echo "Building image $IMAGE (base: ghcr.io/hotosm/fair-utilities-ramp:cpu-latest) ..." + docker build -t "$IMAGE" -f "$REPO_ROOT/models/ramp/Dockerfile" "$REPO_ROOT" else - BUILD_ARGS="--build-arg BUILD_TYPE=gpu" + echo "Building image $IMAGE (base: ghcr.io/hotosm/fair-utilities-ramp:gpu-latest) ..." + docker build --build-arg BASE_IMAGE=ghcr.io/hotosm/fair-utilities-ramp:gpu-latest \ + -t "$IMAGE" -f "$REPO_ROOT/models/ramp/Dockerfile" "$REPO_ROOT" fi - echo "Building image $IMAGE ..." - docker build $BUILD_ARGS -t "$IMAGE" -f "$REPO_ROOT/models/ramp/Dockerfile" "$REPO_ROOT" fi GPU_ARGS=() diff --git a/models/ramp/tests/test_plan.yaml b/models/ramp/tests/test_plan.yaml deleted file mode 100644 index b7bea389..00000000 --- a/models/ramp/tests/test_plan.yaml +++ /dev/null @@ -1,58 +0,0 @@ -# RAMP smoke tests use data/sample (train/oam + train/osm). -# The test auto-prepares input from data/sample layout to RAMP format (PNG + labels.geojson). - -tests: - - id: import_smoke - description: Verify all critical runtime packages import correctly inside container - checks: - - import tensorflow - - import segmentation_models (set_framework tf.keras) - - import ramp - - import hot_fair_utilities - - import osgeo.gdal - - import solaris - - - id: dataset_layout - description: Verify required input folder and files are present (or data/sample train/oam + train/osm) - required_paths: - - "train/oam (data/sample) or input/" - - "train/osm (data/sample) or input/labels.geojson" - required_patterns: - - "train/oam/OAM-*.tif (data/sample) or input/*.png" - - - id: preprocess - description: Run hot_fair_utilities.preprocess with multimasks=True; validate output structure - expected_outputs: - - preprocessed_test/chips/*.tif - - preprocessed_test/multimasks/*.mask.tif - invariants: - - chip_count == mask_count - - - id: val_split - description: Shuffle and move a fraction of chip+mask pairs to val directories - expected_outputs: - - preprocessed_test/val_chips/*.tif - - preprocessed_test/val_multimasks/*.mask.tif - invariants: - - at_least_one_pair_moved - - - id: train_smoke - description: > - Compile EfficientNetB0 U-Net, run 2-epoch training loop, save Keras SavedModel. - Validates that training does not crash and a SavedModel directory is created. - expected_outputs: - - preprocessed_test/checkpoints/smoke_.tf/ - - - id: inference_smoke - description: > - Load the SavedModel checkpoint and run chip-by-chip inference. - Each chip produces a single-band uint8 .pred.tif with 4-class values. - expected_outputs: - - prediction_test/output/*.pred.tif - - - id: polygonize_smoke - description: > - Use GDAL Polygonize via ramp.utils.mask_to_vec_utils to convert - predicted multimasks into per-chip GeoJSON building polygons. - expected_outputs: - - prediction_test/vectors/*.geojson diff --git a/pyproject.toml b/pyproject.toml index 109273a2..102f27a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,6 +81,12 @@ line-length = 120 [tool.ruff.lint] select = ["E", "F", "I", "UP", "B", "SIM", "RUF"] +# models/ramp/pipeline.py: ZenML @step / @pipeline signatures must keep typing.Union, +# Optional, List, Tuple, Dict (not PEP 604 / builtin generics). + +[tool.ruff.lint.per-file-ignores] +"models/ramp/pipeline.py" = ["UP006", "UP007", "UP035", "UP045"] + [tool.pytest.ini_options] testpaths = ["tests"] From c01993e4d3bf9dd52e43920965d8792ba4914cf6 Mon Sep 17 00:00:00 2001 From: Abdelrahman Katkat Date: Tue, 14 Apr 2026 10:57:05 +0200 Subject: [PATCH 04/30] feat(ramp): enhance pipeline with input resolution and model loading improvements - Added functions to resolve local and remote input directories and files, improving flexibility in handling model paths. - Implemented a zip extraction utility for loading models from compressed files. - Updated `resolve_model_href` to support both local and remote SavedModel directories, enhancing compatibility with various model formats. - Modified smoke tests to validate the new `split_dataset` functionality alongside training wrappers, ensuring robust model training and validation. --- .gitignore | 10 +- models/ramp/pipeline.py | 243 +++++++++++++++--- .../ramp/tests/inside_container_smoke_test.py | 20 +- 3 files changed, 231 insertions(+), 42 deletions(-) diff --git a/.gitignore b/.gitignore index 7ee5ae04..ccf237b5 100644 --- a/.gitignore +++ b/.gitignore @@ -221,7 +221,15 @@ implementation.md # data data/sample/predict/predictions data/sample/ramp_work +data/sample/yolo_work stac_catalog # hatch-vcs generated version file -fair/_version.py \ No newline at end of file +fair/_version.py + + +# runs +runs/ + +# weights +yolov8s_v2-seg.pt diff --git a/models/ramp/pipeline.py b/models/ramp/pipeline.py index c8a260f4..c175acd7 100644 --- a/models/ramp/pipeline.py +++ b/models/ramp/pipeline.py @@ -61,6 +61,31 @@ def _to_local_path(path_value: str, purpose: str) -> Path: return Path(str(upath_obj)) +def _resolve_input_directory(path_value: str, purpose: str) -> Path: + """Resolve local or remote (S3/HTTP/etc.) directories to a local path.""" + if "://" in str(path_value): + # Available in the runtime (same helper used by YOLO model packs). + from fair.utils.data import resolve_directory + + return Path(str(resolve_directory(path_value, pattern="*"))) + return _to_local_path(path_value, purpose) + + +def _resolve_input_file(path_value: str, purpose: str) -> Path: + """Resolve local or remote (S3/HTTP/etc.) files to a local path.""" + if "://" in str(path_value): + from fair.utils.data import resolve_path + + return Path(str(resolve_path(path_value))) + return _to_local_path(path_value, purpose) + + +def _extract_zip(zip_path: Path, dest_dir: Path) -> None: + dest_dir.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(zip_path, "r") as zf: + zf.extractall(dest_dir) + + def _ensure_ramp_baseline(data_base_path: str, baseline_rel_path: str) -> Path: """Return the directory that contains baseline weights; download under data_base_path if missing. @@ -115,6 +140,46 @@ def _get_file(fname, origin, *args, **kwargs): ku.get_file = _get_file +def _patch_predictor_savedmodel_loader_for_tf215() -> None: + """Patch fairpredictor's SavedModel directory loader for TF 2.15 compatibility. + + Why this patch exists: + Some fairpredictor versions load SavedModel directories using `keras.layers.TFSMLayer(...)`. + `TFSMLayer` is not available in TensorFlow 2.15's `tf.keras.layers`, so inference fails with: + AttributeError: module 'keras.api._v2.keras.layers' has no attribute 'TFSMLayer' + + We monkey-patch `predictor.prediction._load_keras_model` so directory-based SavedModels use + `tf.saved_model.load(...)` and a small `.predict(...)` wrapper. + + We patch the module object (via importlib) to ensure downstream `from predictor.prediction import ...` + uses the patched function. + """ + import importlib + import os + from pathlib import Path + + import tensorflow as tf + + pred = importlib.import_module("predictor.prediction") + if getattr(pred, "_fair_models_tf215_savedmodel_patch_applied", False): + return + + original_loader = getattr(pred, "_load_keras_model", None) + if original_loader is None: + raise RuntimeError("predictor.prediction._load_keras_model not found; fairpredictor API changed.") + + def _safe_load_keras_model(keras_backend, path: str): + if os.path.isdir(path) and (Path(path) / "saved_model.pb").exists(): + # TF 2.15 (Keras 2.x) can load SavedModel directories directly; avoid TFSMLayer entirely. + return tf.keras.models.load_model(path, compile=False) + + return original_loader(keras_backend, path) + + _safe_load_keras_model._fair_models_tf215_savedmodel_loader = True # type: ignore[attr-defined] + pred._load_keras_model = _safe_load_keras_model + pred._fair_models_tf215_savedmodel_patch_applied = True + + def resolve_model_href( model_uri: str, cache_dir: Optional[Path] = None, @@ -134,22 +199,43 @@ def resolve_model_href( cache_dir = cache_dir or _DEFAULT_MODEL_CACHE - if not (model_uri.startswith("http://") or model_uri.startswith("https://")): - resolved = _to_local_path(model_uri, "model_uri").resolve() - if resolved.exists(): - return str(resolved) - raise FileNotFoundError(f"Model path not found: {resolved}") + is_http = model_uri.startswith("http://") or model_uri.startswith("https://") + + # SavedModel directory (local or remote) + if not Path(model_uri.split("?", 1)[0]).suffix and not model_uri.lower().endswith(".zip"): + resolved_dir = (_resolve_input_directory(model_uri, "model_uri") if "://" in model_uri else _to_local_path(model_uri, "model_uri")).resolve() + if (resolved_dir / "saved_model.pb").exists(): + return str(resolved_dir) + if resolved_dir.exists(): + # Let downstream error messages be explicit. + raise FileNotFoundError(f"SavedModel directory missing saved_model.pb: {resolved_dir}") + raise FileNotFoundError(f"Model path not found: {resolved_dir}") + # Remote/local ZIP containing SavedModel if model_uri.lower().endswith(".zip"): - base_name = Path(model_uri.split("/")[-1]).stem - dest_dir = cache_dir / base_name - dest_dir.mkdir(parents=True, exist_ok=True) - if not any(dest_dir.rglob("saved_model.pb")): + base_name = Path(model_uri.split("?", 1)[0]).name + stem = Path(base_name).stem or "ramp_model" + dest_dir = cache_dir / stem + if any(dest_dir.rglob("saved_model.pb")): + for sub in dest_dir.rglob("saved_model.pb"): + return str(sub.parent) + + if is_http: _download_and_extract_zip(model_uri, dest_dir) + else: + local_zip = _resolve_input_file(model_uri, "model_uri") + _extract_zip(local_zip, dest_dir) + for sub in dest_dir.rglob("saved_model.pb"): return str(sub.parent) raise RuntimeError(f"Zip from {model_uri} did not contain a valid SavedModel") + # Plain local path to file/dir (fallback) + resolved = _to_local_path(model_uri, "model_uri").resolve() + if resolved.exists(): + return str(resolved) + raise FileNotFoundError(f"Model path not found: {resolved}") + raise ValueError( f"Unsupported model_uri: {model_uri}. " "Use a local path or an HTTP(S) URL to a .zip file containing the SavedModel." @@ -173,8 +259,9 @@ def preprocess( """ from hot_fair_utilities import preprocess as _preprocess + local_input = _resolve_input_directory(input_path, "input_path") _preprocess( - input_path=input_path, + input_path=str(local_input), output_path=output_path, rasterize=True, rasterize_options=["binary"], @@ -190,9 +277,11 @@ def postprocess(prediction_masks_dir: str, output_dir: str) -> dict: """Run fairpredictor-style postprocessing and return merged prediction GeoJSON content.""" import json - from geomltoolkits.regularizer import VectorizeMasks - from geomltoolkits.utils import merge_rasters, validate_polygon_geometries - from predictor.utils import morphological_cleaning + # geomltoolkits>=2 moved modules (no geomltoolkits.regularizer/utils) + from geomltoolkits.geometry.validate import validate_polygon_geometries + from geomltoolkits.raster.merge import merge_rasters + from geomltoolkits.raster.morphology import morphological_cleaning + from geomltoolkits.raster.vectorize import vectorize_mask pred_dir = _to_local_path(prediction_masks_dir, "prediction_masks_dir") out_dir = _to_local_path(output_dir, "output_dir") @@ -204,19 +293,27 @@ def postprocess(prediction_masks_dir: str, output_dir: str) -> dict: merged_mask_path = out_dir / "merged_prediction_mask.tif" merged_geojson_path = out_dir / "predictions.geojson" - tmp_dir = out_dir / "tmp" - tmp_dir.mkdir(parents=True, exist_ok=True) merge_rasters(str(pred_dir), str(merged_mask_path)) morphological_cleaning(str(merged_mask_path)) - gdf = VectorizeMasks( + gdf = vectorize_mask( + input_tiff=str(merged_mask_path), + output_geojson=str(merged_geojson_path), simplify_tolerance=0.5, min_area=3, orthogonalize=True, - tmp_dir=str(tmp_dir), ortho_skew_tolerance_deg=15, ortho_max_angle_change_deg=15, - ).convert(str(merged_mask_path), str(merged_geojson_path)) + ) + + geojson_dict = json.loads(gdf.to_json()) + if not geojson_dict.get("features"): + # geomltoolkits validator raises on empty FeatureCollections; treat this as a valid + # "no buildings found" prediction and keep the already-written GeoJSON file. + if not merged_geojson_path.is_file(): + with merged_geojson_path.open("w", encoding="utf-8") as fh: + json.dump(geojson_dict, fh) + return geojson_dict if gdf.crs and gdf.crs != "EPSG:4326": gdf = gdf.to_crs("EPSG:4326") @@ -234,7 +331,7 @@ def postprocess(prediction_masks_dir: str, output_dir: str) -> dict: gdf.to_file(merged_geojson_path, driver="GeoJSON") validated_geojson = validate_polygon_geometries( - json.loads(gdf.to_json()), + geojson_dict, output_path=str(merged_geojson_path), ) if isinstance(validated_geojson, str) and Path(validated_geojson).is_file(): @@ -297,23 +394,70 @@ def run_preprocessing( @step def train_model( data_base_path: str, - data_loader: List[Tuple[Any, Any]], preprocessed_path: str, + data_loader: Optional[List[Tuple[Any, Any]]] = None, stac_item_path: str = "models/ramp/stac-item.json", val_fraction: Optional[float] = None, + split_info: Optional[Dict[str, Any]] = None, ) -> Any: """ZenML step wrapper for RAMP training; returns the best Keras SavedModel checkpoint.""" - if not data_loader: + if data_loader is not None and not data_loader: raise RuntimeError("Preprocessing returned an empty dataloader; no chip/mask pairs found.") return train_ramp_model( data_base_path=data_base_path, preprocessed_path=preprocessed_path, stac_item_path=stac_item_path, val_fraction=val_fraction, + split_info=split_info, log_zenml_step_metadata=True, ) +@step +def split_dataset( + preprocessed_path: str, + output_path: str, + hyperparameters: Optional[Dict[str, Any]] = None, +) -> Annotated[Dict[str, Any], "split_info"]: + """Create train/validation directories for RAMP and return split metadata. + + Uses the existing utilities implementation: + `hot_fair_utilities.training.ramp.prepare_data.split_training_2_validation`. + """ + from hot_fair_utilities.training.ramp.prepare_data import split_training_2_validation + + hyperparameters = hyperparameters or {} + preprocessed_dir = _to_local_path(preprocessed_path, "preprocessed_path") + if not preprocessed_dir.is_dir(): + raise FileNotFoundError(f"Preprocessed directory not found: {preprocessed_dir}") + + out_dir = _to_local_path(output_path, "output_path") + ramp_train_dir = out_dir / "ramp_training_work" + + split_training_2_validation( + str(preprocessed_dir), + str(ramp_train_dir), + multimasks=True, + ) + + train_count = len(list((ramp_train_dir / "chips").glob("*.tif"))) + val_count = len(list((ramp_train_dir / "val-chips").glob("*.tif"))) + total = train_count + val_count + val_ratio = (float(val_count) / float(total)) if total else 0.0 + + split_info: Dict[str, Any] = { + "strategy": "random", + "val_ratio": val_ratio, + "train_count": train_count, + "val_count": val_count, + "seed": int(hyperparameters.get("split_seed", 42)), + "ramp_train_dir": str(ramp_train_dir), + "source": "hot_fair_utilities.training.ramp.prepare_data.split_training_2_validation", + } + log_metadata(metadata={"fair/split": split_info}) + return split_info + + def train_ramp_model( data_base_path: str, preprocessed_path: str, @@ -323,6 +467,7 @@ def train_ramp_model( batch_size: Annotated[Optional[int], "1 <= batch_size <= 8"] = None, backbone: Optional[str] = None, early_stopping_patience: Annotated[Optional[int], "1 <= early_stopping_patience <= 20"] = None, + split_info: Optional[Dict[str, Any]] = None, log_zenml_step_metadata: bool = False, ) -> Any: """Fine-tune EfficientNetB0 + U-Net on 4-class multimask chips. @@ -363,7 +508,6 @@ def train_ramp_model( import segmentation_models as sm from hot_fair_utilities.training.ramp.cleanup import extract_highest_accuracy_model from hot_fair_utilities.training.ramp.config import RAMP_CONFIG - from hot_fair_utilities.training.ramp.prepare_data import split_training_2_validation from hot_fair_utilities.training.ramp.run_training import ( manage_fine_tuning_config, run_main_train_code, @@ -408,20 +552,24 @@ def train_ramp_model( if not 1 <= int(eff_patience) <= 20: raise ValueError(f"Resolved early_stopping_patience={eff_patience} is outside [1, 20]") - # Split preprocessed_path into train + val dirs under a dedicated work dir - # (split_training_2_validation requires src != dst) - ramp_train_dir_path = _to_local_path( - str(Path(data_base_path) / "ramp_training_work"), - "ramp_training_work", - ) - if ramp_train_dir_path.exists(): - rmtree(ramp_train_dir_path) - ramp_train_dir = str(ramp_train_dir_path) - split_training_2_validation( - str(_to_local_path(preprocessed_path, "preprocessed_path")), - ramp_train_dir, - multimasks=True, - ) + # Prefer a precomputed split (CI-visible split_dataset step), fallback to internal split for compatibility. + if split_info and split_info.get("ramp_train_dir"): + ramp_train_dir = str(_to_local_path(str(split_info["ramp_train_dir"]), "ramp_train_dir")) + else: + from hot_fair_utilities.training.ramp.prepare_data import split_training_2_validation + + ramp_train_dir_path = _to_local_path( + str(Path(data_base_path) / "ramp_training_work"), + "ramp_training_work", + ) + if ramp_train_dir_path.exists(): + rmtree(ramp_train_dir_path) + ramp_train_dir = str(ramp_train_dir_path) + split_training_2_validation( + str(_to_local_path(preprocessed_path, "preprocessed_path")), + ramp_train_dir, + multimasks=True, + ) # Build config from RAMP_CONFIG via the utilities helper, then apply overrides cfg = manage_fine_tuning_config(ramp_train_dir, eff_epochs, eff_batch, freeze_layers=False, multimasks=True) @@ -490,7 +638,10 @@ def infer_ramp_model( import tempfile import tensorflow as tf - from predictor.prediction import run_prediction + + # Ensure fairpredictor does not use TFSMLayer on TF 2.15. + _patch_predictor_savedmodel_loader_for_tf215() + from predictor.prediction import run_prediction # noqa: E402 cache = Path(model_cache_dir) if model_cache_dir else None if isinstance(model_uri, (str, Path)): @@ -504,7 +655,17 @@ def infer_ramp_model( "model_uri must be a str, pathlib.Path, or a tf.keras.Model from training (compile=False load)." ) - input_dir = _to_local_path(input_path, "input_path") + # Fail fast if our patch didn't apply (helps diagnose environment mismatches early). + import importlib + + pred = importlib.import_module("predictor.prediction") + if not getattr(getattr(pred, "_load_keras_model", None), "_fair_models_tf215_savedmodel_loader", False): + raise RuntimeError( + "fairpredictor SavedModel loader patch not applied; " + "expected predictor.prediction._load_keras_model to be patched for TF 2.15." + ) + + input_dir = _resolve_input_directory(input_path, "input_path") out_dir = _to_local_path(prediction_path, "prediction_path") out_dir.mkdir(parents=True, exist_ok=True) @@ -565,11 +726,17 @@ def training_pipeline( boundary_width=boundary_width, contact_spacing=contact_spacing, ) + split_info = split_dataset( + preprocessed_path=f"{output_path}/preprocessed", + output_path=output_path, + hyperparameters=hyperparams, + ) train_model( data_base_path=output_path, data_loader=data_loader, preprocessed_path=f"{output_path}/preprocessed", stac_item_path=stac_item_path, + split_info=split_info, ) diff --git a/models/ramp/tests/inside_container_smoke_test.py b/models/ramp/tests/inside_container_smoke_test.py index f3097b38..024bc761 100644 --- a/models/ramp/tests/inside_container_smoke_test.py +++ b/models/ramp/tests/inside_container_smoke_test.py @@ -4,7 +4,8 @@ 1) Critical imports (tensorflow, ramp, segmentation_models, osgeo.gdal, solaris) 2) ``pipeline.preprocess`` path (georeference + multimask generation) 3) ``pipeline.run_preprocessing`` contract: chip/mask arrays + on-disk counts -4) Training wrappers: ``pipeline.train_model`` and ``pipeline.train_ramp_model`` +4) ``pipeline.split_dataset`` + training wrappers: + ``pipeline.train_model`` and ``pipeline.train_ramp_model`` both return loaded ``tf.keras.Model`` 5) ``pipeline.resolve_model_href`` for local SavedModel directories 6) ``pipeline.run_inference`` returns merged GeoJSON dict content @@ -232,18 +233,31 @@ def main() -> None: ) # ------------------------------------------------------------------------- - _stage("Test 4: Training smoke run via train_model/train_ramp_model wrappers") + _stage("Test 4: split_dataset + training smoke run") # train_model wraps train_ramp_model and validates preprocessing output. # train_ramp_model handles val split internally and returns loaded tf.keras.Model. import tensorflow as tf + split_info = ramp_pipeline.split_dataset( + preprocessed_path=str(preprocessed_dir), + output_path=str(dataset_root), + hyperparameters={"split_seed": 42}, + ) + _assert(split_info["train_count"] > 0, "split_dataset produced zero train chips.") + _assert(split_info["val_count"] > 0, "split_dataset produced zero validation chips.") + print( + "PASS: split_dataset produced " + f"{split_info['train_count']} train and {split_info['val_count']} validation chips" + ) + trained_model_step = ramp_pipeline.train_model( data_base_path=str(dataset_root), - data_loader=data_loader_contract, preprocessed_path=str(preprocessed_dir), + data_loader=data_loader_contract, stac_item_path="/workspace/models/ramp/stac-item.json", val_fraction=0.15, + split_info=split_info, ) _assert(isinstance(trained_model_step, tf.keras.Model), "train_model did not return a Keras model checkpoint.") From 394a8269055280beae571d8d7b57c79b5b59555e Mon Sep 17 00:00:00 2001 From: Abdelrahman Katkat Date: Tue, 21 Apr 2026 14:46:12 +0200 Subject: [PATCH 05/30] refactor(ramp): update Dockerfile and pipeline for improved structure and dependencies - Refactored Dockerfile to separate build, runtime, test, and inference stages for better clarity and efficiency. - Updated `pyproject.toml` to remove unnecessary lint ignores. - Enhanced `pipeline.py` with improved model resolution and added support for lazy imports. - Revised README.md for clarity on architecture and usage. - Added test fixtures for a toy dataset and implemented step tests for the RAMP pipeline. - Updated STAC item schema to version 1.1.0 and included additional metadata properties. --- models/ramp/Dockerfile | 89 +- models/ramp/README.md | 291 +--- models/ramp/pipeline.py | 1237 ++++++++++------- models/ramp/stac-item.json | 217 ++- models/ramp/tests/conftest.py | 128 ++ .../ramp/tests/inside_container_smoke_test.py | 336 ----- models/ramp/tests/run_docker_tests.ps1 | 77 - models/ramp/tests/run_docker_tests.sh | 41 - models/ramp/tests/test_steps.py | 170 +++ pyproject.toml | 6 - 10 files changed, 1316 insertions(+), 1276 deletions(-) create mode 100644 models/ramp/tests/conftest.py delete mode 100644 models/ramp/tests/inside_container_smoke_test.py delete mode 100644 models/ramp/tests/run_docker_tests.ps1 delete mode 100644 models/ramp/tests/run_docker_tests.sh create mode 100644 models/ramp/tests/test_steps.py diff --git a/models/ramp/Dockerfile b/models/ramp/Dockerfile index d19d2457..338fd42b 100644 --- a/models/ramp/Dockerfile +++ b/models/ramp/Dockerfile @@ -1,23 +1,86 @@ # syntax=docker/dockerfile:1.7 -# Base: ghcr.io/hotosm/fair-utilities-ramp:cpu-latest (or :gpu-latest via --build-arg BASE_IMAGE=...) -# Build from fAIr-models root: docker build -f models/ramp/Dockerfile -t ramp-v1:cpu . +# Base image: ghcr.io/hotosm/fair-utilities-ramp:cpu-latest (or :gpu-latest via BASE_IMAGE build arg). +# Build from the fAIr-models repo root: +# docker build -f models/ramp/Dockerfile --target test -t ramp:test . +# docker build -f models/ramp/Dockerfile --target runtime -t ramp:runtime . +# docker build -f models/ramp/Dockerfile --target inference -t ramp:inference . ARG BASE_IMAGE=ghcr.io/hotosm/fair-utilities-ramp:cpu-latest -FROM ${BASE_IMAGE} -ENV MPLBACKEND=Agg \ - PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 \ - RAMP_HOME=/workspace \ - SM_FRAMEWORK=tf.keras +# --------------------------------------------------------------------------- +# Builder stage: install model-pack deps into the base image venv +# --------------------------------------------------------------------------- +FROM ${BASE_IMAGE} AS builder + +ENV UV_LINK_MODE=copy + +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /usr/local/bin/ WORKDIR /workspace -RUN /app/.venv/bin/python -m pip install --no-cache-dir "fair-py-ops==0.0.6" +COPY pyproject.toml README.md fair_zenml_patch.pth /tmp/fair-src/ +COPY fair /tmp/fair-src/fair -# Smoke-test extras only (ZenML[server], STAC, fairpredictor); drop for production slim images. -RUN /app/.venv/bin/python -m pip install --no-cache-dir \ - "zenml[server]==0.93.3" \ +# The fair-utilities-ramp image already provides hot_fair_utilities, ramp-fair, +# segmentation_models, and TensorFlow under /app/.venv. Add the fAIr model-pack +# deps (fair itself, STAC validation, fairpredictor, onnx tooling) into that +# same venv so training, tests, and inference share one interpreter. +RUN --mount=type=cache,target=/root/.cache/uv \ + SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 \ + uv pip install --python /app/.venv/bin/python \ "pystac[validation]>=1.14.3" \ "universal-pathlib" \ - "fairpredictor>=0.5.1" + "fairpredictor>=0.5.1" \ + "onnx>=1.16" \ + "tf2onnx>=1.16" \ + /tmp/fair-src[k8s] + +# --------------------------------------------------------------------------- +# Runtime stage: production image (no test deps) +# --------------------------------------------------------------------------- +FROM ${BASE_IMAGE} AS runtime + +WORKDIR /workspace + +ENV PATH="/app/.venv/bin:$PATH" \ + MPLBACKEND=Agg \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + RAMP_HOME=/app \ + SM_FRAMEWORK=tf.keras + +COPY --from=builder /app/.venv /app/.venv + +COPY models/ramp models/ramp +COPY models/conftest.py models/conftest.py +COPY models/test_integration.py models/test_integration.py + +ENTRYPOINT ["/app/.venv/bin/python"] + +# --------------------------------------------------------------------------- +# Test stage: add pytest + ZenML server deps for step & integration tests +# --------------------------------------------------------------------------- +FROM runtime AS test + +COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv +COPY --from=builder /tmp/fair-src /tmp/fair-src + +RUN --mount=type=cache,target=/root/.cache/uv \ + SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 \ + uv pip install --python /app/.venv/bin/python /tmp/fair-src[test] + +# --------------------------------------------------------------------------- +# Inference stage: runtime + serving deps for the smoke test / live API +# --------------------------------------------------------------------------- +FROM runtime AS inference + +COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv +COPY --from=builder /tmp/fair-src /tmp/fair-src + +RUN --mount=type=cache,target=/root/.cache/uv \ + SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 \ + uv pip install --python /app/.venv/bin/python /tmp/fair-src[serve] + +ENV PYTHONPATH=/workspace +EXPOSE 8080 +CMD ["/app/.venv/bin/uvicorn", "fair.serve.base:create_app", "--factory", "--host", "0.0.0.0", "--port", "8080"] diff --git a/models/ramp/README.md b/models/ramp/README.md index df1106b7..e55d1893 100644 --- a/models/ramp/README.md +++ b/models/ramp/README.md @@ -1,275 +1,60 @@ -# RAMP — EfficientNetB0 + U-Net Building Semantic Segmentation +# RAMP EfficientNetB0 + U-Net Building Segmentation -RAMP (Replicable AI for Microplanning) is an EfficientNetB0 encoder + U-Net decoder -for pixel-wise 4-class building segmentation on 256 × 256 px aerial image chips. +Semantic segmentation model for building footprint extraction from RGB aerial imagery, derived from the RAMP (Replicable AI for Microplanning) project. -## Architecture overview +## Architecture -| Component | Detail | -| --- | --- | -| Encoder | EfficientNet-B0 (ImageNet pre-trained via segmentation_models) | -| Decoder | U-Net symmetric decoder with skip connections | -| Output | 4-class sparse categorical mask (channels-last, uint8) | -| Classes | 0=background, 1=building, 2=boundary, 3=contact-point | -| Loss | Sparse categorical crossentropy | -| Metric | Sparse categorical accuracy (`val_sparse_categorical_accuracy`) | -| Framework | TensorFlow 2.15.1 / Keras | +- **Model**: EfficientNetB0 encoder + U-Net decoder (`EffUnet`) +- **Framework**: TensorFlow 2.15 / `tf.keras` (via `segmentation_models` with `SM_FRAMEWORK=tf.keras`) +- **Task**: Semantic segmentation (sparse categorical crossentropy) +- **Input**: RGB chips (256x256, float32, channels-last) +- **Classes**: 4 (background=0, building=1, boundary=2, contact=3) -The boundary (class 2) and contact-point (class 3) channels help the model cleanly -separate adjacent buildings at inference time, even when they share a wall. +The boundary (class 2) and contact (class 3) channels help the model cleanly separate adjacent buildings at inference time, even when they share a wall. The `predict()` helper collapses the 4-class softmax to a binary building mask before vectorization. -## Key difference from YOLO packs +## Pretrained Source -| Stage | YOLO v8 v1/v2 | RAMP | -| --- | --- | --- | -| Preprocessing | `hot_fair_utilities.preprocess` | same | -| Training | `hot_fair_utilities.training.yolo_v8_*` (ultralytics) | `ramp.training.*` (TF/Keras) | -| Inference | `hot_fair_utilities.predict` (ultralytics) | `fairpredictor.predictor.prediction.run_prediction` | -| Postprocessing | `hot_fair_utilities.polygonize` (AutoBFE) | `geomltoolkits` vectorization + geometry validation (writes `predictions.geojson`) | +Baseline RAMP weights (TensorFlow SavedModel) hosted by HOTOSM: -## Model pack contents +- Checkpoint: `https://api-prod.fair.hotosm.org/api/v1/workspace/download/ramp/baseline.zip` +- ONNX model: `https://api-prod.fair.hotosm.org/api/v1/workspace/download/ramp/ramp-v1.onnx` -| File | Purpose | -| --- | --- | -| `pipeline.py` | ZenML `@step` / `@pipeline` entrypoints (pre → train → infer → post); `resolve_model_href()` for URLs | -| `stac-item.json` | STAC MLM item — model weights (mlm:model href), entrypoints; weights from local path or HTTP(S) `.zip` | -| `Dockerfile` | Runtime on top of `ghcr.io/hotosm/fair-utilities-ramp` (CPU/GPU); see [Docker image composition](#docker-image-composition) | +## Pipeline -## Docker image composition +Training pipeline steps (ZenML) defined in `pipeline.py`: -The RAMP image is **layered**, not a single monolithic install: +- `split_dataset` - preprocesses chips + labels into 4-class multimasks and produces a seeded random train/validation split via `hot_fair_utilities.split_training_2_validation` +- `train_model` - fine-tunes RAMP on the split, returning the best SavedModel serialized as a zipped byte stream +- `evaluate_model` - computes `fair:accuracy`, `fair:mean_iou` (building IoU), `fair:precision`, and `fair:recall` on the validation split +- `export_onnx` - converts the trained SavedModel to an ONNX byte stream via `tf2onnx` -1. **Base (hot-fair-utilities on GHCR)** - TensorFlow, GDAL, `hot-fair-utilities` RAMP extras, fairpredictor, geomltoolkits, and related geospatial/ML stack. You do not build this locally; Docker pulls `cpu-latest` or `gpu-latest`. +Inference is served through `fair.serve.base`, which calls the module-level `predict(session, input_images, params) -> FeatureCollection`: each chip is preprocessed, run through an `onnxruntime` session, decoded to a binary building mask, and vectorized to georeferenced polygons. -2. **`fair-py-ops` (pinned)** - Installed in this Dockerfile to match `fAIr-models/pyproject.toml` (`fair-py-ops==0.0.6`). This is the long-term registry/orchestration contract for the repo. +## Base Image -3. **Temporary test-only packages (remove before production merge)** - The Dockerfile installs **`zenml[server]==0.93.3`** and **`pystac[validation]>=1.14.3`**, aligned with `pyproject.toml` (`zenml>=0.93.3`, `pystac[validation]>=1.14.3`). - **Why `[server]` on ZenML:** `pipeline.py` wraps logic in `@step`. Calling a step (e.g. `run_preprocessing(...)`) runs ZenML’s single-step pipeline, which initializes the default **SQL** zen store. That code path imports **`sqlalchemy_utils`** (and related SQL stack). Those are **not** included in a bare `pip install zenml==…` — they are part of ZenML’s **`server`** optional extra (same idea as the repo’s `[dependency-groups] local` → `zenml[server]`). Without `[server]`, you get `ModuleNotFoundError: No module named 'sqlalchemy_utils'`. - **`fAIr-utilities/docker/Dockerfile.ramp`** does not install ZenML, PySTAC, `fair-py-ops`, or this SQL stack; those are added only in this model Dockerfile layer. - - **Before you push production-oriented code**, delete the **second** `RUN pip install …` block in `models/ramp/Dockerfile` (the one that installs `zenml[server]` and `pystac`). Keep the `fair-py-ops` `RUN` unless your platform injects it another way. - After removal, in-container smoke tests that import `pipeline.py` will fail unless you refactor (e.g. split core vs ZenML) or run tests only on infrastructure where ZenML is pre-installed. - -**STAC in code vs `pystac` on PyPI:** -Training hyperparameters in this pack are read from **`stac-item.json` with plain `json`** — you do not need the `pystac` Python package for that path. The Dockerfile adds `pystac[validation]` **only for testing parity** with the repo’s declared dependencies; production can rely on STAC-as-JSON plus platform tooling. - -**`SM_FRAMEWORK=tf.keras`:** -The `segmentation_models` / `efficientnet` stack reads `SM_FRAMEWORK` when the package is **first imported**. The default path uses standalone `keras` and `efficientnet.keras`, which rely on removed Keras 2 APIs (`keras.utils.generic_utils`) and fail on TensorFlow 2.15+ (bundled Keras 3). The Dockerfile sets `SM_FRAMEWORK=tf.keras` so `efficientnet.tfkeras` is used. Do not unset this in the RAMP container unless you pin an older TensorFlow/Keras stack. - -## Data directory layout - -**Option A: Use shared `data/sample`** (recommended for tests) - -The smoke tests use `data/sample` (train/oam + train/osm). The test script -auto-converts OAM GeoTIFFs to PNG and merges OSM labels into the RAMP format. - -**Option B: Legacy layout** - -```text -dataset/ -├── input/ -│ ├── *.png # OAM chips (PNG, no geo-reference) -│ └── labels.geojson # combined building polygon labels -├── preprocessed/ # created by preprocess (hot_fair_utilities) -│ ├── chips/ # georeferenced .tif chips (EPSG:3857) -│ ├── labels/ # per-chip .geojson labels -│ ├── multimasks/ # 4-class .mask.tif targets -│ └── (no training outputs here) # training uses a separate work dir -├── ramp_training_work/ # created by train_ramp_model -│ ├── chips/ # training chips (copied from preprocessed) -│ ├── multimasks/ # training masks (copied from preprocessed) -│ ├── val-chips/ # validation split chips (hyphenated) -│ ├── val-multimasks/ # validation split masks (hyphenated) -│ └── model-checkpts/ # SavedModel checkpoints + best model selection -└── prediction/ - ├── input/ # chips for inference (GeoTIFFs; typically copy from preprocessed/chips/) - ├── output/ # fairpredictor outputs (includes georeference/*.tif) - └── vectors/ # merged GeoJSON: predictions.geojson (+ merged_prediction_mask.tif, tmp/) -``` - -## Pipeline steps - -```text -input/ (PNG chips + labels.geojson) - │ - ▼ -[run_preprocessing] hot_fair_utilities.preprocess (georeference + multimask; returns chip/mask arrays) - │ preprocessed/chips/ + preprocessed/multimasks/ - ▼ -[train_model] hot_fair_utilities.training.ramp.* (EfficientNetB0 U-Net, TF/Keras) - │ best checkpoint (SavedModel) loaded as tf.keras.Model + val_sparse_categorical_accuracy - ▼ -[run_inference] fairpredictor.run_prediction (georeferenced prediction rasters) - │ prediction/output/georeference/*.tif - ▼ -[run_postprocessing] geomltoolkits vectorization + validation (writes predictions.geojson; returns dict) - │ - ▼ -prediction/vectors/predictions.geojson (merged building footprints) -``` - -## Running locally (outside ZenML) - -```python -from models.ramp.pipeline import infer_ramp_model, train_ramp_model - -# Training (expects you've already run preprocessing to produce chips/ + multimasks/) -trained_model = train_ramp_model( - data_base_path="data/sample/ramp_work", - preprocessed_path="data/sample/ramp_work/preprocessed_test", - stac_item_path="models/ramp/stac-item.json", -) - -# Inference run (model_uri can be a local SavedModel dir, an HTTP(S) .zip, or a tf.keras.Model) -final_geojson = infer_ramp_model( - model_uri=trained_model, - input_path="data/sample/ramp_work/preprocessed_test/chips", - prediction_path="data/sample/ramp_work/prediction_test/output", - output_dir="data/sample/ramp_work/prediction_test/vectors", -) -# final_geojson is a dict (FeatureCollection-like). predictions.geojson is also written under output_dir. -``` - -## Building the Docker image - -The `fAIr-utilities` team publishes pre-built base images to GitHub Container Registry (GHCR) on -every push to `master`. You **do not need to build or clone the `fAIr-utilities` repo yourself**. -Docker pulls the image automatically. - -| Flavour | GHCR image | -| --- | --- | -| CPU (default) | `ghcr.io/hotosm/fair-utilities-ramp:cpu-latest` | -| GPU (CUDA) | `ghcr.io/hotosm/fair-utilities-ramp:gpu-latest` | - -Each build adds (see Dockerfile comments): - -- `fair-py-ops==0.0.6` (from `pyproject.toml`) -- **Temporary:** `zenml[server]==0.93.3` and `pystac[validation]>=1.14.3` — the `[server]` extra pulls the SQL stack (`sqlalchemy-utils`, etc.) required when `@step` runs; remove the dedicated `RUN` before a production merge if slim images must not carry orchestration deps (see [Docker image composition](#docker-image-composition)). - -```bash -# CPU image (base image pulled from GHCR automatically — no --build-arg needed) -docker build -t ramp-v1:cpu -f models/ramp/Dockerfile . - -# GPU image (override the ARG to select the GPU base) -docker build -t ramp-v1:gpu \ - --build-arg BASE_IMAGE=ghcr.io/hotosm/fair-utilities-ramp:gpu-latest \ - -f models/ramp/Dockerfile . -``` - -```powershell -# PowerShell (Windows) — do NOT use "\" for line continuation -docker build -t ramp-v1:cpu -f models/ramp/Dockerfile . -docker build -t ramp-v1:gpu --build-arg BASE_IMAGE=ghcr.io/hotosm/fair-utilities-ramp:gpu-latest -f models/ramp/Dockerfile . -``` - -> **Testing unreleased `fAIr-utilities` code?** If GHCR doesn't yet contain a feature you need, -> add a `pip install` from git inside your Dockerfile layer: -> ```dockerfile -> RUN pip install --no-cache-dir \ -> "hot-fair-utilities @ git+https://github.com/hotosm/fAIr-utilities.git@" -> ``` -> You do not need to build the base image yourself. - -## Running the smoke tests - -The in-container script `models/ramp/tests/inside_container_smoke_test.py` imports `pipeline.py`, -which loads **ZenML** at import time. For that to work inside Docker, the image must either include -the temporary **ZenML + PySTAC** `RUN` in the Dockerfile (current approach for local/CI testing) or -you must refactor tests / pipeline imports (see [Docker image composition](#docker-image-composition)). - -```powershell -# PowerShell (Windows) -$env:BUILD_IMAGE = "1" -$env:CPU_ONLY = "1" -.\models\ramp\tests\run_docker_tests.ps1 -``` - -```bash -# Bash (Linux / macOS / Git Bash) -BUILD_IMAGE=1 CPU_ONLY=1 ./models/ramp/tests/run_docker_tests.sh -``` - -> **Note**: The smoke tests use `data/sample` (train/oam OAM tiles + train/osm labels). -> Run from the fAIr-models repo root so `/workspace/data/sample` is available in the container. - -### Running the smoke script directly (after building the image) - -If you already built the image (see “Building the Docker image”), you can run the smoke test script directly: -`models/ramp/tests/inside_container_smoke_test.py`. +Training, test, and inference Docker stages all build on +`ghcr.io/hotosm/fair-utilities-ramp:cpu-latest` (or `:gpu-latest` via the +`BASE_IMAGE` build arg), which provides TensorFlow, GDAL, `hot_fair_utilities` +RAMP extras, and the RAMP runtime under `/app/.venv`. ```bash -# CPU image -docker run --rm -v "$(pwd):/workspace" ramp-v1:cpu \ - python /workspace/models/ramp/tests/inside_container_smoke_test.py \ - --dataset-root /workspace/data/sample \ - --epochs 2 --batch-size 4 --backbone efficientnetb0 - -# GPU image -docker run --rm --gpus all -v "$(pwd):/workspace" ramp-v1:gpu \ - python /workspace/models/ramp/tests/inside_container_smoke_test.py \ - --dataset-root /workspace/data/sample \ - --epochs 2 --batch-size 4 --backbone efficientnetb0 -``` - -```powershell -# PowerShell (Windows) — same idea, no Bash "\" line continuation -docker run --rm -v "${PWD}:/workspace" ramp-v1:cpu python /workspace/models/ramp/tests/inside_container_smoke_test.py --dataset-root /workspace/data/sample --epochs 2 --batch-size 4 --backbone efficientnetb0 - -docker run --rm --gpus all -v "${PWD}:/workspace" ramp-v1:gpu python /workspace/models/ramp/tests/inside_container_smoke_test.py --dataset-root /workspace/data/sample --epochs 2 --batch-size 4 --backbone efficientnetb0 -``` - -If you want to test a different dataset layout, point `--dataset-root` at a directory that contains either: - -- `train/oam/*.tif` + `train/osm/*.geojson` (sample-style), or -- `input/*.png` + `input/labels.geojson` (legacy-style) - -## Model weights (STAC mlm:model asset) - -The STAC Item's `assets.model.href` points to pretrained weights. Supported sources: - -| Source | Example | -| --- | --- | -| Local SavedModel directory | `/workspace/ramp-data/baseline` | -| HTTP(S) `.zip` containing a SavedModel | `https://example.com/ramp_model.zip` | - - For remote weights, publish an HTTP(S) `.zip` that contains a SavedModel directory with `saved_model.pb` and `variables/`. Downloaded zips are cached under `/workspace/.ramp_model_cache/`. - -## Registering in the STAC catalog - -```python -from fair.stac.catalog_manager import CatalogManager - -cm = CatalogManager() -cm.register_model("models/ramp/stac-item.json") +# Build targets +docker build -f models/ramp/Dockerfile --target runtime -t ramp:runtime . +docker build -f models/ramp/Dockerfile --target test -t ramp:test . +docker build -f models/ramp/Dockerfile --target inference -t ramp:inference . ``` -## Dependencies from hot_fair_utilities - -This pack uses **both preprocessing and training** paths from `hot-fair-utilities`: - -| Used | Not used | -| --- | --- | -| `hot_fair_utilities.preprocess` (georeference + multimasks) | `hot_fair_utilities.predict` (YOLO / ultralytics) | -| `hot_fair_utilities.training.ramp.*` (RAMP_CONFIG, split/train helpers) | `hot_fair_utilities.polygonize` (AutoBFE, YOLO output) | - -The `ultralytics` and `torch` packages are installed transitively by -`hot-fair-utilities` but are never imported at runtime for RAMP. +## Limitations and Bias -## Key design decisions +- Training data and baseline weights are derived from the RAMP corpus (primarily humanitarian-mapping contexts); performance on dense urban scenes with complex roof structures may be lower than on sparser rural settlements. +- The model is sensitive to imagery with strong color casts, motion blur, or significant off-nadir angle; preprocess inputs to approximately nadir RGB at the target zoom before inference. +- Binary building output from `predict()` discards the boundary/contact auxiliary classes after decoding; downstream polygonization may still merge neighbouring buildings that share a footprint edge. -**Why ramp-fair and not inline model code?** -`ramp-fair` provides the complete EfficientNetB0 U-Net + training utilities. -This pack is thin: it declares *how* to run the model (pipeline.py) and -*what* it is (stac-item.json). Upgrading means bumping the `ramp-fair` pin. +## Citation -**Where do “ramp” + “solaris” come from?** -They are pulled in via the `hot-fair-utilities[ramp]` / `[ramp-gpu]` extras in the Dockerfile. The RAMP runtime is intentionally installed as a consistent bundle inside the image so local machines don’t need to compile the full stack. +RAMP - Replicable AI for Microplanning. Upstream source: https://github.com/radiantearth/ramp-code -**Why is validation split handled in training?** -RAMP training expects explicit train/val directories. `train_ramp_model` calls `split_training_2_validation`, creating `ramp_training_work/val-chips` and `ramp_training_work/val-multimasks` under a dedicated training work directory. +## License -**Why one Dockerfile per model?** -The RAMP stack depends on TensorFlow + GDAL + geospatial libs with tight version coupling. Keeping a per-model image prevents version conflicts and makes the runtime reproducible. +- Model weights and code: Apache-2.0 +- Training data: ODbL-1.0 (OpenStreetMap-derived labels) diff --git a/models/ramp/pipeline.py b/models/ramp/pipeline.py index c175acd7..2a06e6ee 100644 --- a/models/ramp/pipeline.py +++ b/models/ramp/pipeline.py @@ -1,50 +1,50 @@ """ZenML pipeline for RAMP (EfficientNetB0 + U-Net) building semantic segmentation. -Entrypoints referenced by stac-item.json. -Runtime: ramp-fair (TensorFlow/Keras), hot-fair-utilities (preprocessing + training). - -Implements the fAIr entrypoints: - - pre_processing_function → preprocess() - - post_processing_function → postprocess() - - mlm:entrypoint (training) → training_pipeline() - - inference (model from STAC mlm:model asset href) → inference_pipeline() - -Model weights: Backend passes model_uri from STAC Item (assets.model.href). -Supports direct HTTP(S) URLs to .zip archives and local paths. -Google Drive is not supported; weights should be published to HTTP or staged locally. - -Inference/postprocess use the **fairpredictor** PyPI distribution; its importable top-level -package is ``predictor`` (``pip install fairpredictor`` → ``import predictor``), same as -``hot_fair_utilities.inference.predict``. - -All heavy imports are lazy: this module is importable in the fAIr-models -host environment where tensorflow, ramp, and solaris are not installed. +Follows the fAIr-models contract: platform-provided hyperparameters (no runtime STAC reads), +ONNX export for portable inference, and a module-level ``predict`` entrypoint used by +``fair.serve.base``. + +Pipeline contract: + training_pipeline(base_model_weights, dataset_chips, dataset_labels, num_classes, hyperparameters) + -> split_dataset -> train_model (bytes) -> evaluate_model (fair:* metrics) -> export_onnx (bytes) + inference_pipeline(model_uri, input_images, ...) + -> run_inference -> FeatureCollection + +Runtime: TensorFlow/Keras via ramp-fair + hot-fair-utilities (preprocessing + training). +All heavy imports (tensorflow, segmentation_models, hot_fair_utilities, tf2onnx) are lazy so +this module is importable in lightweight environments (e.g. fair.utils.model_validator AST checks). """ +from __future__ import annotations + +import hashlib +import io +import json +import os import re +import shutil +import tempfile import zipfile from pathlib import Path -from shutil import copy2, rmtree -from typing import Annotated, Any, Dict, List, Optional, Tuple, Union +from typing import Annotated, Any from urllib.request import urlretrieve from zenml import log_metadata, pipeline, step +from fair.zenml.steps import load_model + _DEFAULT_MODEL_CACHE = Path("/workspace/.ramp_model_cache") _DEFAULT_RAMP_BASELINE_URL = ( "https://api-prod.fair.hotosm.org/api/v1/workspace/download/ramp/baseline.zip" ) +_QUBVEL_EFFICIENTNET_RELEASE = ( + "https://github.com/qubvel/efficientnet/releases/download/v0.0.1/" +) -def _download_and_extract_zip(zip_url: str, dest_dir: Path) -> None: - """Download a ZIP URL and extract in dest_dir.""" - dest_dir.mkdir(parents=True, exist_ok=True) - zip_name = Path(zip_url.split("/")[-1]).name or "archive.zip" - zip_path = dest_dir / zip_name - urlretrieve(zip_url, zip_path) - with zipfile.ZipFile(zip_path, "r") as zf: - zf.extractall(dest_dir) - zip_path.unlink(missing_ok=True) +# --------------------------------------------------------------------------- +# Path / resource resolvers +# --------------------------------------------------------------------------- def _to_local_path(path_value: str, purpose: str) -> Path: @@ -55,57 +55,131 @@ def _to_local_path(path_value: str, purpose: str) -> Path: protocol = getattr(upath_obj, "protocol", "") or "" if protocol not in ("", "file"): raise NotImplementedError( - f"{purpose} requires a local filesystem path. " - f"Received protocol={protocol!r} for {path_value!r}." + f"{purpose} requires a local filesystem path. Received protocol={protocol!r} for {path_value!r}." ) return Path(str(upath_obj)) def _resolve_input_directory(path_value: str, purpose: str) -> Path: - """Resolve local or remote (S3/HTTP/etc.) directories to a local path.""" - if "://" in str(path_value): - # Available in the runtime (same helper used by YOLO model packs). - from fair.utils.data import resolve_directory + """Resolve local/remote dataset directories to a local path.""" + from fair.utils.data import resolve_directory - return Path(str(resolve_directory(path_value, pattern="*"))) + if "://" in str(path_value): + return resolve_directory(path_value, pattern="*") return _to_local_path(path_value, purpose) def _resolve_input_file(path_value: str, purpose: str) -> Path: - """Resolve local or remote (S3/HTTP/etc.) files to a local path.""" - if "://" in str(path_value): - from fair.utils.data import resolve_path + """Resolve local/remote file paths to a local path.""" + from fair.utils.data import resolve_path - return Path(str(resolve_path(path_value))) + if "://" in str(path_value): + return resolve_path(path_value) return _to_local_path(path_value, purpose) +def _download_and_extract_zip(zip_url: str, dest_dir: Path) -> None: + """Download a ZIP URL and extract in dest_dir.""" + dest_dir.mkdir(parents=True, exist_ok=True) + zip_name = Path(zip_url.split("/")[-1]).name or "archive.zip" + zip_path = dest_dir / zip_name + urlretrieve(zip_url, zip_path) + with zipfile.ZipFile(zip_path, "r") as zf: + zf.extractall(dest_dir) + zip_path.unlink(missing_ok=True) + + def _extract_zip(zip_path: Path, dest_dir: Path) -> None: dest_dir.mkdir(parents=True, exist_ok=True) with zipfile.ZipFile(zip_path, "r") as zf: zf.extractall(dest_dir) -def _ensure_ramp_baseline(data_base_path: str, baseline_rel_path: str) -> Path: - """Return the directory that contains baseline weights; download under data_base_path if missing. +def resolve_model_href( + model_uri: str, + cache_dir: Path | None = None, +) -> str: + """Resolve model_uri to a local path. + + Supports: + - Local SavedModel directory → returned as-is + - Local .zip file containing SavedModel → extracted, returned as directory + - HTTP(S) URL to .zip → downloaded, extracted, cached, returned as directory + - Local / HTTP .onnx → downloaded (if needed) and returned as file path + """ + if not isinstance(model_uri, str): + raise TypeError("model_uri must be a string") + if cache_dir is not None and not isinstance(cache_dir, Path): + raise TypeError("cache_dir must be a pathlib.Path or None") + + cache_dir = cache_dir or _DEFAULT_MODEL_CACHE + is_http = model_uri.startswith(("http://", "https://")) + clean_uri = model_uri.split("?", 1)[0] + suffix = Path(clean_uri).suffix.lower() + + # ONNX: return as local file path. + if suffix == ".onnx": + if is_http: + cache_dir.mkdir(parents=True, exist_ok=True) + base_name = Path(clean_uri).name or "model.onnx" + dest = cache_dir / base_name + if not dest.is_file(): + urlretrieve(model_uri, dest) + return str(dest) + resolved = _to_local_path(model_uri, "model_uri").resolve() + if resolved.is_file(): + return str(resolved) + raise FileNotFoundError(f"ONNX model not found: {resolved}") + + # Zipped SavedModel. + if suffix == ".zip": + base_name = Path(clean_uri).name + stem = Path(base_name).stem or "ramp_model" + dest_dir = cache_dir / stem + for existing in dest_dir.rglob("saved_model.pb"): + return str(existing.parent) + + if is_http: + _download_and_extract_zip(model_uri, dest_dir) + else: + local_zip = _resolve_input_file(model_uri, "model_uri") + _extract_zip(local_zip, dest_dir) + + for sub in dest_dir.rglob("saved_model.pb"): + return str(sub.parent) + raise RuntimeError(f"Zip from {model_uri} did not contain a valid SavedModel") + + # Directory (SavedModel) - local or remote. + resolved_dir = ( + _resolve_input_directory(model_uri, "model_uri") if "://" in model_uri else _to_local_path(model_uri, "model_uri") + ).resolve() + if resolved_dir.is_dir() and (resolved_dir / "saved_model.pb").exists(): + return str(resolved_dir) + if resolved_dir.exists(): + raise FileNotFoundError(f"SavedModel directory missing saved_model.pb: {resolved_dir}") + raise FileNotFoundError(f"Model path not found: {resolved_dir}") + - ``baseline_rel_path`` is e.g. ``ramp-data/baseline/checkpoint.tf`` (file relative to RAMP_HOME). +def _ensure_ramp_baseline(base_model_weights: str, data_base_path: str | Path) -> Path: + """Return a local SavedModel directory for fine-tuning, downloading if necessary. + + ``base_model_weights`` may be an HTTP(S) .zip URL (preferred), a local .zip, or a SavedModel + directory. A pre-provisioned baseline under ``/app/ramp-data/baseline`` (from the RAMP + utilities Docker image) is used when present and no explicit URL is provided. """ - rel = Path(baseline_rel_path) - local_dir = Path(data_base_path) / rel.parent - local_ck = local_dir / rel.name - if local_ck.is_file() or (local_dir / "saved_model.pb").exists(): - return local_dir - image_ck = Path("/app") / baseline_rel_path - if image_ck.is_file(): - return image_ck.parent - _download_and_extract_zip(_DEFAULT_RAMP_BASELINE_URL, local_dir) - return local_dir + image_ck = Path("/app/ramp-data/baseline") + if not base_model_weights and (image_ck / "saved_model.pb").exists(): + return image_ck + if not base_model_weights: + base_model_weights = _DEFAULT_RAMP_BASELINE_URL + + return Path(resolve_model_href(base_model_weights, cache_dir=Path(data_base_path) / ".baseline_cache")) -_QUBVEL_EFFICIENTNET_RELEASE = ( - "https://github.com/qubvel/efficientnet/releases/download/v0.0.1/" -) + +# --------------------------------------------------------------------------- +# Keras / segmentation_models compatibility patches +# --------------------------------------------------------------------------- def _patch_keras_get_file_for_efficientnet_weights() -> None: @@ -140,106 +214,92 @@ def _get_file(fname, origin, *args, **kwargs): ku.get_file = _get_file -def _patch_predictor_savedmodel_loader_for_tf215() -> None: - """Patch fairpredictor's SavedModel directory loader for TF 2.15 compatibility. - - Why this patch exists: - Some fairpredictor versions load SavedModel directories using `keras.layers.TFSMLayer(...)`. - `TFSMLayer` is not available in TensorFlow 2.15's `tf.keras.layers`, so inference fails with: - AttributeError: module 'keras.api._v2.keras.layers' has no attribute 'TFSMLayer' +# --------------------------------------------------------------------------- +# Dataset materialization (chips + labels → hot_fair_utilities input layout) +# --------------------------------------------------------------------------- - We monkey-patch `predictor.prediction._load_keras_model` so directory-based SavedModels use - `tf.saved_model.load(...)` and a small `.predict(...)` wrapper. - We patch the module object (via importlib) to ensure downstream `from predictor.prediction import ...` - uses the patched function. - """ - import importlib - import os - from pathlib import Path +def _select_or_merge_labels(labels_path: Path, destination: Path) -> None: + """Materialize a single labels.geojson for hot_fair_utilities preprocess.""" + if labels_path.is_file(): + shutil.copy2(labels_path, destination) + return - import tensorflow as tf + if not labels_path.is_dir(): + raise FileNotFoundError(f"dataset_labels path not found: {labels_path}") - pred = importlib.import_module("predictor.prediction") - if getattr(pred, "_fair_models_tf215_savedmodel_patch_applied", False): + geojson_files = sorted(labels_path.glob("*.geojson")) + if not geojson_files: + raise FileNotFoundError(f"No .geojson files found in labels directory: {labels_path}") + if len(geojson_files) == 1: + shutil.copy2(geojson_files[0], destination) return - original_loader = getattr(pred, "_load_keras_model", None) - if original_loader is None: - raise RuntimeError("predictor.prediction._load_keras_model not found; fairpredictor API changed.") + import geopandas as gpd + import pandas as pd - def _safe_load_keras_model(keras_backend, path: str): - if os.path.isdir(path) and (Path(path) / "saved_model.pb").exists(): - # TF 2.15 (Keras 2.x) can load SavedModel directories directly; avoid TFSMLayer entirely. - return tf.keras.models.load_model(path, compile=False) + gdfs = [gpd.read_file(p) for p in geojson_files] + crs = gdfs[0].crs or "EPSG:4326" + merged = gpd.GeoDataFrame(pd.concat([g.to_crs(crs) for g in gdfs], ignore_index=True), crs=crs) + for col in merged.columns: + if col == "geometry": + continue + if pd.api.types.is_extension_array_dtype(merged[col].dtype): + merged[col] = merged[col].astype(object).where(merged[col].notna(), None) + merged.to_file(destination, driver="GeoJSON") - return original_loader(keras_backend, path) - _safe_load_keras_model._fair_models_tf215_savedmodel_loader = True # type: ignore[attr-defined] - pred._load_keras_model = _safe_load_keras_model - pred._fair_models_tf215_savedmodel_patch_applied = True +def _materialize_training_input(dataset_chips: str, dataset_labels: str, work_dir: Path) -> Path: + """Create the preprocess input folder with PNG chips and a single labels.geojson. + Accepts .tif/.tiff/.png chip files on disk; TIFFs are converted to 3-band PNGs while + preserving filename stem (e.g. OAM-{x}-{y}-{z}.png) so hot_fair_utilities label clipping + can parse tile ids. + """ + chips_dir = _resolve_input_directory(dataset_chips, "dataset_chips") + labels_path = _resolve_input_file(dataset_labels, "dataset_labels") -def resolve_model_href( - model_uri: str, - cache_dir: Optional[Path] = None, -) -> str: - """Resolve model_uri to a local SavedModel directory path. + input_dir = work_dir / "input" + if input_dir.exists(): + shutil.rmtree(input_dir) + input_dir.mkdir(parents=True, exist_ok=True) - Supports: - - Local path: returned as-is if it exists - - Direct HTTP(S) URL to .zip: downloaded, extracted, cached + tif_paths = sorted(list(chips_dir.glob("*.tif")) + list(chips_dir.glob("*.tiff"))) + png_paths = sorted(chips_dir.glob("*.png")) - Returns the absolute path to the SavedModel directory. - """ - if not isinstance(model_uri, str): - raise TypeError("model_uri must be a string") - if cache_dir is not None and not isinstance(cache_dir, Path): - raise TypeError("cache_dir must be a pathlib.Path or None") + if tif_paths: + import numpy as np + import rasterio + from PIL import Image - cache_dir = cache_dir or _DEFAULT_MODEL_CACHE + for tif_path in tif_paths: + png_path = input_dir / (tif_path.stem + ".png") + with rasterio.open(tif_path) as src: + data = src.read() + if data.shape[0] < 3: + continue + rgb = np.transpose(data[:3], (1, 2, 0)) + rgb = (rgb * 255).astype(np.uint8) if rgb.max() <= 1.0 else np.clip(rgb, 0, 255).astype(np.uint8) + Image.fromarray(rgb).save(png_path) - is_http = model_uri.startswith("http://") or model_uri.startswith("https://") - - # SavedModel directory (local or remote) - if not Path(model_uri.split("?", 1)[0]).suffix and not model_uri.lower().endswith(".zip"): - resolved_dir = (_resolve_input_directory(model_uri, "model_uri") if "://" in model_uri else _to_local_path(model_uri, "model_uri")).resolve() - if (resolved_dir / "saved_model.pb").exists(): - return str(resolved_dir) - if resolved_dir.exists(): - # Let downstream error messages be explicit. - raise FileNotFoundError(f"SavedModel directory missing saved_model.pb: {resolved_dir}") - raise FileNotFoundError(f"Model path not found: {resolved_dir}") - - # Remote/local ZIP containing SavedModel - if model_uri.lower().endswith(".zip"): - base_name = Path(model_uri.split("?", 1)[0]).name - stem = Path(base_name).stem or "ramp_model" - dest_dir = cache_dir / stem - if any(dest_dir.rglob("saved_model.pb")): - for sub in dest_dir.rglob("saved_model.pb"): - return str(sub.parent) + for png_path in png_paths: + shutil.copy2(png_path, input_dir / png_path.name) - if is_http: - _download_and_extract_zip(model_uri, dest_dir) - else: - local_zip = _resolve_input_file(model_uri, "model_uri") - _extract_zip(local_zip, dest_dir) + if not list(input_dir.glob("*.png")): + raise FileNotFoundError(f"No train chips (.tif/.tiff/.png) found in {chips_dir}") - for sub in dest_dir.rglob("saved_model.pb"): - return str(sub.parent) - raise RuntimeError(f"Zip from {model_uri} did not contain a valid SavedModel") + _select_or_merge_labels(labels_path, input_dir / "labels.geojson") + return input_dir - # Plain local path to file/dir (fallback) - resolved = _to_local_path(model_uri, "model_uri").resolve() - if resolved.exists(): - return str(resolved) - raise FileNotFoundError(f"Model path not found: {resolved}") - raise ValueError( - f"Unsupported model_uri: {model_uri}. " - "Use a local path or an HTTP(S) URL to a .zip file containing the SavedModel." - ) +def _training_cache_dir(dataset_chips: str, dataset_labels: str) -> Path: + cache_key = hashlib.sha256(f"{dataset_chips}|{dataset_labels}".encode()).hexdigest()[:16] + return Path(tempfile.gettempdir()) / f"ramp_training_{cache_key}" + + +# --------------------------------------------------------------------------- +# Preprocess / postprocess (STAC pre/post_processing_function references) +# --------------------------------------------------------------------------- def preprocess( @@ -248,14 +308,12 @@ def preprocess( boundary_width: int = 3, contact_spacing: int = 8, ) -> str: - """Preprocess OAM chips + labels for RAMP training. + """Preprocess OAM PNG chips + labels into RAMP 4-class multimasks. - Step 1 — Georeference PNGs → chips/*.tif (EPSG:3857) - Step 2 — Reproject + clip labels → labels/*.geojson (per chip) - Step 3 — Generate 4-channel sparse multimasks → multimasks/*.mask.tif - Classes: 0=background, 1=building, 2=boundary, 3=contact-point - - Returns the preprocessed output directory path. + Emits: + - preprocessed/chips/*.tif (georeferenced RGB chips, EPSG:3857) + - preprocessed/labels/*.geojson (per-chip labels, reprojected + clipped) + - preprocessed/multimasks/*.mask.tif (4-class sparse categorical masks) """ from hot_fair_utilities import preprocess as _preprocess @@ -273,11 +331,8 @@ def preprocess( return output_path -def postprocess(prediction_masks_dir: str, output_dir: str) -> dict: - """Run fairpredictor-style postprocessing and return merged prediction GeoJSON content.""" - import json - - # geomltoolkits>=2 moved modules (no geomltoolkits.regularizer/utils) +def postprocess(prediction_masks_dir: str, output_dir: str) -> dict[str, Any]: + """Merge prediction TIFF tiles into a building-footprint GeoJSON (EPSG:4326).""" from geomltoolkits.geometry.validate import validate_polygon_geometries from geomltoolkits.raster.merge import merge_rasters from geomltoolkits.raster.morphology import morphological_cleaning @@ -289,7 +344,7 @@ def postprocess(prediction_masks_dir: str, output_dir: str) -> dict: pred_tifs = sorted(pred_dir.glob("*.tif")) if not pred_tifs: - raise RuntimeError(f"No *.tif files found in {pred_dir}") + return {"type": "FeatureCollection", "features": []} merged_mask_path = out_dir / "merged_prediction_mask.tif" merged_geojson_path = out_dir / "predictions.geojson" @@ -308,11 +363,8 @@ def postprocess(prediction_masks_dir: str, output_dir: str) -> dict: geojson_dict = json.loads(gdf.to_json()) if not geojson_dict.get("features"): - # geomltoolkits validator raises on empty FeatureCollections; treat this as a valid - # "no buildings found" prediction and keep the already-written GeoJSON file. if not merged_geojson_path.is_file(): - with merged_geojson_path.open("w", encoding="utf-8") as fh: - json.dump(geojson_dict, fh) + merged_geojson_path.write_text(json.dumps(geojson_dict), encoding="utf-8") return geojson_dict if gdf.crs and gdf.crs != "EPSG:4326": @@ -320,7 +372,7 @@ def postprocess(prediction_masks_dir: str, output_dir: str) -> dict: elif not gdf.crs: gdf.set_crs("EPSG:3857", inplace=True) gdf = gdf.to_crs("EPSG:4326") - # Pandas extension dtypes break some Fiona GeoJSON writes; coerce to object. + import pandas as pd for col in gdf.columns: @@ -336,173 +388,89 @@ def postprocess(prediction_masks_dir: str, output_dir: str) -> dict: ) if isinstance(validated_geojson, str) and Path(validated_geojson).is_file(): if Path(validated_geojson) != merged_geojson_path: - copy2(validated_geojson, merged_geojson_path) - with merged_geojson_path.open("r", encoding="utf-8") as file_handle: - return json.load(file_handle) + shutil.copy2(validated_geojson, merged_geojson_path) + return json.loads(merged_geojson_path.read_text(encoding="utf-8")) if isinstance(validated_geojson, dict): return validated_geojson return json.loads(gdf.to_json()) -def _load_hyperparams_from_stac(stac_item_path: str) -> dict: - """Load mlm:hyperparameters from a STAC Item JSON file. - - Relative paths are resolved against /workspace (container) or cwd (local). - """ - import json - - path = _to_local_path(stac_item_path, "stac_item_path") - if not path.is_absolute(): - for base in (Path("/workspace"), Path.cwd()): - candidate = base / path - if candidate.is_file(): - path = candidate - break - else: - path = Path("/workspace") / path - with open(path, encoding="utf-8") as f: - item = json.load(f) - return dict(item.get("properties", {}).get("mlm:hyperparameters", {})) - - -@step -def run_preprocessing( - input_path: str, - output_path: str, - boundary_width: int = 3, - contact_spacing: int = 8, -) -> List[Tuple[Any, Any]]: - """Georeference OAM chips and return chip/mask data arrays.""" - import rasterio +# --------------------------------------------------------------------------- +# Train / eval / export helpers (stateful TF pieces stay behind lazy imports) +# --------------------------------------------------------------------------- - preprocessed = Path(preprocess(input_path, output_path, boundary_width, contact_spacing)) - chips_dir = preprocessed / "chips" - masks_dir = preprocessed / "multimasks" - data_loader: list[tuple[Any, Any]] = [] - for chip in sorted(chips_dir.glob("*.tif")): - mask = masks_dir / f"{chip.stem}.mask.tif" - if mask.is_file(): - with rasterio.open(chip) as chip_src: - chip_data = chip_src.read() - with rasterio.open(mask) as mask_src: - mask_data = mask_src.read() - data_loader.append((chip_data, mask_data)) - return data_loader +def _prepare_training_split( + dataset_chips: str, + dataset_labels: str, + hyperparameters: dict[str, Any], + force_rebuild: bool = False, +) -> dict[str, Any]: + """Preprocess + split chips/labels into RAMP train/val layout; return split_info.""" + from hot_fair_utilities.training.ramp.prepare_data import split_training_2_validation -@step -def train_model( - data_base_path: str, - preprocessed_path: str, - data_loader: Optional[List[Tuple[Any, Any]]] = None, - stac_item_path: str = "models/ramp/stac-item.json", - val_fraction: Optional[float] = None, - split_info: Optional[Dict[str, Any]] = None, -) -> Any: - """ZenML step wrapper for RAMP training; returns the best Keras SavedModel checkpoint.""" - if data_loader is not None and not data_loader: - raise RuntimeError("Preprocessing returned an empty dataloader; no chip/mask pairs found.") - return train_ramp_model( - data_base_path=data_base_path, - preprocessed_path=preprocessed_path, - stac_item_path=stac_item_path, - val_fraction=val_fraction, - split_info=split_info, - log_zenml_step_metadata=True, + val_fraction = float( + hyperparameters.get( + "training.val_ratio", + hyperparameters.get("val_fraction", hyperparameters.get("val_ratio", 0.15)), + ) ) + if not 0.0 < val_fraction < 1.0: + raise ValueError("val_fraction must be in (0.0, 1.0)") + boundary_width = int( + hyperparameters.get("training.boundary_width", hyperparameters.get("boundary_width", 3)) + ) + contact_spacing = int( + hyperparameters.get("training.contact_spacing", hyperparameters.get("contact_spacing", 8)) + ) + seed = int(hyperparameters.get("training.split_seed", hyperparameters.get("split_seed", 42))) + work_dir = _training_cache_dir(dataset_chips, dataset_labels) + preprocessed_dir = work_dir / "preprocessed" + ramp_train_dir = work_dir / "ramp_training_work" -@step -def split_dataset( - preprocessed_path: str, - output_path: str, - hyperparameters: Optional[Dict[str, Any]] = None, -) -> Annotated[Dict[str, Any], "split_info"]: - """Create train/validation directories for RAMP and return split metadata. - - Uses the existing utilities implementation: - `hot_fair_utilities.training.ramp.prepare_data.split_training_2_validation`. - """ - from hot_fair_utilities.training.ramp.prepare_data import split_training_2_validation - - hyperparameters = hyperparameters or {} - preprocessed_dir = _to_local_path(preprocessed_path, "preprocessed_path") - if not preprocessed_dir.is_dir(): - raise FileNotFoundError(f"Preprocessed directory not found: {preprocessed_dir}") - - out_dir = _to_local_path(output_path, "output_path") - ramp_train_dir = out_dir / "ramp_training_work" + if force_rebuild and work_dir.exists(): + shutil.rmtree(work_dir) - split_training_2_validation( - str(preprocessed_dir), - str(ramp_train_dir), - multimasks=True, - ) + if not ramp_train_dir.exists(): + work_dir.mkdir(parents=True, exist_ok=True) + input_dir = _materialize_training_input(dataset_chips, dataset_labels, work_dir) + preprocess(str(input_dir), str(preprocessed_dir), boundary_width, contact_spacing) + split_training_2_validation(str(preprocessed_dir), str(ramp_train_dir), multimasks=True) train_count = len(list((ramp_train_dir / "chips").glob("*.tif"))) val_count = len(list((ramp_train_dir / "val-chips").glob("*.tif"))) - total = train_count + val_count - val_ratio = (float(val_count) / float(total)) if total else 0.0 - split_info: Dict[str, Any] = { + return { "strategy": "random", - "val_ratio": val_ratio, + "val_ratio": val_fraction, + "seed": seed, "train_count": train_count, "val_count": val_count, - "seed": int(hyperparameters.get("split_seed", 42)), - "ramp_train_dir": str(ramp_train_dir), - "source": "hot_fair_utilities.training.ramp.prepare_data.split_training_2_validation", + "description": "Preprocess chips+labels into 4-class multimasks, then random train/val split.", + "_work_dir": str(work_dir), + "_preprocessed_dir": str(preprocessed_dir), + "_ramp_train_dir": str(ramp_train_dir), } - log_metadata(metadata={"fair/split": split_info}) - return split_info def train_ramp_model( - data_base_path: str, - preprocessed_path: str, - stac_item_path: str = "models/ramp/stac-item.json", - val_fraction: Annotated[Optional[float], "0.0 <= val_fraction <= 0.5"] = None, - num_epochs: Annotated[Optional[int], "1 <= num_epochs <= 20"] = None, - batch_size: Annotated[Optional[int], "1 <= batch_size <= 8"] = None, - backbone: Optional[str] = None, - early_stopping_patience: Annotated[Optional[int], "1 <= early_stopping_patience <= 20"] = None, - split_info: Optional[Dict[str, Any]] = None, - log_zenml_step_metadata: bool = False, -) -> Any: - """Fine-tune EfficientNetB0 + U-Net on 4-class multimask chips. - - Uses hot_fair_utilities.training.ramp for training orchestration. - RAMP_CONFIG is used as the base configuration; hyperparameters from the - STAC Item and keyword arguments selectively override the base. - - Val split is handled internally by split_training_2_validation: - preprocessed_path → ramp_training_work/ (train + val-chips + val-multimasks). - - Sets ``RAMP_HOME`` before importing training helpers: ``run_training`` caches - ``working_ramp_home`` at import time, so ``/app`` is used when the GHCR baseline exists there. - - If the RAMP baseline exists under the hot-fair-utilities image (``/app/ramp-data/baseline/``) - it is used for fine-tuning. Otherwise weights are resolved under ``data_base_path`` or downloaded. - - Returns the best checkpoint as a loaded ``tf.keras.Model`` (SavedModel on disk is loaded with compile=False). - """ - import os - - # run_training.run_training sets ``working_ramp_home = os.environ["RAMP_HOME"]`` at *import* time. - # That value is used for ``Path(working_ramp_home) / saved_model_path`` when loading the baseline. - # Dataset paths in cfg are absolute (from manage_fine_tuning_config), so they still resolve correctly. - # Docker + GHCR base: baseline lives under /app; do not rely on /workspace/ramp-data (bind mount hides it). - resolved_base = str(Path(data_base_path).resolve()) - image_baseline_ck = Path("/app/ramp-data/baseline/checkpoint.tf") - if image_baseline_ck.is_file(): + ramp_train_dir: str, + base_model_weights: str, + hyperparameters: dict[str, Any], + data_base_path: str | None = None, +) -> Path: + """Fine-tune EfficientNetB0 + U-Net and return the best SavedModel directory path.""" + # run_training reads RAMP_HOME at import time; set it first so saved_model lookups resolve. + data_base_path = str(Path(data_base_path).resolve()) if data_base_path else str(Path(ramp_train_dir).resolve()) + image_baseline_ck = Path("/app/ramp-data/baseline/saved_model.pb") + if image_baseline_ck.exists(): os.environ["RAMP_HOME"] = "/app" else: - os.environ["RAMP_HOME"] = resolved_base + os.environ["RAMP_HOME"] = data_base_path - # segmentation_models configures efficientnet at import time via SM_FRAMEWORK. os.environ.setdefault("SM_FRAMEWORK", "tf.keras") - import tensorflow as tf _patch_keras_get_file_for_efficientnet_weights() import segmentation_models as sm @@ -515,248 +483,569 @@ def train_ramp_model( sm.set_framework("tf.keras") - # Load STAC hyperparams and apply call-site overrides - hyperparams = _load_hyperparams_from_stac(stac_item_path) - if val_fraction is not None: - if not 0.0 <= val_fraction <= 0.5: - raise ValueError("val_fraction must be in [0.0, 0.5]") - hyperparams["val_fraction"] = val_fraction - if num_epochs is not None: - if not 1 <= num_epochs <= 20: - raise ValueError("num_epochs must be in [1, 20] for RAMP runtime limits") - hyperparams["num_epochs"] = num_epochs - if batch_size is not None: - if not 1 <= batch_size <= 8: - raise ValueError("batch_size must be in [1, 8] for RAMP runtime limits") - hyperparams["batch_size"] = batch_size - if backbone is not None: - hyperparams["backbone"] = backbone - if early_stopping_patience is not None: - if not 1 <= early_stopping_patience <= 20: - raise ValueError("early_stopping_patience must be in [1, 20]") - hyperparams["early_stopping_patience"] = early_stopping_patience - - # Resolve effective values from STAC overrides or RAMP_CONFIG defaults - eff_epochs = hyperparams.get("num_epochs", hyperparams.get("epochs", RAMP_CONFIG["num_epochs"])) - eff_batch = hyperparams.get("batch_size", RAMP_CONFIG["batch_size"]) - eff_backbone = hyperparams.get("backbone", RAMP_CONFIG["model"]["model_fn_parms"]["backbone"]) - eff_lr = hyperparams.get("learning_rate", RAMP_CONFIG["optimizer"]["optimizer_fn_parms"]["learning_rate"]) - eff_patience = hyperparams.get( - "early_stopping_patience", - RAMP_CONFIG["early_stopping"]["early_stopping_parms"]["patience"], + epochs = int(hyperparameters.get("training.epochs", hyperparameters.get("epochs", RAMP_CONFIG["num_epochs"]))) + batch_size = int( + hyperparameters.get("training.batch_size", hyperparameters.get("batch_size", RAMP_CONFIG["batch_size"])) + ) + backbone = str( + hyperparameters.get( + "training.backbone", + hyperparameters.get("backbone", RAMP_CONFIG["model"]["model_fn_parms"]["backbone"]), + ) + ) + learning_rate = float( + hyperparameters.get( + "training.learning_rate", + hyperparameters.get( + "learning_rate", RAMP_CONFIG["optimizer"]["optimizer_fn_parms"]["learning_rate"] + ), + ) + ) + patience = int( + hyperparameters.get( + "training.early_stopping_patience", + hyperparameters.get( + "early_stopping_patience", RAMP_CONFIG["early_stopping"]["early_stopping_parms"]["patience"] + ), + ) ) - if not 1 <= int(eff_epochs) <= 20: - raise ValueError(f"Resolved num_epochs={eff_epochs} is outside [1, 20]") - if not 1 <= int(eff_batch) <= 8: - raise ValueError(f"Resolved batch_size={eff_batch} is outside [1, 8]") - if not 1 <= int(eff_patience) <= 20: - raise ValueError(f"Resolved early_stopping_patience={eff_patience} is outside [1, 20]") - - # Prefer a precomputed split (CI-visible split_dataset step), fallback to internal split for compatibility. - if split_info and split_info.get("ramp_train_dir"): - ramp_train_dir = str(_to_local_path(str(split_info["ramp_train_dir"]), "ramp_train_dir")) + if not 1 <= epochs <= 200: + raise ValueError(f"Resolved epochs={epochs} is outside [1, 200]") + if not 1 <= batch_size <= 64: + raise ValueError(f"Resolved batch_size={batch_size} is outside [1, 64]") + if not 1 <= patience <= 50: + raise ValueError(f"Resolved early_stopping_patience={patience} is outside [1, 50]") + + cfg = manage_fine_tuning_config(ramp_train_dir, epochs, batch_size, freeze_layers=False, multimasks=True) + cfg["model"]["model_fn_parms"]["backbone"] = backbone + cfg["optimizer"]["optimizer_fn_parms"]["learning_rate"] = learning_rate + cfg["early_stopping"]["early_stopping_parms"]["patience"] = patience + + if cfg["saved_model"]["use_saved_model"]: + baseline_dir = _ensure_ramp_baseline(base_model_weights, data_base_path) + cfg["saved_model"]["use_saved_model"] = (baseline_dir / "saved_model.pb").exists() + + run_main_train_code(cfg) + _final_accuracy, final_model_path = extract_highest_accuracy_model(ramp_train_dir) + return Path(final_model_path) + + +def _zip_savedmodel_dir(saved_model_dir: Path) -> bytes: + """Zip a SavedModel directory into bytes for ZenML artifact persistence.""" + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: + for p in saved_model_dir.rglob("*"): + if p.is_file(): + zf.write(p, arcname=p.relative_to(saved_model_dir)) + return buf.getvalue() + + +def _unzip_savedmodel_bytes(blob: bytes) -> Path: + """Extract a SavedModel zipped as bytes to a temp directory and return its path.""" + dest = Path(tempfile.mkdtemp(prefix="ramp_savedmodel_")) + with zipfile.ZipFile(io.BytesIO(blob), "r") as zf: + zf.extractall(dest) + if (dest / "saved_model.pb").exists(): + return dest + for candidate in dest.rglob("saved_model.pb"): + return candidate.parent + raise RuntimeError("Zipped bytes do not contain a SavedModel (no saved_model.pb found).") + + +def _restore_checkpoint(trained_model: Any) -> Path: + """Restore a trained RAMP SavedModel from bytes, a SavedModel directory, or a .zip file.""" + if isinstance(trained_model, bytes): + return _unzip_savedmodel_bytes(trained_model) + if isinstance(trained_model, (str, Path)): + p = Path(str(trained_model)) + if p.is_dir() and (p / "saved_model.pb").exists(): + return p + if p.is_file() and p.suffix.lower() == ".zip": + return _unzip_savedmodel_bytes(p.read_bytes()) + raise TypeError(f"Cannot restore RAMP checkpoint from {type(trained_model).__name__}") + + +def _convert_savedmodel_to_onnx_bytes(saved_model_dir: Path, opset: int = 13) -> bytes: + """Convert a TF SavedModel directory to ONNX bytes via tf2onnx.""" + import tf2onnx + + with tempfile.TemporaryDirectory() as tmp: + onnx_path = Path(tmp) / "model.onnx" + tf2onnx.convert.from_saved_model( + str(saved_model_dir), + output_path=str(onnx_path), + opset=opset, + ) + return onnx_path.read_bytes() + + +# --------------------------------------------------------------------------- +# ONNX serving: predict(session, input_images, params) -> FeatureCollection +# --------------------------------------------------------------------------- + + +def _build_feature_collection(features: list[dict[str, Any]]) -> dict[str, Any]: + return {"type": "FeatureCollection", "features": features} + + +def _extract_ramp_shapes(session: Any) -> tuple[str, int, int]: + """Return (input_name, input_height, input_width) for a RAMP ONNX session (NHWC).""" + input_meta = session.get_inputs()[0] + shape = input_meta.shape + if len(shape) != 4: + raise RuntimeError(f"Unexpected ONNX input shape: {shape}") + # RAMP ONNX is [batch, H, W, bands] (channels last) — keep this order. + height = int(shape[1]) if isinstance(shape[1], int) and shape[1] > 0 else 256 + width = int(shape[2]) if isinstance(shape[2], int) and shape[2] > 0 else 256 + return input_meta.name, height, width + + +def _prepare_onnx_image(img_path: Path, input_height: int, input_width: int) -> tuple[Any, Any, Any]: + """Load an RGB chip and return (batch, transform, meta) for ONNX inference.""" + import numpy as np + import rasterio + from PIL import Image + + with rasterio.open(img_path) as src: + arr = src.read([1, 2, 3]).astype(np.float32) / 255.0 + transform = src.transform + crs = src.crs + src_height = src.height + src_width = src.width + + resized = [ + np.asarray(Image.fromarray(arr[c]).resize((input_width, input_height), Image.Resampling.BILINEAR)) + for c in range(arr.shape[0]) + ] + hwc = np.stack(resized, axis=-1).astype(np.float32) # NHWC + batch = hwc[np.newaxis, ...] + return batch, transform, (src_width, src_height, input_width, input_height, crs) + + +def _decode_ramp_building_mask( + output: Any, + input_height: int, + input_width: int, + src_height: int, + src_width: int, + min_class_value: int = 1, +) -> Any: + """Decode a RAMP ONNX output tensor into a (src_height, src_width) uint8 building mask. + + Accepts either a 4-class softmax/logits tensor [B,H,W,C] or a [B,H,W,1] class-index tensor. + Pixels with class >= ``min_class_value`` (default 1=building) become 1; others 0. + """ + import numpy as np + from PIL import Image + + arr = np.asarray(output) + if arr.ndim == 5: + arr = arr[0] + if arr.ndim == 4 and arr.shape[0] == 1: + arr = arr[0] + if arr.ndim == 3 and arr.shape[-1] > 1: + class_idx = arr.argmax(axis=-1) + elif arr.ndim == 3 and arr.shape[-1] == 1: + class_idx = arr[..., 0] + elif arr.ndim == 2: + class_idx = arr else: - from hot_fair_utilities.training.ramp.prepare_data import split_training_2_validation + raise RuntimeError(f"Unexpected RAMP ONNX output shape: {arr.shape}") + + class_idx = np.asarray(class_idx).astype(np.int32) + # Collapse multiclass (background=0, building=1, boundary=2, contact=3) to binary building. + binary = (class_idx == min_class_value).astype(np.uint8) + if binary.shape != (src_height, src_width): + resized = Image.fromarray(binary * 255).resize((src_width, src_height), Image.Resampling.NEAREST) + binary = (np.asarray(resized) > 127).astype(np.uint8) + return binary - ramp_train_dir_path = _to_local_path( - str(Path(data_base_path) / "ramp_training_work"), - "ramp_training_work", + +def _vectorize_binary_mask(mask: Any, transform: Any, crs: Any, confidence: float) -> list[dict[str, Any]]: + import numpy as np + import rasterio.features + from pyproj import Transformer + + mask_uint8 = np.asarray(mask).astype(np.uint8) + transformer = Transformer.from_crs(crs, "EPSG:4326", always_xy=True) if crs and str(crs) != "EPSG:4326" else None + + features: list[dict[str, Any]] = [] + for geom, value in rasterio.features.shapes(mask_uint8, transform=transform): + if int(value) < 1: + continue + if transformer is not None: + coords = geom["coordinates"] + geom["coordinates"] = [[list(transformer.transform(x, y)) for x, y in ring] for ring in coords] + features.append( + { + "type": "Feature", + "properties": {"class": 1, "confidence": round(confidence, 4)}, + "geometry": geom, + } ) - if ramp_train_dir_path.exists(): - rmtree(ramp_train_dir_path) - ramp_train_dir = str(ramp_train_dir_path) - split_training_2_validation( - str(_to_local_path(preprocessed_path, "preprocessed_path")), - ramp_train_dir, - multimasks=True, + return features + + +def predict(session: Any, input_images: str, params: dict[str, Any]) -> dict[str, Any]: + """Run RAMP ONNX inference and return a FeatureCollection of building polygons. + + Required by ``fair.serve.base``. ``session`` is an ``onnxruntime.InferenceSession`` built by + the serving layer from the STAC ``assets.model`` ONNX artifact. + """ + from fair.utils.data import resolve_directory + + confidence_threshold = float(params.get("confidence_threshold", 0.5)) + min_class_value = int(params.get("min_class_value", 1)) + + input_name, input_height, input_width = _extract_ramp_shapes(session) + input_dir = resolve_directory(input_images) + patterns = ("*.png", "*.tif", "*.tiff", "*.jpg") + img_paths = sorted(p for pat in patterns for p in input_dir.glob(pat)) + if not img_paths: + raise FileNotFoundError(f"No input images found in {input_dir}") + + features: list[dict[str, Any]] = [] + for img_path in img_paths: + batch, transform, meta = _prepare_onnx_image(img_path, input_height, input_width) + src_width, src_height, _iw, _ih, crs = meta + outputs = session.run(None, {input_name: batch}) + if not outputs: + continue + mask = _decode_ramp_building_mask( + outputs[0], + input_height=input_height, + input_width=input_width, + src_height=src_height, + src_width=src_width, + min_class_value=min_class_value, ) + features.extend(_vectorize_binary_mask(mask, transform, crs, confidence_threshold)) + return _build_feature_collection(features) - # Build config from RAMP_CONFIG via the utilities helper, then apply overrides - cfg = manage_fine_tuning_config(ramp_train_dir, eff_epochs, eff_batch, freeze_layers=False, multimasks=True) - cfg["model"]["model_fn_parms"]["backbone"] = eff_backbone - cfg["optimizer"]["optimizer_fn_parms"]["learning_rate"] = eff_lr - cfg["early_stopping"]["early_stopping_parms"]["patience"] = eff_patience - # Use baseline checkpoint for fine-tuning if present, else train from scratch - saved_rel = cfg["saved_model"]["saved_model_path"] - baseline_dir: Path - if cfg["saved_model"]["use_saved_model"]: - baseline_dir = _ensure_ramp_baseline( - data_base_path=data_base_path, - baseline_rel_path=saved_rel, +# --------------------------------------------------------------------------- +# ZenML @step primitives +# --------------------------------------------------------------------------- + + +@step +def split_dataset( + dataset_chips: str, + dataset_labels: str, + hyperparameters: dict[str, Any], +) -> Annotated[dict[str, Any], "split_info"]: + """Preprocess chips+labels and create the RAMP train/val layout.""" + split_info = _prepare_training_split(dataset_chips, dataset_labels, hyperparameters) + log_metadata(metadata={"fair/split": {k: v for k, v in split_info.items() if not k.startswith("_")}}) + return split_info + + +@step +def train_model( + dataset_chips: str, + dataset_labels: str, + base_model_weights: str, + hyperparameters: dict[str, Any], + split_info: dict[str, Any], + num_classes: int = 4, + model_name: str | None = None, + base_model_id: str | None = None, + dataset_id: str | None = None, +) -> Annotated[bytes, "trained_model"]: + """Fine-tune RAMP EfficientNetB0 U-Net; return the best SavedModel as zipped bytes.""" + _ = (num_classes, model_name, base_model_id, dataset_id) + + ramp_train_dir = Path(split_info["_ramp_train_dir"]) + if not ramp_train_dir.exists(): + split_info = _prepare_training_split( + dataset_chips, dataset_labels, hyperparameters, force_rebuild=True ) - ck = baseline_dir / Path(saved_rel).name - cfg["saved_model"]["use_saved_model"] = ck.is_file() or (baseline_dir / "saved_model.pb").exists() + ramp_train_dir = Path(split_info["_ramp_train_dir"]) + + work_dir = split_info.get("_work_dir") or str(ramp_train_dir.parent) + final_model_path = train_ramp_model( + ramp_train_dir=str(ramp_train_dir), + base_model_weights=base_model_weights, + hyperparameters=hyperparameters, + data_base_path=work_dir, + ) + saved_model_dir = final_model_path if final_model_path.is_dir() else final_model_path.parent + if not (saved_model_dir / "saved_model.pb").exists(): + raise RuntimeError(f"Expected SavedModel at {saved_model_dir}; not found.") - run_main_train_code(cfg) - final_accuracy, final_model_path = extract_highest_accuracy_model(ramp_train_dir) + blob = _zip_savedmodel_dir(saved_model_dir) + log_metadata(metadata={"saved_model_dir": str(saved_model_dir), "checkpoint_bytes": len(blob)}) + return blob - if log_zenml_step_metadata: - log_metadata( - metadata={ - "best_val_accuracy": float(final_accuracy), - "best_model_path": str(final_model_path), - } + +@step +def evaluate_model( + trained_model: Any, + dataset_chips: str, + dataset_labels: str, + hyperparameters: dict[str, Any], + split_info: dict[str, Any], + class_names: list[str] | None = None, +) -> Annotated[dict[str, Any], "metrics"]: + """Compute per-pixel building-class metrics on the validation split.""" + _ = class_names + + import numpy as np + import rasterio + + ramp_train_dir = Path(split_info.get("_ramp_train_dir", "")) + if not ramp_train_dir.exists(): + split_info = _prepare_training_split( + dataset_chips, dataset_labels, hyperparameters, force_rebuild=True ) + ramp_train_dir = Path(split_info["_ramp_train_dir"]) + + val_chips_dir = ramp_train_dir / "val-chips" + val_masks_dir = ramp_train_dir / "val-multimasks" + pairs: list[tuple[Path, Path]] = [] + for chip in sorted(val_chips_dir.glob("*.tif")): + mask = val_masks_dir / f"{chip.stem}.mask.tif" + if mask.is_file(): + pairs.append((chip, mask)) + + saved_model_dir = _restore_checkpoint(trained_model) + + if not pairs: + # No val data to evaluate against (e.g. CI mocks); return zeroed metrics with the + # required fair:* keys so downstream validation still sees the expected schema. + zero_metrics: dict[str, Any] = { + "fair:accuracy": 0.0, + "fair:mean_iou": 0.0, + "fair:precision": 0.0, + "fair:recall": 0.0, + } + log_metadata(metadata=zero_metrics) + return zero_metrics + + import tensorflow as tf + + model = tf.keras.models.load_model(str(saved_model_dir), compile=False) + + tp = fp = fn = 0 + correct = total = 0 + for chip_path, mask_path in pairs: + with rasterio.open(chip_path) as src: + chip = src.read([1, 2, 3]).astype(np.float32) / 255.0 + with rasterio.open(mask_path) as src: + gt = src.read(1).astype(np.int32) + batch = np.transpose(chip, (1, 2, 0))[np.newaxis, ...] + pred = model.predict(batch, verbose=0) + pred_arr = np.asarray(pred) + if pred_arr.ndim == 4 and pred_arr.shape[-1] > 1: + pred_idx = pred_arr[0].argmax(axis=-1) + elif pred_arr.ndim == 4 and pred_arr.shape[-1] == 1: + pred_idx = pred_arr[0, ..., 0].astype(np.int32) + else: + pred_idx = pred_arr[0].astype(np.int32) + + gt_bin = (gt == 1).astype(np.uint8) + pr_bin = (pred_idx == 1).astype(np.uint8) + tp += int(((gt_bin == 1) & (pr_bin == 1)).sum()) + fp += int(((gt_bin == 0) & (pr_bin == 1)).sum()) + fn += int(((gt_bin == 1) & (pr_bin == 0)).sum()) + correct += int((pred_idx == gt).sum()) + total += int(gt.size) + + precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0 + recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0 + iou = tp / (tp + fp + fn) if (tp + fp + fn) > 0 else 0.0 + accuracy = correct / total if total > 0 else 0.0 + + metrics_dict: dict[str, Any] = { + "fair:accuracy": float(accuracy), + "fair:mean_iou": float(iou), + "fair:precision": float(precision), + "fair:recall": float(recall), + } + log_metadata(metadata=metrics_dict) + return metrics_dict + + +@step +def export_onnx(trained_model: Any) -> Annotated[bytes, "onnx_model"]: + """Convert the trained RAMP SavedModel to ONNX bytes and validate.""" + import onnx - return tf.keras.models.load_model(str(final_model_path), compile=False) + saved_model_dir = _restore_checkpoint(trained_model) + onnx_bytes = _convert_savedmodel_to_onnx_bytes(saved_model_dir) + onnx.checker.check_model(onnx.load_from_string(onnx_bytes)) + log_metadata(metadata={"onnx_bytes": len(onnx_bytes)}) + return onnx_bytes @step def run_inference( - model_uri: Union[str, Path, Any], - input_path: str, + model_uri: str | Path | Any, + input_images: str, prediction_path: str, output_dir: str, - model_cache_dir: Optional[str] = None, -) -> Dict[str, Any]: - """ZenML step wrapper for RAMP inference returning final GeoJSON content. - - model_uri may be a STAC/local path, HTTP(S) .zip URL, or a ``tf.keras.Model`` from training. - """ + confidence: float = 0.5, + model_cache_dir: str | None = None, +) -> Annotated[dict[str, Any], "predictions"]: + """Native-TF inference over georeferenced chips → building-footprint GeoJSON.""" return infer_ramp_model( model_uri=model_uri, - input_path=input_path, + input_path=input_images, prediction_path=prediction_path, output_dir=output_dir, + confidence=confidence, model_cache_dir=model_cache_dir, ) -def infer_ramp_model( - model_uri: Union[str, Path, Any], - input_path: str, - prediction_path: str, - output_dir: str, - model_cache_dir: Optional[str] = None, - max_chips: Optional[int] = None, -) -> Dict[str, Any]: - """Run fairpredictor inference and return final merged GeoJSON content. +def _patch_predictor_savedmodel_loader_for_tf215() -> None: + """Patch fairpredictor's SavedModel directory loader for TF 2.15 compatibility. - model_uri: local path, HTTP(S) URL to a .zip SavedModel, or a ``tf.keras.Model`` from ``train_ramp_model``. + ``TFSMLayer`` is absent in TF 2.15's ``tf.keras.layers``; fall back to ``load_model``. """ - import tempfile + import importlib import tensorflow as tf - # Ensure fairpredictor does not use TFSMLayer on TF 2.15. + pred = importlib.import_module("predictor.prediction") + if getattr(pred, "_fair_models_tf215_savedmodel_patch_applied", False): + return + + original_loader = getattr(pred, "_load_keras_model", None) + if original_loader is None: + raise RuntimeError("predictor.prediction._load_keras_model not found; fairpredictor API changed.") + + def _safe_load_keras_model(keras_backend, path: str): + if os.path.isdir(path) and (Path(path) / "saved_model.pb").exists(): + return tf.keras.models.load_model(path, compile=False) + return original_loader(keras_backend, path) + + _safe_load_keras_model._fair_models_tf215_savedmodel_loader = True # type: ignore[attr-defined] + pred._load_keras_model = _safe_load_keras_model + pred._fair_models_tf215_savedmodel_patch_applied = True + + +def infer_ramp_model( + model_uri: str | Path | Any, + input_path: str, + prediction_path: str, + output_dir: str, + confidence: float = 0.5, + model_cache_dir: str | None = None, +) -> dict[str, Any]: + """Run fairpredictor-style TF inference and return the merged GeoJSON content.""" _patch_predictor_savedmodel_loader_for_tf215() - from predictor.prediction import run_prediction # noqa: E402 + from predictor.prediction import run_prediction cache = Path(model_cache_dir) if model_cache_dir else None - if isinstance(model_uri, (str, Path)): + if isinstance(model_uri, bytes): + model_dir = str(_unzip_savedmodel_bytes(model_uri)) + elif isinstance(model_uri, (str, Path)): model_dir = resolve_model_href(str(model_uri), cache_dir=cache) - elif isinstance(model_uri, tf.keras.Model): - tmp = Path(tempfile.mkdtemp(prefix="ramp_infer_savedmodel_")) - model_uri.save(str(tmp)) - model_dir = str(tmp) else: - raise TypeError( - "model_uri must be a str, pathlib.Path, or a tf.keras.Model from training (compile=False load)." - ) - - # Fail fast if our patch didn't apply (helps diagnose environment mismatches early). - import importlib - - pred = importlib.import_module("predictor.prediction") - if not getattr(getattr(pred, "_load_keras_model", None), "_fair_models_tf215_savedmodel_loader", False): - raise RuntimeError( - "fairpredictor SavedModel loader patch not applied; " - "expected predictor.prediction._load_keras_model to be patched for TF 2.15." - ) + raise TypeError("model_uri must be a str, Path, or zipped SavedModel bytes.") input_dir = _resolve_input_directory(input_path, "input_path") out_dir = _to_local_path(prediction_path, "prediction_path") out_dir.mkdir(parents=True, exist_ok=True) - chip_files = sorted(input_dir.glob("**/*.tif")) - if not chip_files: + if not any(input_dir.glob("**/*.tif")): raise RuntimeError( - f"No GeoTIFF chips (*.tif) found in {input_dir}. " - "RAMP inference expects georeferenced chips." + f"No GeoTIFF chips (*.tif) found in {input_dir}. RAMP inference expects georeferenced chips." ) - run_input_dir = input_dir - if max_chips is not None and max_chips > 0: - subset_dir = out_dir / "subset_input" - if subset_dir.exists(): - rmtree(subset_dir) - subset_dir.mkdir(parents=True, exist_ok=True) - for chip_file in chip_files[:max_chips]: - copy2(chip_file, subset_dir / chip_file.name) - run_input_dir = subset_dir - georef_dir = run_prediction( checkpoint_path=model_dir, - input_path=str(run_input_dir), + input_path=str(input_dir), prediction_path=str(out_dir), - confidence=0.5, + confidence=confidence, crs="3857", ) - return postprocess(str(georef_dir), output_dir) + final_output_dir = Path(output_dir or tempfile.mkdtemp(prefix="ramp_postprocess_")) + final_output_dir.mkdir(parents=True, exist_ok=True) + return postprocess(str(georef_dir), str(final_output_dir)) @step -def run_postprocessing( - prediction_path: Union[Path, str], - output_dir: str, -) -> Dict[str, Any]: - """Run fairpredictor-style postprocessing and return merged GeoJSON content.""" - return postprocess(str(prediction_path), output_dir) +def run_preprocessing( + input_path: str, + output_path: str, + boundary_width: int = 3, + contact_spacing: int = 8, +) -> str: + """STAC entrypoint wrapper for RAMP preprocessing.""" + return preprocess(input_path, output_path, boundary_width, contact_spacing) + + +@step +def run_postprocessing(prediction_path: str, output_dir: str) -> dict[str, Any]: + """STAC entrypoint wrapper for RAMP postprocessing.""" + return postprocess(prediction_path, output_dir) + + +# --------------------------------------------------------------------------- +# @pipeline definitions +# --------------------------------------------------------------------------- @pipeline def training_pipeline( - input_path: str, - output_path: str, - stac_item_path: str = "models/ramp/stac-item.json", + base_model_weights: str, + dataset_chips: str, + dataset_labels: str, + num_classes: int, + hyperparameters: dict[str, Any], ) -> None: - """Full RAMP training: georeference + multimask → val split → EfficientNetB0 U-Net. - - Hyperparameters (backbone, epochs, batch_size, boundary_width, contact_spacing, etc.) - are loaded from the STAC Item at stac_item_path. - """ - hyperparams = _load_hyperparams_from_stac(stac_item_path) - boundary_width = hyperparams.get("boundary_width", 3) - contact_spacing = hyperparams.get("contact_spacing", 8) - - data_loader = run_preprocessing( - input_path=input_path, - output_path=f"{output_path}/preprocessed", - boundary_width=boundary_width, - contact_spacing=contact_spacing, - ) + """RAMP training pipeline: split → train → evaluate → export ONNX.""" split_info = split_dataset( - preprocessed_path=f"{output_path}/preprocessed", - output_path=output_path, - hyperparameters=hyperparams, + dataset_chips=dataset_chips, + dataset_labels=dataset_labels, + hyperparameters=hyperparameters, ) - train_model( - data_base_path=output_path, - data_loader=data_loader, - preprocessed_path=f"{output_path}/preprocessed", - stac_item_path=stac_item_path, + trained_model = train_model( + dataset_chips=dataset_chips, + dataset_labels=dataset_labels, + base_model_weights=base_model_weights, + hyperparameters=hyperparameters, split_info=split_info, + num_classes=num_classes, ) + evaluate_model( + trained_model=trained_model, + dataset_chips=dataset_chips, + dataset_labels=dataset_labels, + hyperparameters=hyperparameters, + split_info=split_info, + ) + export_onnx(trained_model=trained_model) @pipeline def inference_pipeline( - model_uri: Union[str, Path, Any], - input_path: str, - prediction_path: str, - output_dir: str, - model_cache_dir: Optional[str] = None, -) -> Dict[str, Any]: - """RAMP inference: load model → predict → postprocess → final GeoJSON. - - model_uri: local path, HTTP(S) URL to a .zip SavedModel, or a ``tf.keras.Model`` from training. - """ - final_geojson = run_inference( - model_uri=model_uri, - input_path=input_path, - prediction_path=prediction_path, - output_dir=output_dir, - model_cache_dir=model_cache_dir, + model_uri: str, + input_images: str, + inference_params: dict[str, Any] | None = None, + output_dir: str = "", + chip_size: int = 256, + num_classes: int = 4, + confidence: float = 0.5, + zenml_artifact_version_id: str = "", + prediction_path: str = "", +) -> dict[str, Any]: + """RAMP inference pipeline: load model → predict → postprocess → FeatureCollection.""" + _ = (chip_size, num_classes) + resolved_output_dir = output_dir or str(Path(tempfile.mkdtemp(prefix="ramp_inference_"))) + resolved_confidence = float((inference_params or {}).get("confidence_threshold", confidence)) + prediction_dir = prediction_path or str(Path(resolved_output_dir) / "predictions") + model = ( + load_model(model_uri=model_uri, zenml_artifact_version_id=zenml_artifact_version_id) + if zenml_artifact_version_id + else model_uri + ) + return run_inference( + model_uri=model, + input_images=input_images, + prediction_path=prediction_dir, + output_dir=resolved_output_dir, + confidence=resolved_confidence, ) - return final_geojson diff --git a/models/ramp/stac-item.json b/models/ramp/stac-item.json index 987c70d1..dcef741f 100644 --- a/models/ramp/stac-item.json +++ b/models/ramp/stac-item.json @@ -1,12 +1,13 @@ { "type": "Feature", - "stac_version": "1.0.0", + "stac_version": "1.1.0", "stac_extensions": [ "https://stac-extensions.github.io/mlm/v1.5.1/schema.json", "https://stac-extensions.github.io/version/v1.2.0/schema.json", "https://stac-extensions.github.io/classification/v2.0.0/schema.json", "https://stac-extensions.github.io/file/v2.1.0/schema.json", - "https://stac-extensions.github.io/raster/v1.1.0/schema.json" + "https://stac-extensions.github.io/raster/v1.1.0/schema.json", + "https://hotosm.github.io/fAIr-models/schemas/v1.0.0/base-model/schema.json" ], "id": "ramp-v1", "geometry": { @@ -24,6 +25,10 @@ "bbox": [-180, -90, 180, 90], "properties": { "datetime": "2024-01-01T00:00:00Z", + "created": "2024-01-01T00:00:00Z", + "updated": "2024-01-01T00:00:00Z", + "title": "RAMP EfficientNetB0 + U-Net Building Segmentation", + "description": "RAMP semantic-segmentation base model: EfficientNetB0 encoder with U-Net decoder outputs 4-class sparse-categorical masks (background, building, boundary, contact) from RGB aerial imagery. Packaged for fAIr finetuning and ONNX-based inference.", "mlm:name": "ramp-v1", "mlm:architecture": "EffUnet", "mlm:tasks": [ @@ -33,60 +38,69 @@ "mlm:framework_version": "2.15.1", "mlm:pretrained": true, "mlm:pretrained_source": "https://api-prod.fair.hotosm.org/api/v1/workspace/download/ramp/baseline.zip", + "mlm:accelerator": "amd64", + "mlm:accelerator_constrained": false, + "mlm:total_parameters": 7500000, + "mlm:memory_size": 536870912, "keywords": [ "building", "semantic-segmentation", - "polygon" + "polygon", + "ramp" ], + "license": "Apache-2.0", "version": "1", - "license": "Apache-2.0", - "status": "active", - "mlm:total_parameters": 7500000, - "mlm:memory_size": 536870912, - "mlm:accelerator": null, - "mlm:accelerator_constrained": false, - "licenses": { - "model": "Apache-2.0", - "data": "ODbL-1.0", - "code": "Apache-2.0" - }, - "inference_time_per_256x256": { - "value": 45, - "unit": "ms" - }, - "pipeline_input": { - "type": "directory", - "description": "Directory of georeferenced RGB GeoTIFF chips (256×256, EPSG:3857)", - "expected_files": "*.tif", - "required_params": ["input_path", "model_uri"] - }, - "pipeline_output": { - "type": "directory", - "description": "Directory containing merged prediction GeoJSON (`predictions.geojson`) built from fairpredictor-style postprocessing", - "expected_files": "predictions.geojson", - "geometry_types": ["Polygon", "MultiPolygon"] - }, - "evaluation_metrics": [ + "deprecated": false, + "providers": [ + { + "name": "HOTOSM", + "roles": [ + "producer", + "host" + ], + "url": "https://www.hotosm.org", + "description": "Humanitarian OpenStreetMap Team" + }, + { + "name": "Development Seed", + "roles": [ + "producer" + ], + "url": "https://developmentseed.org", + "description": "Original authors of the RAMP model." + } + ], + "fair:metrics_spec": [ + { + "key": "fair:accuracy", + "name": "Pixel accuracy", + "description": "Per-pixel classification accuracy across all 4 classes on the validation split." + }, + { + "key": "fair:mean_iou", + "name": "Building IoU", + "description": "Intersection-over-Union of the building class (class 1) on the validation split." + }, { - "metric": "val_sparse_categorical_accuracy", - "definition": "Sparse categorical accuracy on validation set", - "value": 0.92 + "key": "fair:precision", + "name": "Building precision", + "description": "Per-pixel precision of the building class (class 1) on the validation split." }, { - "metric": "val_loss", - "definition": "Sparse categorical crossentropy on validation set", - "value": 0.15 + "key": "fair:recall", + "name": "Building recall", + "description": "Per-pixel recall of the building class (class 1) on the validation split." } ], - "zenml_entrypoints": { - "preprocessing": "models.ramp.pipeline:run_preprocessing", - "inference": "models.ramp.pipeline:inference_pipeline", - "postprocessing": "models.ramp.pipeline:run_postprocessing", - "training": "models.ramp.pipeline:train_model" + "fair:split_spec": { + "strategy": "random", + "default_ratio": 0.15, + "seed": 42, + "description": "Preprocess chips + labels into 4-class multimasks, then seeded random train/validation split via hot_fair_utilities.split_training_2_validation." }, - "training_time": { - "value": 4.0, - "unit": "hours" + "inference_time_per_256x256": { + "value": 45, + "unit": "ms" }, "mlm:input": [ { @@ -111,64 +125,73 @@ ], "mlm:output": [ { - "name": "4-class sparse categorical multimask", + "name": "4-class sparse-categorical multimask", "tasks": ["semantic-segmentation"], "result": { - "shape": [-1, 256, 256, 1], - "dim_order": ["batch", "height", "width", "channel"], - "data_type": "uint8" + "shape": [-1, 256, 256, 4], + "dim_order": ["batch", "height", "width", "classes"], + "data_type": "float32" }, "classification:classes": [ { "name": "background", "value": 0, - "description": "Non-building pixels" + "description": "Non-building pixels." }, { "name": "building", "value": 1, - "description": "Building interior pixels" + "description": "Building interior pixels." }, { "name": "boundary", "value": 2, - "description": "Building boundary pixels (helps separate adjacent buildings)" + "description": "Building boundary pixels (helps separate adjacent buildings)." }, { "name": "contact", "value": 3, - "description": "Close-contact points between neighbouring buildings" + "description": "Close-contact points between neighbouring buildings." } ], "post_processing_function": { "format": "python", "expression": "models.ramp.pipeline:postprocess" - } + }, + "bands": [] } ], "mlm:hyperparameters": { - "backbone": "efficientnetb0", - "num_classes": 4, - "epochs": 20, - "batch_size": 8, - "learning_rate": 0.0003, - "loss": "sparse_categorical_crossentropy", - "optimizer": "adam", - "input_img_shape": [256, 256], - "output_img_shape": [256, 256], - "boundary_width": 3, - "contact_spacing": 8, - "val_fraction": 0.15, - "early_stopping_patience": 10, - "augmentation": false + "training.backbone": "efficientnetb0", + "training.epochs": 20, + "training.batch_size": 8, + "training.learning_rate": 0.0003, + "training.val_ratio": 0.15, + "training.split_seed": 42, + "training.boundary_width": 3, + "training.contact_spacing": 8, + "training.early_stopping_patience": 10, + "training.augmentation": false, + "training.imgsz": 256, + "training.num_classes": 4, + "inference.confidence_threshold": 0.5, + "inference.min_class_value": 1 + }, + "licenses": { + "model": "Apache-2.0", + "data": "ODbL-1.0", + "code": "Apache-2.0" } }, "assets": { - "model": { + "checkpoint": { "href": "https://api-prod.fair.hotosm.org/api/v1/workspace/download/ramp/baseline.zip", - "type": "application/octet-stream; framework=tensorflow", - "title": "RAMP EfficientNetB0 U-Net weights (Keras SavedModel)", - "roles": ["mlm:model"], + "title": "RAMP EfficientNetB0 + U-Net baseline weights (zipped TF SavedModel)", + "type": "application/zip; framework=tensorflow", + "roles": [ + "mlm:model", + "mlm:weights" + ], "mlm:artifact_type": "tf.keras.Model", "raster:bands": [ {"name": "red"}, @@ -176,25 +199,67 @@ {"name": "blue"} ] }, + "model": { + "href": "https://api-prod.fair.hotosm.org/api/v1/workspace/download/ramp/ramp-v1.onnx", + "title": "Portable ONNX inference model", + "type": "application/octet-stream; framework=onnx", + "roles": [ + "mlm:model", + "mlm:compiled" + ], + "mlm:artifact_type": "onnx", + "raster:bands": [ + {"name": "red"}, + {"name": "green"}, + {"name": "blue"} + ] + }, "source-code": { "href": "https://github.com/hotosm/fAIr-models/tree/main/models/ramp", "type": "text/html", "title": "RAMP model pack source", - "roles": ["code"], + "roles": [ + "code" + ], "mlm:entrypoint": "models.ramp.pipeline:training_pipeline" }, "mlm:training": { "href": "ghcr.io/hotosm/fair-models/ramp:v1", "type": "application/vnd.oci.image.index.v1+json", "title": "Training Docker Image", - "roles": ["mlm:training-runtime"] + "roles": [ + "mlm:training-runtime" + ] }, "mlm:inference": { "href": "ghcr.io/hotosm/fair-models/ramp:v1", "type": "application/vnd.oci.image.index.v1+json", "title": "Inference Docker Image", - "roles": ["mlm:inference-runtime"] + "roles": [ + "mlm:inference-runtime" + ] + }, + "readme": { + "href": "https://raw.githubusercontent.com/hotosm/fAIr-models/refs/heads/main/models/ramp/README.md", + "type": "text/markdown", + "roles": [ + "metadata" + ], + "title": "Model README" } }, - "links": [] + "links": [ + { + "rel": "license", + "href": "https://www.apache.org/licenses/LICENSE-2.0", + "type": "text/html", + "title": "Apache License 2.0" + }, + { + "rel": "cite-as", + "href": "https://github.com/radiantearth/ramp-code", + "type": "text/html", + "title": "RAMP — Replicable AI for Microplanning" + } + ] } diff --git a/models/ramp/tests/conftest.py b/models/ramp/tests/conftest.py new file mode 100644 index 00000000..1a9de9a2 --- /dev/null +++ b/models/ramp/tests/conftest.py @@ -0,0 +1,128 @@ +"""Per-model test fixtures: toy OAM-tiled RAMP dataset (chips + labels). + +``create_toy_data`` is importable by ``models/test_integration.py`` to materialize a +minimal 4-class semantic-segmentation dataset. Chip TIFFs follow the OAM naming +convention ``OAM-{x}-{y}-{z}.tif`` so ``hot_fair_utilities.preprocessing.clip_labels`` +can recover the tile bounds. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import mercantile +import numpy as np +import pytest +import rasterio +from rasterio.crs import CRS +from rasterio.transform import from_bounds + +CHIP_COUNT = 6 +CHIP_SIZE = 128 +_OAM_ZOOM = 18 +_TILE_XY = [(100, 200), (101, 200), (102, 200), (100, 201), (101, 201), (102, 201)] + +_bounds = mercantile.bounds(mercantile.Tile(x=_TILE_XY[0][0], y=_TILE_XY[0][1], z=_OAM_ZOOM)) +_WEST, _SOUTH, _EAST, _NORTH = _bounds.west, _bounds.south, _bounds.east, _bounds.north +_GEOMETRY = { + "type": "Polygon", + "coordinates": [[[_WEST, _SOUTH], [_EAST, _SOUTH], [_EAST, _NORTH], [_WEST, _NORTH], [_WEST, _SOUTH]]], +} +_BBOX = [_WEST, _SOUTH, _EAST, _NORTH] + + +def create_toy_data(root: Path) -> dict[str, Path]: + chips_dir = root / "chips" + chips_dir.mkdir() + + labels_dir = root / "labels" + labels_dir.mkdir() + + features: list[dict[str, Any]] = [] + for i in range(CHIP_COUNT): + tx, ty = _TILE_XY[i] + tile = mercantile.Tile(x=tx, y=ty, z=_OAM_ZOOM) + b = mercantile.bounds(tile) + transform = from_bounds(b.west, b.south, b.east, b.north, CHIP_SIZE, CHIP_SIZE) + chip_name = f"OAM-{tx}-{ty}-{_OAM_ZOOM}.tif" + with rasterio.open( + chips_dir / chip_name, + "w", + driver="GTiff", + width=CHIP_SIZE, + height=CHIP_SIZE, + count=3, + dtype="uint8", + crs=CRS.from_epsg(4326), + transform=transform, + ) as dst: + dst.write(np.random.randint(0, 255, (3, CHIP_SIZE, CHIP_SIZE), dtype=np.uint8)) + + # Tiny square polygon well inside each chip footprint. + w, h = b.east - b.west, b.north - b.south + cx = b.west + w * 0.35 + cy = b.south + h * 0.35 + s = min(w, h) * 0.25 + features.append( + { + "type": "Feature", + "properties": {"building": 1}, + "geometry": { + "type": "Polygon", + "coordinates": [[[cx, cy], [cx + s, cy], [cx + s, cy + s], [cx, cy + s], [cx, cy]]], + }, + } + ) + + labels_geojson = labels_dir / "labels.geojson" + labels_geojson.write_text(json.dumps({"type": "FeatureCollection", "features": features})) + + stac_path = root / "dataset-stac-item.json" + stac_path.write_text(json.dumps(_build_dataset_stac_item(chips_dir, labels_dir), indent=2)) + return {"chips": chips_dir, "labels": labels_dir, "dataset_stac_item": stac_path} + + +@pytest.fixture(scope="session") +def generate_toy_dataset(tmp_path_factory: pytest.TempPathFactory) -> dict[str, Path]: + return create_toy_data(tmp_path_factory.mktemp("toy_ramp")) + + +def _build_dataset_stac_item(chips_dir: Path, labels_dir: Path) -> dict[str, Any]: + return { + "type": "Feature", + "stac_version": "1.1.0", + "stac_extensions": [ + "https://stac-extensions.github.io/label/v1.0.1/schema.json", + ], + "id": "toy-ramp", + "geometry": _GEOMETRY, + "bbox": _BBOX, + "properties": { + "datetime": "2024-01-01T00:00:00Z", + "description": "Toy RAMP semantic-segmentation dataset", + "label:type": "vector", + "label:tasks": ["segmentation"], + "label:classes": [{"name": "building", "classes": ["building"]}], + "label:description": "Toy polygon labels", + "keywords": ["building"], + "providers": [ + { + "name": "HOTOSM", + "roles": ["producer"], + "url": "https://www.hotosm.org", + "description": "Humanitarian OpenStreetMap Team", + } + ], + "fair:user_id": "test", + "version": "1", + "deprecated": False, + "license": "CC-BY-4.0", + }, + "assets": { + "chips": {"href": str(chips_dir), "type": "image/tiff", "roles": ["data"]}, + "labels": {"href": str(labels_dir), "type": "application/geo+json", "roles": ["labels"]}, + }, + "links": [], + } diff --git a/models/ramp/tests/inside_container_smoke_test.py b/models/ramp/tests/inside_container_smoke_test.py deleted file mode 100644 index 024bc761..00000000 --- a/models/ramp/tests/inside_container_smoke_test.py +++ /dev/null @@ -1,336 +0,0 @@ -"""End-to-end smoke tests for models/ramp Docker runtime. - -Run this script INSIDE the container. It validates: -1) Critical imports (tensorflow, ramp, segmentation_models, osgeo.gdal, solaris) -2) ``pipeline.preprocess`` path (georeference + multimask generation) -3) ``pipeline.run_preprocessing`` contract: chip/mask arrays + on-disk counts -4) ``pipeline.split_dataset`` + training wrappers: - ``pipeline.train_model`` and ``pipeline.train_ramp_model`` - both return loaded ``tf.keras.Model`` -5) ``pipeline.resolve_model_href`` for local SavedModel directories -6) ``pipeline.run_inference`` returns merged GeoJSON dict content -7) ``pipeline.run_postprocessing`` wrapper returns dict and writes output -8) Inference intermediate georeferenced rasters for debug visibility - -All training, inference, and postprocessing steps delegate to pipeline.py -helpers so the smoke test exercises the same code paths as production. - -Data layouts supported: - - data/sample layout: --dataset-root /workspace/data/sample - Uses train/oam/*.tif + train/osm/*.geojson. Converts TIF→PNG and merges - OSM labels into a temporary input directory (hot_fair_utilities expects PNG). - - Legacy RAMP layout: --dataset-root /path/to/dataset - Expects dataset/input/*.png and dataset/input/labels.geojson. - -Usage inside container: - python /workspace/models/ramp/tests/inside_container_smoke_test.py \\ - --dataset-root /workspace/data/sample -""" - -from __future__ import annotations - -import argparse -import os -import shutil -from pathlib import Path -from typing import Any - - -def _assert(condition: bool, message: str) -> None: - if not condition: - raise RuntimeError(message) - - -def _stage(name: str) -> None: - print(f"\n=== {name} ===") - - -def _count_files(path: Path, pattern: str) -> int: - return len(list(path.glob(pattern))) - - -def _load_ramp_pipeline_module(): - """Import models/ramp/pipeline.py directly so smoke tests reuse production helpers.""" - import sys - - module_dir = Path("/workspace/models/ramp") - if str(module_dir) not in sys.path: - sys.path.insert(0, str(module_dir)) - import pipeline as ramp_pipeline - - return ramp_pipeline - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Run RAMP container smoke tests.") - parser.add_argument( - "--dataset-root", - default="/workspace/data/sample", - help="Dataset root: data/sample (train/oam + train/osm) or legacy (input/ with PNG + labels.geojson).", - ) - parser.add_argument("--epochs", type=int, default=2) - parser.add_argument("--batch-size", type=int, default=4) - parser.add_argument("--backbone", default="efficientnetb0") - return parser.parse_args() - - -def _prepare_data_sample_layout(dataset_root: Path) -> Path: - """Convert data/sample (train/oam + train/osm) to RAMP input layout. - - hot_fair_utilities.preprocess expects input_path with *.png and labels.geojson. - data/sample has train/oam/*.tif and train/osm/*.geojson. - Creates dataset_root/ramp_work/input/ with PNG chips and labels.geojson. - Returns the working root (ramp_work) for the test. - """ - oam_dir = dataset_root / "train" / "oam" - osm_dir = dataset_root / "train" / "osm" - _assert(oam_dir.is_dir(), f"train/oam not found under {dataset_root}") - _assert(osm_dir.is_dir(), f"train/osm not found under {dataset_root}") - - work_root = dataset_root / "ramp_work" - input_dir = work_root / "input" - input_dir.mkdir(parents=True, exist_ok=True) - - import numpy as np - import rasterio - from PIL import Image - - tif_files = sorted(oam_dir.glob("OAM-*.tif")) - _assert(len(tif_files) > 0, f"No OAM-*.tif files in {oam_dir}") - - for tif_path in tif_files: - png_path = input_dir / (tif_path.stem + ".png") - with rasterio.open(tif_path) as src: - data = src.read() - if data.shape[0] >= 3: - rgb = np.transpose(data[:3], (1, 2, 0)) - if rgb.max() <= 1.0: - rgb = (rgb * 255).astype(np.uint8) - Image.fromarray(rgb).save(png_path) - - geojson_files = sorted(osm_dir.glob("*.geojson")) - _assert(len(geojson_files) > 0, f"No .geojson files in {osm_dir}") - - import geopandas as gpd - import pandas as pd - - gdfs = [gpd.read_file(p) for p in geojson_files] - if len(gdfs) == 1: - merged = gdfs[0] - else: - crs = gdfs[0].crs or "EPSG:4326" - merged = gpd.GeoDataFrame( - pd.concat([g.to_crs(crs) for g in gdfs], ignore_index=True), - crs=crs, - ) - # Fiona/GeoJSON writers cannot reliably handle pandas extension dtypes - # (for example string[python], Int64, boolean). Normalize to plain Python - # objects with None for missing values before writing. - for col in merged.columns: - if col == "geometry": - continue - if pd.api.types.is_extension_array_dtype(merged[col].dtype): - merged[col] = merged[col].astype(object).where(merged[col].notna(), None) - labels_path = input_dir / "labels.geojson" - merged.to_file(labels_path, driver="GeoJSON") - print(f"PASS: prepared {len(tif_files)} PNG chip(s) + labels.geojson from data/sample") - - return work_root - - -def main() -> None: - args = parse_args() - - os.environ.setdefault("RAMP_HOME", "/workspace") - # segmentation_models picks the backend when the package is first imported. - # Must be tf.keras for TF 2.15+ (Keras 3); calling set_framework() after import is too late. - os.environ.setdefault("SM_FRAMEWORK", "tf.keras") - - dataset_root = Path(args.dataset_root).resolve() - - # Support data/sample layout (train/oam + train/osm) or legacy (input/ with PNG + labels) - if (dataset_root / "train" / "oam").is_dir() and (dataset_root / "train" / "osm").is_dir(): - dataset_root = _prepare_data_sample_layout(dataset_root) - - input_dir = dataset_root / "input" - preprocessed_dir = dataset_root / "preprocessed_test" - chips_dir = preprocessed_dir / "chips" - masks_dir = preprocessed_dir / "multimasks" - pred_input_dir = preprocessed_dir / "chips" - pred_output_dir = dataset_root / "prediction_test" / "output" - vectors_dir = dataset_root / "prediction_test" / "vectors" - - # ------------------------------------------------------------------------- - _stage("Test 1: Critical imports") - - import hot_fair_utilities # noqa: F401 - import ramp # noqa: F401 - import segmentation_models as sm - import solaris - import tensorflow as tf - from osgeo import gdal - - sm.set_framework("tf.keras") # redundant if SM_FRAMEWORK already set; keeps intent explicit - print(f"PASS: tensorflow {tf.__version__} / segmentation_models / ramp / gdal imported") - print(f"PASS: GDAL runtime version {gdal.VersionInfo('--version')}") - print(f"PASS: solaris {solaris.__version__} imported") - - # ------------------------------------------------------------------------- - _stage("Test 2: Input dataset layout checks") - - _assert(dataset_root.is_dir(), f"Dataset root not found: {dataset_root}") - _assert(input_dir.is_dir(), f"Input folder not found: {input_dir}") - _assert( - (input_dir / "labels.geojson").is_file(), - f"labels.geojson not found in {input_dir}", - ) - n_png = _count_files(input_dir, "*.png") - _assert(n_png > 0, f"No PNG chips found in {input_dir}. Add OAM PNG chips to run this test.") - print(f"PASS: found {n_png} PNG chip(s) + labels.geojson") - - ramp_pipeline = _load_ramp_pipeline_module() - - # ------------------------------------------------------------------------- - _stage("Test 3: Preprocessing via pipeline.preprocess + run_preprocessing contract") - - shutil.rmtree(preprocessed_dir, ignore_errors=True) - out_path = ramp_pipeline.preprocess( - input_path=str(input_dir), - output_path=str(preprocessed_dir), - boundary_width=3, - contact_spacing=8, - ) - _assert(Path(out_path).resolve() == preprocessed_dir.resolve(), f"Unexpected preprocess output: {out_path}") - - n_chips = _count_files(chips_dir, "*.tif") - n_masks = _count_files(masks_dir, "*.mask.tif") - _assert(n_chips > 0, f"No chips produced in {chips_dir}") - _assert(n_masks > 0, f"No multimasks produced in {masks_dir}") - _assert(n_chips == n_masks, f"Chip count ({n_chips}) != mask count ({n_masks})") - print(f"PASS: preprocessing produced {n_chips} chip(s) + {n_masks} multimask(s)") - - # Use pipeline.run_preprocessing directly so this smoke test follows pipeline.py behavior. - import numpy as np - - data_loader_contract: list[tuple[Any, Any]] = ramp_pipeline.run_preprocessing( - input_path=str(input_dir), - output_path=str(preprocessed_dir), - boundary_width=3, - contact_spacing=8, - ) - _assert(len(data_loader_contract) > 0, "Expected at least one chip/mask array pair.") - _assert( - isinstance(data_loader_contract[0][0], np.ndarray), - "Chip data must be a numpy ndarray (pipeline.run_preprocessing contract).", - ) - _assert( - isinstance(data_loader_contract[0][1], np.ndarray), - "Mask data must be a numpy ndarray (pipeline.run_preprocessing contract).", - ) - print( - f"PASS: dataloader contract — {len(data_loader_contract)} (chip, mask) ndarray pair(s), " - "matching pipeline.run_preprocessing" - ) - - # ------------------------------------------------------------------------- - _stage("Test 4: split_dataset + training smoke run") - # train_model wraps train_ramp_model and validates preprocessing output. - # train_ramp_model handles val split internally and returns loaded tf.keras.Model. - - import tensorflow as tf - - split_info = ramp_pipeline.split_dataset( - preprocessed_path=str(preprocessed_dir), - output_path=str(dataset_root), - hyperparameters={"split_seed": 42}, - ) - _assert(split_info["train_count"] > 0, "split_dataset produced zero train chips.") - _assert(split_info["val_count"] > 0, "split_dataset produced zero validation chips.") - print( - "PASS: split_dataset produced " - f"{split_info['train_count']} train and {split_info['val_count']} validation chips" - ) - - trained_model_step = ramp_pipeline.train_model( - data_base_path=str(dataset_root), - preprocessed_path=str(preprocessed_dir), - data_loader=data_loader_contract, - stac_item_path="/workspace/models/ramp/stac-item.json", - val_fraction=0.15, - split_info=split_info, - ) - _assert(isinstance(trained_model_step, tf.keras.Model), "train_model did not return a Keras model checkpoint.") - - trained_model = ramp_pipeline.train_ramp_model( - data_base_path=str(dataset_root), - preprocessed_path=str(preprocessed_dir), - stac_item_path="/workspace/models/ramp/stac-item.json", - num_epochs=args.epochs, - batch_size=args.batch_size, - backbone=args.backbone, - early_stopping_patience=2, - log_zenml_step_metadata=False, - ) - _assert(isinstance(trained_model, tf.keras.Model), "train_ramp_model did not return a Keras model checkpoint.") - print(f"PASS: {args.epochs}-epoch training completed; train wrappers returned tf.keras.Model") - - # ------------------------------------------------------------------------- - _stage("Test 5: resolve_model_href local SavedModel resolution") - - local_saved_model_dir = dataset_root / "local_smoke_savedmodel" - shutil.rmtree(local_saved_model_dir, ignore_errors=True) - trained_model.save(str(local_saved_model_dir)) - resolved_model_dir = ramp_pipeline.resolve_model_href(str(local_saved_model_dir)) - _assert( - (Path(resolved_model_dir) / "saved_model.pb").is_file(), - f"resolve_model_href did not return a SavedModel dir: {resolved_model_dir}", - ) - print("PASS: resolve_model_href resolved a valid local SavedModel directory") - - # ------------------------------------------------------------------------- - _stage("Test 6: Inference smoke run via run_inference") - - shutil.rmtree(pred_output_dir, ignore_errors=True) - pred_output_dir.mkdir(parents=True, exist_ok=True) - final_geojson = ramp_pipeline.run_inference( - model_uri=resolved_model_dir, - input_path=str(pred_input_dir), - prediction_path=str(pred_output_dir), - output_dir=str(vectors_dir), - ) - _assert(isinstance(final_geojson, dict), "Inference did not return GeoJSON dict content.") - _assert(final_geojson.get("type") == "FeatureCollection", "GeoJSON response missing FeatureCollection type.") - print(f"PASS: run_inference returned GeoJSON content with {len(final_geojson.get('features', []))} feature(s)") - - # ------------------------------------------------------------------------- - _stage("Test 7: Postprocessing via run_postprocessing wrapper") - - georef_dir = pred_output_dir / "georeference" - postprocess_out_dir = dataset_root / "prediction_test" / "vectors_postprocess_wrapper" - shutil.rmtree(postprocess_out_dir, ignore_errors=True) - postprocess_out_dir.mkdir(parents=True, exist_ok=True) - wrapped_geojson = ramp_pipeline.run_postprocessing( - prediction_path=str(georef_dir), - output_dir=str(postprocess_out_dir), - ) - _assert(isinstance(wrapped_geojson, dict), "run_postprocessing did not return GeoJSON dict content.") - _assert( - (postprocess_out_dir / "predictions.geojson").is_file(), - "run_postprocessing did not write predictions.geojson", - ) - print("PASS: run_postprocessing returned dict and wrote predictions.geojson") - - # ------------------------------------------------------------------------- - _stage("Test 8: Inference intermediate artifacts") - - _assert(georef_dir.is_dir(), f"Georeferenced output directory not found: {georef_dir}") - n_pred = _count_files(georef_dir, "*.tif") - _assert(n_pred > 0, f"No georeferenced prediction .tif files produced in {georef_dir}") - print(f"PASS: fairpredictor produced {n_pred} georeferenced prediction .tif file(s)") - - # ------------------------------------------------------------------------- - _stage("ALL TESTS PASSED") - - -if __name__ == "__main__": - main() diff --git a/models/ramp/tests/run_docker_tests.ps1 b/models/ramp/tests/run_docker_tests.ps1 deleted file mode 100644 index 7f08a78c..00000000 --- a/models/ramp/tests/run_docker_tests.ps1 +++ /dev/null @@ -1,77 +0,0 @@ -# RAMP Docker smoke test — PowerShell (Windows). -# Same behavior as run_docker_tests.sh. -# -# Base images are pulled from GHCR (no local build needed): -# CPU: ghcr.io/hotosm/fair-utilities-ramp:cpu-latest (default) -# GPU: ghcr.io/hotosm/fair-utilities-ramp:gpu-latest -# -# Bash syntax "VAR=1 ./script.sh" does not work in PowerShell. Use either: -# - This script, or -# - $env:BUILD_IMAGE = "1"; $env:CPU_ONLY = "1"; .\models\ramp\tests\run_docker_tests.ps1 -# Optional smoke args: -# - $env:SMOKE_DATASET_ROOT (default: /workspace/data/sample) -# - $env:SMOKE_EPOCHS (default: 2) -# - $env:SMOKE_BATCH_SIZE (default: 4) -# - $env:SMOKE_BACKBONE (default: efficientnetb0) -# -# Examples (run from fAIr-models repo root): -# .\models\ramp\tests\run_docker_tests.ps1 -# $env:BUILD_IMAGE = "1"; $env:CPU_ONLY = "1"; .\models\ramp\tests\run_docker_tests.ps1 -# .\models\ramp\tests\run_docker_tests.ps1 -RepoRoot "E:\path\to\fAIr-models" -Image "ramp-v1:cpu" - -#Requires -Version 5.1 -param( - [string] $RepoRoot = (Get-Location).Path, - [string] $Image = "" -) - -$ErrorActionPreference = "Stop" - -function Invoke-NativeOrThrow { - param([scriptblock] $Command) - & $Command - if ($LASTEXITCODE -ne 0) { - throw "Command failed with exit code $LASTEXITCODE." - } -} - -$buildImage = if ($null -ne $env:BUILD_IMAGE) { $env:BUILD_IMAGE } else { "0" } -$cpuOnly = if ($null -ne $env:CPU_ONLY) { $env:CPU_ONLY } else { "0" } -$smokeDatasetRoot = if ($null -ne $env:SMOKE_DATASET_ROOT) { $env:SMOKE_DATASET_ROOT } else { "/workspace/data/sample" } -$smokeEpochs = if ($null -ne $env:SMOKE_EPOCHS) { $env:SMOKE_EPOCHS } else { "2" } -$smokeBatchSize = if ($null -ne $env:SMOKE_BATCH_SIZE) { $env:SMOKE_BATCH_SIZE } else { "4" } -$smokeBackbone = if ($null -ne $env:SMOKE_BACKBONE) { $env:SMOKE_BACKBONE } else { "efficientnetb0" } - -if (-not $Image) { - if ($cpuOnly -eq "1") { - $Image = "ramp-v1:cpu" - } - else { - $Image = "ramp-v1:gpu" - } -} - -if ($buildImage -eq "1") { - if ($cpuOnly -eq "1") { - $Image = "ramp-v1:cpu" - Write-Host "Building image $Image (base: ghcr.io/hotosm/fair-utilities-ramp:cpu-latest) ..." - # No --build-arg needed — the Dockerfile default already points at the GHCR CPU image. - Invoke-NativeOrThrow { docker build -t $Image -f "$RepoRoot/models/ramp/Dockerfile" $RepoRoot } - } - else { - $Image = "ramp-v1:gpu" - Write-Host "Building image $Image (base: ghcr.io/hotosm/fair-utilities-ramp:gpu-latest) ..." - Invoke-NativeOrThrow { docker build --build-arg BASE_IMAGE=ghcr.io/hotosm/fair-utilities-ramp:gpu-latest -t $Image -f "$RepoRoot/models/ramp/Dockerfile" $RepoRoot } - } -} - -if ($cpuOnly -ne "1") { - Write-Host "Running container smoke tests..." - Invoke-NativeOrThrow { docker run --rm --gpus all -v "${RepoRoot}:/workspace" $Image python /workspace/models/ramp/tests/inside_container_smoke_test.py --dataset-root $smokeDatasetRoot --epochs $smokeEpochs --batch-size $smokeBatchSize --backbone $smokeBackbone } -} -else { - Write-Host "Running container smoke tests..." - Invoke-NativeOrThrow { docker run --rm -v "${RepoRoot}:/workspace" $Image python /workspace/models/ramp/tests/inside_container_smoke_test.py --dataset-root $smokeDatasetRoot --epochs $smokeEpochs --batch-size $smokeBatchSize --backbone $smokeBackbone } -} - -Write-Host "Done." diff --git a/models/ramp/tests/run_docker_tests.sh b/models/ramp/tests/run_docker_tests.sh deleted file mode 100644 index 185e30c6..00000000 --- a/models/ramp/tests/run_docker_tests.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -# Run RAMP in-container smoke tests. Default: /workspace/data/sample, script defaults for epochs/batch. -# Base images come from GHCR (no local build of fAIr-utilities needed): -# CPU: ghcr.io/hotosm/fair-utilities-ramp:cpu-latest (default) -# GPU: ghcr.io/hotosm/fair-utilities-ramp:gpu-latest -# Windows PowerShell: use run_docker_tests.ps1 (Bash-style VAR=1 ./script.sh does not work in PowerShell). -# To pass --epochs / --batch-size / --backbone / a custom --dataset-root, use docker run manually -# (see research/fAIr_3.0/ramp_model/03_ramp_test_explained.md). -set -euo pipefail - -REPO_ROOT="${1:-$(pwd)}" -IMAGE="${2:-ramp-v1:gpu}" -BUILD_IMAGE="${BUILD_IMAGE:-0}" -CPU_ONLY="${CPU_ONLY:-0}" - -if [[ "$BUILD_IMAGE" == "1" ]]; then - if [[ "$CPU_ONLY" == "1" ]]; then - IMAGE="ramp-v1:cpu" - # No --build-arg needed; Dockerfile default is ghcr.io/hotosm/fair-utilities-ramp:cpu-latest - echo "Building image $IMAGE (base: ghcr.io/hotosm/fair-utilities-ramp:cpu-latest) ..." - docker build -t "$IMAGE" -f "$REPO_ROOT/models/ramp/Dockerfile" "$REPO_ROOT" - else - echo "Building image $IMAGE (base: ghcr.io/hotosm/fair-utilities-ramp:gpu-latest) ..." - docker build --build-arg BASE_IMAGE=ghcr.io/hotosm/fair-utilities-ramp:gpu-latest \ - -t "$IMAGE" -f "$REPO_ROOT/models/ramp/Dockerfile" "$REPO_ROOT" - fi -fi - -GPU_ARGS=() -if [[ "$CPU_ONLY" != "1" ]]; then - GPU_ARGS=(--gpus all) -fi - -echo "Running container smoke tests..." -docker run --rm "${GPU_ARGS[@]}" \ - -v "$REPO_ROOT:/workspace" \ - "$IMAGE" \ - python /workspace/models/ramp/tests/inside_container_smoke_test.py \ - --dataset-root /workspace/data/sample - -echo "Done." diff --git a/models/ramp/tests/test_steps.py b/models/ramp/tests/test_steps.py new file mode 100644 index 00000000..25e21bca --- /dev/null +++ b/models/ramp/tests/test_steps.py @@ -0,0 +1,170 @@ +"""Step tests for the RAMP pipeline. + +Each test calls ``step.entrypoint(...)`` directly. Heavy RAMP/TF operations +(``train_ramp_model``, SavedModel loading, tf2onnx conversion) are patched so +tests run quickly and do not require GPU resources. +""" + +from __future__ import annotations + +from contextlib import contextmanager +from pathlib import Path +from typing import Any +from unittest.mock import patch + + +@contextmanager +def _noop_mlflow_ctx(*_args: Any, **_kwargs: Any): + yield + + +def test_split_dataset(toy_chips: Path, toy_labels: Path, base_hyperparameters: dict[str, Any]) -> None: + from models.ramp.pipeline import split_dataset + + hyperparameters = dict(base_hyperparameters) + hyperparameters.update( + { + "epochs": 1, + "batch_size": 1, + "val_fraction": 0.25, + "split_seed": 42, + "boundary_width": 1, + "contact_spacing": 2, + } + ) + + with patch("models.ramp.pipeline.log_metadata"): + result = split_dataset.entrypoint( + dataset_chips=str(toy_chips), + dataset_labels=str(toy_labels), + hyperparameters=hyperparameters, + ) + + assert result["strategy"] == "random" + assert result["train_count"] > 0 + assert result["val_count"] > 0 + assert "_ramp_train_dir" in result + assert "_preprocessed_dir" in result + ramp_dir = Path(result["_ramp_train_dir"]) + assert ramp_dir.exists() + assert (ramp_dir / "chips").is_dir() + assert (ramp_dir / "val-chips").is_dir() + + +def test_train_model(toy_chips: Path, toy_labels: Path, base_hyperparameters: dict[str, Any], tmp_path: Path) -> None: + from models.ramp.pipeline import train_model + + ramp_train_dir = tmp_path / "ramp_training_work" + (ramp_train_dir / "chips").mkdir(parents=True) + (ramp_train_dir / "val-chips").mkdir(parents=True) + + fake_saved_model_dir = tmp_path / "saved_model" + fake_saved_model_dir.mkdir() + (fake_saved_model_dir / "saved_model.pb").write_bytes(b"\x08\x01") # magic-enough stub + + split_info = { + "_work_dir": str(tmp_path), + "_preprocessed_dir": str(tmp_path / "preprocessed"), + "_ramp_train_dir": str(ramp_train_dir), + "strategy": "random", + "val_ratio": 0.25, + "seed": 42, + "train_count": 3, + "val_count": 1, + "description": "test split", + } + hyperparameters = dict(base_hyperparameters) + hyperparameters.update({"epochs": 1, "batch_size": 1}) + + with ( + patch("models.ramp.pipeline.mlflow_training_context", _noop_mlflow_ctx, create=True), + patch("models.ramp.pipeline.train_ramp_model", return_value=fake_saved_model_dir), + patch("models.ramp.pipeline._zip_savedmodel_dir", return_value=b"fake-savedmodel-zip"), + patch("models.ramp.pipeline.log_metadata"), + ): + model_bytes = train_model.entrypoint( + dataset_chips=str(toy_chips), + dataset_labels=str(toy_labels), + base_model_weights="https://example.com/baseline.zip", + hyperparameters=hyperparameters, + split_info=split_info, + num_classes=4, + ) + + assert isinstance(model_bytes, bytes) + assert model_bytes == b"fake-savedmodel-zip" + + +def test_evaluate_model( + toy_chips: Path, toy_labels: Path, base_hyperparameters: dict[str, Any], tmp_path: Path +) -> None: + from models.ramp.pipeline import evaluate_model + + ramp_train_dir = tmp_path / "ramp_training_work" + (ramp_train_dir / "chips").mkdir(parents=True) + (ramp_train_dir / "val-chips").mkdir(parents=True) + (ramp_train_dir / "val-multimasks").mkdir(parents=True) + + split_info = { + "_work_dir": str(tmp_path), + "_preprocessed_dir": str(tmp_path / "preprocessed"), + "_ramp_train_dir": str(ramp_train_dir), + "strategy": "random", + "val_ratio": 0.25, + "seed": 42, + "train_count": 3, + "val_count": 1, + "description": "test split", + } + + fake_saved_model_dir = tmp_path / "saved_model" + fake_saved_model_dir.mkdir() + (fake_saved_model_dir / "saved_model.pb").write_bytes(b"\x08\x01") + + with ( + patch("models.ramp.pipeline.mlflow_training_context", _noop_mlflow_ctx, create=True), + patch("models.ramp.pipeline._restore_checkpoint", return_value=fake_saved_model_dir), + patch("models.ramp.pipeline.log_metadata"), + ): + metrics = evaluate_model.entrypoint( + trained_model=b"fake", + dataset_chips=str(toy_chips), + dataset_labels=str(toy_labels), + hyperparameters=base_hyperparameters, + split_info=split_info, + ) + + expected = {"fair:accuracy", "fair:mean_iou", "fair:precision", "fair:recall"} + assert set(metrics.keys()) == expected + for key in expected: + assert isinstance(metrics[key], float) + + +def test_export_onnx(tmp_path: Path) -> None: + import onnx + from onnx import TensorProto, helper + + from models.ramp.pipeline import export_onnx + + # Build a toy ONNX model and capture its bytes. + x = helper.make_tensor_value_info("input", TensorProto.FLOAT, [1, 1]) + y = helper.make_tensor_value_info("output", TensorProto.FLOAT, [1, 1]) + node = helper.make_node("Identity", ["input"], ["output"]) + graph = helper.make_graph([node], "toy", [x], [y]) + toy_model = helper.make_model(graph, producer_name="test") + toy_bytes = toy_model.SerializeToString() + + fake_saved_model_dir = tmp_path / "saved_model" + fake_saved_model_dir.mkdir() + (fake_saved_model_dir / "saved_model.pb").write_bytes(b"\x08\x01") + + with ( + patch("models.ramp.pipeline._restore_checkpoint", return_value=fake_saved_model_dir), + patch("models.ramp.pipeline._convert_savedmodel_to_onnx_bytes", return_value=toy_bytes), + patch("models.ramp.pipeline.log_metadata"), + ): + exported = export_onnx.entrypoint(trained_model=b"fake") + + assert isinstance(exported, bytes) + loaded = onnx.load_from_string(exported) + onnx.checker.check_model(loaded) diff --git a/pyproject.toml b/pyproject.toml index efe95dfa..162ae2f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -122,12 +122,6 @@ line-length = 120 [tool.ruff.lint] select = ["E", "F", "I", "UP", "B", "SIM", "RUF"] -# models/ramp/pipeline.py: ZenML @step / @pipeline signatures must keep typing.Union, -# Optional, List, Tuple, Dict (not PEP 604 / builtin generics). - -[tool.ruff.lint.per-file-ignores] -"models/ramp/pipeline.py" = ["UP006", "UP007", "UP035", "UP045"] - [tool.pytest.ini_options] testpaths = ["tests", "fair/serve/tests"] addopts = "--cov=fair --cov-report=term-missing" From 98d3d85f79383bc36b70a048e0a6c455d7af5edb Mon Sep 17 00:00:00 2001 From: Abdelrahman Katkat Date: Wed, 22 Apr 2026 23:35:21 +0200 Subject: [PATCH 06/30] refactor(ramp): improve model resolution logic and update STAC item sources - Refactored model resolution logic in `pipeline.py` for clarity and efficiency. - Updated `pretrained_source` and `checkpoint` URLs in `stac-item.json` to point to Hugging Face. - Adjusted metadata properties in `stac-item.json` for consistency and accuracy. --- models/ramp/pipeline.py | 39 ++++++++++++++------------------------ models/ramp/stac-item.json | 11 +++++------ 2 files changed, 19 insertions(+), 31 deletions(-) diff --git a/models/ramp/pipeline.py b/models/ramp/pipeline.py index 2a06e6ee..a4eda88c 100644 --- a/models/ramp/pipeline.py +++ b/models/ramp/pipeline.py @@ -1,18 +1,4 @@ """ZenML pipeline for RAMP (EfficientNetB0 + U-Net) building semantic segmentation. - -Follows the fAIr-models contract: platform-provided hyperparameters (no runtime STAC reads), -ONNX export for portable inference, and a module-level ``predict`` entrypoint used by -``fair.serve.base``. - -Pipeline contract: - training_pipeline(base_model_weights, dataset_chips, dataset_labels, num_classes, hyperparameters) - -> split_dataset -> train_model (bytes) -> evaluate_model (fair:* metrics) -> export_onnx (bytes) - inference_pipeline(model_uri, input_images, ...) - -> run_inference -> FeatureCollection - -Runtime: TensorFlow/Keras via ramp-fair + hot-fair-utilities (preprocessing + training). -All heavy imports (tensorflow, segmentation_models, hot_fair_utilities, tf2onnx) are lazy so -this module is importable in lightweight environments (e.g. fair.utils.model_validator AST checks). """ from __future__ import annotations @@ -26,7 +12,7 @@ import tempfile import zipfile from pathlib import Path -from typing import Annotated, Any +from typing import Annotated, Any, cast from urllib.request import urlretrieve from zenml import log_metadata, pipeline, step @@ -150,9 +136,10 @@ def resolve_model_href( raise RuntimeError(f"Zip from {model_uri} did not contain a valid SavedModel") # Directory (SavedModel) - local or remote. - resolved_dir = ( - _resolve_input_directory(model_uri, "model_uri") if "://" in model_uri else _to_local_path(model_uri, "model_uri") - ).resolve() + if "://" in model_uri: + resolved_dir = _resolve_input_directory(model_uri, "model_uri").resolve() + else: + resolved_dir = _to_local_path(model_uri, "model_uri").resolve() if resolved_dir.is_dir() and (resolved_dir / "saved_model.pb").exists(): return str(resolved_dir) if resolved_dir.exists(): @@ -163,9 +150,10 @@ def resolve_model_href( def _ensure_ramp_baseline(base_model_weights: str, data_base_path: str | Path) -> Path: """Return a local SavedModel directory for fine-tuning, downloading if necessary. - ``base_model_weights`` may be an HTTP(S) .zip URL (preferred), a local .zip, or a SavedModel - directory. A pre-provisioned baseline under ``/app/ramp-data/baseline`` (from the RAMP - utilities Docker image) is used when present and no explicit URL is provided. + ``base_model_weights`` may be an HTTP(S) .zip URL (preferred), a local .zip, + or a SavedModel directory. A pre-provisioned baseline under + ``/app/ramp-data/baseline`` (from the RAMP utilities Docker image) is used + when present and no explicit URL is provided. """ image_ck = Path("/app/ramp-data/baseline") if not base_model_weights and (image_ck / "saved_model.pb").exists(): @@ -908,10 +896,11 @@ def _patch_predictor_savedmodel_loader_for_tf215() -> None: import tensorflow as tf pred = importlib.import_module("predictor.prediction") - if getattr(pred, "_fair_models_tf215_savedmodel_patch_applied", False): + pred_mod = cast(Any, pred) + if getattr(pred_mod, "_fair_models_tf215_savedmodel_patch_applied", False): return - original_loader = getattr(pred, "_load_keras_model", None) + original_loader = getattr(pred_mod, "_load_keras_model", None) if original_loader is None: raise RuntimeError("predictor.prediction._load_keras_model not found; fairpredictor API changed.") @@ -921,8 +910,8 @@ def _safe_load_keras_model(keras_backend, path: str): return original_loader(keras_backend, path) _safe_load_keras_model._fair_models_tf215_savedmodel_loader = True # type: ignore[attr-defined] - pred._load_keras_model = _safe_load_keras_model - pred._fair_models_tf215_savedmodel_patch_applied = True + pred_mod._load_keras_model = _safe_load_keras_model + pred_mod._fair_models_tf215_savedmodel_patch_applied = True def infer_ramp_model( diff --git a/models/ramp/stac-item.json b/models/ramp/stac-item.json index dcef741f..2dc8e434 100644 --- a/models/ramp/stac-item.json +++ b/models/ramp/stac-item.json @@ -37,7 +37,7 @@ "mlm:framework": "TensorFlow", "mlm:framework_version": "2.15.1", "mlm:pretrained": true, - "mlm:pretrained_source": "https://api-prod.fair.hotosm.org/api/v1/workspace/download/ramp/baseline.zip", + "mlm:pretrained_source": "https://huggingface.co/hotosm/ramp/resolve/83c77a7e5feb3af62e3604d7bb96c6c6e9ff1a96/baseline.zip", "mlm:accelerator": "amd64", "mlm:accelerator_constrained": false, "mlm:total_parameters": 7500000, @@ -45,8 +45,7 @@ "keywords": [ "building", "semantic-segmentation", - "polygon", - "ramp" + "polygon" ], "license": "Apache-2.0", "version": "1", @@ -185,14 +184,14 @@ }, "assets": { "checkpoint": { - "href": "https://api-prod.fair.hotosm.org/api/v1/workspace/download/ramp/baseline.zip", + "href": "https://huggingface.co/hotosm/ramp/resolve/83c77a7e5feb3af62e3604d7bb96c6c6e9ff1a96/baseline.zip", "title": "RAMP EfficientNetB0 + U-Net baseline weights (zipped TF SavedModel)", "type": "application/zip; framework=tensorflow", "roles": [ "mlm:model", "mlm:weights" ], - "mlm:artifact_type": "tf.keras.Model", + "mlm:artifact_type": "tf.keras.Model.save", "raster:bands": [ {"name": "red"}, {"name": "green"}, @@ -200,7 +199,7 @@ ] }, "model": { - "href": "https://api-prod.fair.hotosm.org/api/v1/workspace/download/ramp/ramp-v1.onnx", + "href": "https://huggingface.co/hotosm/ramp/resolve/83c77a7e5feb3af62e3604d7bb96c6c6e9ff1a96/ramp-v1.onnx", "title": "Portable ONNX inference model", "type": "application/octet-stream; framework=onnx", "roles": [ From 0cdc85b98439a033861230e7fb3782cf3c93680d Mon Sep 17 00:00:00 2001 From: Abdelrahman Katkat Date: Thu, 23 Apr 2026 10:12:52 +0200 Subject: [PATCH 07/30] fix(ramp): update pretrained model source URLs in STAC item --- models/ramp/stac-item.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/ramp/stac-item.json b/models/ramp/stac-item.json index 2dc8e434..e488bf32 100644 --- a/models/ramp/stac-item.json +++ b/models/ramp/stac-item.json @@ -37,7 +37,7 @@ "mlm:framework": "TensorFlow", "mlm:framework_version": "2.15.1", "mlm:pretrained": true, - "mlm:pretrained_source": "https://huggingface.co/hotosm/ramp/resolve/83c77a7e5feb3af62e3604d7bb96c6c6e9ff1a96/baseline.zip", + "mlm:pretrained_source": "https://api-prod.fair.hotosm.org/api/v1/workspace/download/ramp/baseline.zip", "mlm:accelerator": "amd64", "mlm:accelerator_constrained": false, "mlm:total_parameters": 7500000, @@ -184,7 +184,7 @@ }, "assets": { "checkpoint": { - "href": "https://huggingface.co/hotosm/ramp/resolve/83c77a7e5feb3af62e3604d7bb96c6c6e9ff1a96/baseline.zip", + "href": "https://api-prod.fair.hotosm.org/api/v1/workspace/download/ramp/baseline.zip", "title": "RAMP EfficientNetB0 + U-Net baseline weights (zipped TF SavedModel)", "type": "application/zip; framework=tensorflow", "roles": [ From b582bf0f380bda4128920b1f98062fd3d091a057 Mon Sep 17 00:00:00 2001 From: Abdelrahman Katkat Date: Tue, 28 Apr 2026 13:43:45 +0300 Subject: [PATCH 08/30] refactor(ramp): streamline model resolution and enhance ZIP handling - Simplified the `resolve_model_href` function to focus on .onnx and .zip formats. - Improved error handling for unsupported model formats and missing files. - Updated ZIP extraction logic to ensure proper directory creation and cache management. - Removed deprecated code and comments for better readability. --- models/ramp/pipeline.py | 162 ++++++++++------------------------------ 1 file changed, 39 insertions(+), 123 deletions(-) diff --git a/models/ramp/pipeline.py b/models/ramp/pipeline.py index a4eda88c..cfbeb693 100644 --- a/models/ramp/pipeline.py +++ b/models/ramp/pipeline.py @@ -27,12 +27,6 @@ "https://github.com/qubvel/efficientnet/releases/download/v0.0.1/" ) - -# --------------------------------------------------------------------------- -# Path / resource resolvers -# --------------------------------------------------------------------------- - - def _to_local_path(path_value: str, purpose: str) -> Path: """Resolve a path with UPath and ensure local filesystem semantics.""" from upath import UPath @@ -67,7 +61,7 @@ def _resolve_input_file(path_value: str, purpose: str) -> Path: def _download_and_extract_zip(zip_url: str, dest_dir: Path) -> None: """Download a ZIP URL and extract in dest_dir.""" dest_dir.mkdir(parents=True, exist_ok=True) - zip_name = Path(zip_url.split("/")[-1]).name or "archive.zip" + zip_name = Path(zip_url.split("/")[-1]).name or "ramp_v1.zip" zip_path = dest_dir / zip_name urlretrieve(zip_url, zip_path) with zipfile.ZipFile(zip_path, "r") as zf: @@ -81,71 +75,57 @@ def _extract_zip(zip_path: Path, dest_dir: Path) -> None: zf.extractall(dest_dir) -def resolve_model_href( - model_uri: str, - cache_dir: Path | None = None, -) -> str: - """Resolve model_uri to a local path. - - Supports: - - Local SavedModel directory → returned as-is - - Local .zip file containing SavedModel → extracted, returned as directory - - HTTP(S) URL to .zip → downloaded, extracted, cached, returned as directory - - Local / HTTP .onnx → downloaded (if needed) and returned as file path +def resolve_model_href(model_uri: str, cache_dir: Path | None = None) -> str: + """Resolve only .onnx or .zip model URIs to local paths. + - .onnx -> local .onnx file path + - .zip -> extracted SavedModel directory path (folder containing saved_model.pb) """ if not isinstance(model_uri, str): raise TypeError("model_uri must be a string") if cache_dir is not None and not isinstance(cache_dir, Path): raise TypeError("cache_dir must be a pathlib.Path or None") - cache_dir = cache_dir or _DEFAULT_MODEL_CACHE + cache_dir.mkdir(parents=True, exist_ok=True) is_http = model_uri.startswith(("http://", "https://")) clean_uri = model_uri.split("?", 1)[0] suffix = Path(clean_uri).suffix.lower() - - # ONNX: return as local file path. + # ONNX path if suffix == ".onnx": if is_http: - cache_dir.mkdir(parents=True, exist_ok=True) - base_name = Path(clean_uri).name or "model.onnx" - dest = cache_dir / base_name + dest = cache_dir / (Path(clean_uri).name or "model.onnx") if not dest.is_file(): urlretrieve(model_uri, dest) return str(dest) - resolved = _to_local_path(model_uri, "model_uri").resolve() - if resolved.is_file(): - return str(resolved) - raise FileNotFoundError(f"ONNX model not found: {resolved}") - - # Zipped SavedModel. + # local onnx path + p = Path(model_uri).resolve() + if p.is_file(): + return str(p) + raise FileNotFoundError(f"ONNX model not found: {p}") + # ZIP path -> must contain saved_model.pb somewhere inside if suffix == ".zip": - base_name = Path(clean_uri).name - stem = Path(base_name).stem or "ramp_model" + stem = Path(clean_uri).stem or "ramp_model" dest_dir = cache_dir / stem + # cache hit for existing in dest_dir.rglob("saved_model.pb"): return str(existing.parent) - if is_http: - _download_and_extract_zip(model_uri, dest_dir) + zip_path = cache_dir / (Path(clean_uri).name or "model.zip") + if not zip_path.is_file(): + urlretrieve(model_uri, zip_path) else: - local_zip = _resolve_input_file(model_uri, "model_uri") - _extract_zip(local_zip, dest_dir) - - for sub in dest_dir.rglob("saved_model.pb"): - return str(sub.parent) + zip_path = Path(model_uri).resolve() + if not zip_path.is_file(): + raise FileNotFoundError(f"ZIP model not found: {zip_path}") + dest_dir.mkdir(parents=True, exist_ok=True) + import zipfile + with zipfile.ZipFile(zip_path, "r") as zf: + zf.extractall(dest_dir) + for existing in dest_dir.rglob("saved_model.pb"): + return str(existing.parent) raise RuntimeError(f"Zip from {model_uri} did not contain a valid SavedModel") - - # Directory (SavedModel) - local or remote. - if "://" in model_uri: - resolved_dir = _resolve_input_directory(model_uri, "model_uri").resolve() - else: - resolved_dir = _to_local_path(model_uri, "model_uri").resolve() - if resolved_dir.is_dir() and (resolved_dir / "saved_model.pb").exists(): - return str(resolved_dir) - if resolved_dir.exists(): - raise FileNotFoundError(f"SavedModel directory missing saved_model.pb: {resolved_dir}") - raise FileNotFoundError(f"Model path not found: {resolved_dir}") - + raise ValueError( + f"Unsupported model format for {model_uri!r}. Only .onnx and .zip are accepted." + ) def _ensure_ramp_baseline(base_model_weights: str, data_base_path: str | Path) -> Path: """Return a local SavedModel directory for fine-tuning, downloading if necessary. @@ -165,11 +145,6 @@ def _ensure_ramp_baseline(base_model_weights: str, data_base_path: str | Path) - return Path(resolve_model_href(base_model_weights, cache_dir=Path(data_base_path) / ".baseline_cache")) -# --------------------------------------------------------------------------- -# Keras / segmentation_models compatibility patches -# --------------------------------------------------------------------------- - - def _patch_keras_get_file_for_efficientnet_weights() -> None: """Redirect broken Callidior EfficientNet weight URLs to qubvel's GitHub release assets. @@ -180,63 +155,27 @@ def _patch_keras_get_file_for_efficientnet_weights() -> None: import tensorflow as tf ku = tf.keras.utils - if getattr(ku.get_file, "_ramp_efficientnet_mirror", False): + if hasattr(ku.get_file, "_ramp_efficientnet_mirror"): return - _orig = ku.get_file + original = ku.get_file def _get_file(fname, origin, *args, **kwargs): - if isinstance(origin, str) and "Callidior" in origin and isinstance(fname, str): + if isinstance(origin, str) and "Callidior" in origin: m = re.match( r"^(efficientnet-b\d+)_weights_tf_dim_ordering_tf_kernels_autoaugment_notop\.h5$", - fname, + fname or "", ) if m: alt = f"{m.group(1)}_imagenet_1000_notop.h5" origin = f"{_QUBVEL_EFFICIENTNET_RELEASE}{alt}" kwargs = dict(kwargs) kwargs["file_hash"] = None - return _orig(fname, origin, *args, **kwargs) + return original(fname, origin, *args, **kwargs) _get_file._ramp_efficientnet_mirror = True # type: ignore[attr-defined] ku.get_file = _get_file - -# --------------------------------------------------------------------------- -# Dataset materialization (chips + labels → hot_fair_utilities input layout) -# --------------------------------------------------------------------------- - - -def _select_or_merge_labels(labels_path: Path, destination: Path) -> None: - """Materialize a single labels.geojson for hot_fair_utilities preprocess.""" - if labels_path.is_file(): - shutil.copy2(labels_path, destination) - return - - if not labels_path.is_dir(): - raise FileNotFoundError(f"dataset_labels path not found: {labels_path}") - - geojson_files = sorted(labels_path.glob("*.geojson")) - if not geojson_files: - raise FileNotFoundError(f"No .geojson files found in labels directory: {labels_path}") - if len(geojson_files) == 1: - shutil.copy2(geojson_files[0], destination) - return - - import geopandas as gpd - import pandas as pd - - gdfs = [gpd.read_file(p) for p in geojson_files] - crs = gdfs[0].crs or "EPSG:4326" - merged = gpd.GeoDataFrame(pd.concat([g.to_crs(crs) for g in gdfs], ignore_index=True), crs=crs) - for col in merged.columns: - if col == "geometry": - continue - if pd.api.types.is_extension_array_dtype(merged[col].dtype): - merged[col] = merged[col].astype(object).where(merged[col].notna(), None) - merged.to_file(destination, driver="GeoJSON") - - def _materialize_training_input(dataset_chips: str, dataset_labels: str, work_dir: Path) -> Path: """Create the preprocess input folder with PNG chips and a single labels.geojson. @@ -276,7 +215,9 @@ def _materialize_training_input(dataset_chips: str, dataset_labels: str, work_di if not list(input_dir.glob("*.png")): raise FileNotFoundError(f"No train chips (.tif/.tiff/.png) found in {chips_dir}") - _select_or_merge_labels(labels_path, input_dir / "labels.geojson") + if not labels_path.is_file(): + raise FileNotFoundError(f"dataset_labels file not found: {labels_path}") + shutil.copy2(labels_path, input_dir / "labels.geojson") return input_dir @@ -284,12 +225,6 @@ def _training_cache_dir(dataset_chips: str, dataset_labels: str) -> Path: cache_key = hashlib.sha256(f"{dataset_chips}|{dataset_labels}".encode()).hexdigest()[:16] return Path(tempfile.gettempdir()) / f"ramp_training_{cache_key}" - -# --------------------------------------------------------------------------- -# Preprocess / postprocess (STAC pre/post_processing_function references) -# --------------------------------------------------------------------------- - - def preprocess( input_path: str, output_path: str, @@ -384,11 +319,6 @@ def postprocess(prediction_masks_dir: str, output_dir: str) -> dict[str, Any]: return json.loads(gdf.to_json()) -# --------------------------------------------------------------------------- -# Train / eval / export helpers (stateful TF pieces stay behind lazy imports) -# --------------------------------------------------------------------------- - - def _prepare_training_split( dataset_chips: str, dataset_labels: str, @@ -567,11 +497,6 @@ def _convert_savedmodel_to_onnx_bytes(saved_model_dir: Path, opset: int = 13) -> return onnx_path.read_bytes() -# --------------------------------------------------------------------------- -# ONNX serving: predict(session, input_images, params) -> FeatureCollection -# --------------------------------------------------------------------------- - - def _build_feature_collection(features: list[dict[str, Any]]) -> dict[str, Any]: return {"type": "FeatureCollection", "features": features} @@ -711,10 +636,6 @@ def predict(session: Any, input_images: str, params: dict[str, Any]) -> dict[str return _build_feature_collection(features) -# --------------------------------------------------------------------------- -# ZenML @step primitives -# --------------------------------------------------------------------------- - @step def split_dataset( @@ -972,11 +893,6 @@ def run_postprocessing(prediction_path: str, output_dir: str) -> dict[str, Any]: return postprocess(prediction_path, output_dir) -# --------------------------------------------------------------------------- -# @pipeline definitions -# --------------------------------------------------------------------------- - - @pipeline def training_pipeline( base_model_weights: str, From 96ca4ede6dce78f70fd2fba7198d08c970280ed8 Mon Sep 17 00:00:00 2001 From: Abdelrahman Katkat Date: Tue, 28 Apr 2026 14:32:23 +0300 Subject: [PATCH 09/30] refactor(ramp): clean up code formatting and improve readability in pipeline.py - Consolidated multi-line string definitions into single lines for consistency. - Removed unnecessary blank lines to enhance code clarity. - Streamlined parameter retrieval in several functions for better readability. --- models/ramp/pipeline.py | 41 ++++++++++++++--------------------------- 1 file changed, 14 insertions(+), 27 deletions(-) diff --git a/models/ramp/pipeline.py b/models/ramp/pipeline.py index cfbeb693..8bb294fe 100644 --- a/models/ramp/pipeline.py +++ b/models/ramp/pipeline.py @@ -1,5 +1,4 @@ -"""ZenML pipeline for RAMP (EfficientNetB0 + U-Net) building semantic segmentation. -""" +"""ZenML pipeline for RAMP (EfficientNetB0 + U-Net) building semantic segmentation.""" from __future__ import annotations @@ -20,12 +19,9 @@ from fair.zenml.steps import load_model _DEFAULT_MODEL_CACHE = Path("/workspace/.ramp_model_cache") -_DEFAULT_RAMP_BASELINE_URL = ( - "https://api-prod.fair.hotosm.org/api/v1/workspace/download/ramp/baseline.zip" -) -_QUBVEL_EFFICIENTNET_RELEASE = ( - "https://github.com/qubvel/efficientnet/releases/download/v0.0.1/" -) +_DEFAULT_RAMP_BASELINE_URL = "https://api-prod.fair.hotosm.org/api/v1/workspace/download/ramp/baseline.zip" +_QUBVEL_EFFICIENTNET_RELEASE = "https://github.com/qubvel/efficientnet/releases/download/v0.0.1/" + def _to_local_path(path_value: str, purpose: str) -> Path: """Resolve a path with UPath and ensure local filesystem semantics.""" @@ -118,14 +114,14 @@ def resolve_model_href(model_uri: str, cache_dir: Path | None = None) -> str: raise FileNotFoundError(f"ZIP model not found: {zip_path}") dest_dir.mkdir(parents=True, exist_ok=True) import zipfile + with zipfile.ZipFile(zip_path, "r") as zf: zf.extractall(dest_dir) for existing in dest_dir.rglob("saved_model.pb"): return str(existing.parent) raise RuntimeError(f"Zip from {model_uri} did not contain a valid SavedModel") - raise ValueError( - f"Unsupported model format for {model_uri!r}. Only .onnx and .zip are accepted." - ) + raise ValueError(f"Unsupported model format for {model_uri!r}. Only .onnx and .zip are accepted.") + def _ensure_ramp_baseline(base_model_weights: str, data_base_path: str | Path) -> Path: """Return a local SavedModel directory for fine-tuning, downloading if necessary. @@ -176,6 +172,7 @@ def _get_file(fname, origin, *args, **kwargs): _get_file._ramp_efficientnet_mirror = True # type: ignore[attr-defined] ku.get_file = _get_file + def _materialize_training_input(dataset_chips: str, dataset_labels: str, work_dir: Path) -> Path: """Create the preprocess input folder with PNG chips and a single labels.geojson. @@ -225,6 +222,7 @@ def _training_cache_dir(dataset_chips: str, dataset_labels: str) -> Path: cache_key = hashlib.sha256(f"{dataset_chips}|{dataset_labels}".encode()).hexdigest()[:16] return Path(tempfile.gettempdir()) / f"ramp_training_{cache_key}" + def preprocess( input_path: str, output_path: str, @@ -336,12 +334,8 @@ def _prepare_training_split( ) if not 0.0 < val_fraction < 1.0: raise ValueError("val_fraction must be in (0.0, 1.0)") - boundary_width = int( - hyperparameters.get("training.boundary_width", hyperparameters.get("boundary_width", 3)) - ) - contact_spacing = int( - hyperparameters.get("training.contact_spacing", hyperparameters.get("contact_spacing", 8)) - ) + boundary_width = int(hyperparameters.get("training.boundary_width", hyperparameters.get("boundary_width", 3))) + contact_spacing = int(hyperparameters.get("training.contact_spacing", hyperparameters.get("contact_spacing", 8))) seed = int(hyperparameters.get("training.split_seed", hyperparameters.get("split_seed", 42))) work_dir = _training_cache_dir(dataset_chips, dataset_labels) @@ -414,9 +408,7 @@ def train_ramp_model( learning_rate = float( hyperparameters.get( "training.learning_rate", - hyperparameters.get( - "learning_rate", RAMP_CONFIG["optimizer"]["optimizer_fn_parms"]["learning_rate"] - ), + hyperparameters.get("learning_rate", RAMP_CONFIG["optimizer"]["optimizer_fn_parms"]["learning_rate"]), ) ) patience = int( @@ -636,7 +628,6 @@ def predict(session: Any, input_images: str, params: dict[str, Any]) -> dict[str return _build_feature_collection(features) - @step def split_dataset( dataset_chips: str, @@ -666,9 +657,7 @@ def train_model( ramp_train_dir = Path(split_info["_ramp_train_dir"]) if not ramp_train_dir.exists(): - split_info = _prepare_training_split( - dataset_chips, dataset_labels, hyperparameters, force_rebuild=True - ) + split_info = _prepare_training_split(dataset_chips, dataset_labels, hyperparameters, force_rebuild=True) ramp_train_dir = Path(split_info["_ramp_train_dir"]) work_dir = split_info.get("_work_dir") or str(ramp_train_dir.parent) @@ -704,9 +693,7 @@ def evaluate_model( ramp_train_dir = Path(split_info.get("_ramp_train_dir", "")) if not ramp_train_dir.exists(): - split_info = _prepare_training_split( - dataset_chips, dataset_labels, hyperparameters, force_rebuild=True - ) + split_info = _prepare_training_split(dataset_chips, dataset_labels, hyperparameters, force_rebuild=True) ramp_train_dir = Path(split_info["_ramp_train_dir"]) val_chips_dir = ramp_train_dir / "val-chips" From a2195de49bf6c5e6c99b888432aa32d880a06076 Mon Sep 17 00:00:00 2001 From: Abdelrahman Katkat Date: Wed, 29 Apr 2026 13:15:53 +0300 Subject: [PATCH 10/30] fix(ramp): enforce requirement for RAMP baseline weights and update STAC item references - Removed the default baseline URL in `pipeline.py` and raised a ValueError if weights are not provided. - Updated `pretrained_source` and `checkpoint` URLs in `stac-item.json` to point to the new Hugging Face location. - Adjusted the `create_toy_data` function to ensure it uses a GeoJSON file for labels, aligning with the expectations in `pipeline.py`. --- models/ramp/pipeline.py | 8 +++++--- models/ramp/stac-item.json | 4 ++-- models/ramp/tests/conftest.py | 9 +++++---- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/models/ramp/pipeline.py b/models/ramp/pipeline.py index 8bb294fe..33cd2d90 100644 --- a/models/ramp/pipeline.py +++ b/models/ramp/pipeline.py @@ -19,7 +19,6 @@ from fair.zenml.steps import load_model _DEFAULT_MODEL_CACHE = Path("/workspace/.ramp_model_cache") -_DEFAULT_RAMP_BASELINE_URL = "https://api-prod.fair.hotosm.org/api/v1/workspace/download/ramp/baseline.zip" _QUBVEL_EFFICIENTNET_RELEASE = "https://github.com/qubvel/efficientnet/releases/download/v0.0.1/" @@ -129,14 +128,17 @@ def _ensure_ramp_baseline(base_model_weights: str, data_base_path: str | Path) - ``base_model_weights`` may be an HTTP(S) .zip URL (preferred), a local .zip, or a SavedModel directory. A pre-provisioned baseline under ``/app/ramp-data/baseline`` (from the RAMP utilities Docker image) is used - when present and no explicit URL is provided. + when present and no explicit URL is provided. If the pre-provisioned baseline + is missing, callers must provide `base_model_weights` (typically from STAC). """ image_ck = Path("/app/ramp-data/baseline") if not base_model_weights and (image_ck / "saved_model.pb").exists(): return image_ck if not base_model_weights: - base_model_weights = _DEFAULT_RAMP_BASELINE_URL + raise ValueError( + "RAMP baseline weights are required but were not provided. " + ) return Path(resolve_model_href(base_model_weights, cache_dir=Path(data_base_path) / ".baseline_cache")) diff --git a/models/ramp/stac-item.json b/models/ramp/stac-item.json index e488bf32..60a593f1 100644 --- a/models/ramp/stac-item.json +++ b/models/ramp/stac-item.json @@ -37,7 +37,7 @@ "mlm:framework": "TensorFlow", "mlm:framework_version": "2.15.1", "mlm:pretrained": true, - "mlm:pretrained_source": "https://api-prod.fair.hotosm.org/api/v1/workspace/download/ramp/baseline.zip", + "mlm:pretrained_source": "https://huggingface.co/hotosm/ramp/resolve/74daea54694f2e4924f1222520c614c7f5c029fe/v1-baseline.zip", "mlm:accelerator": "amd64", "mlm:accelerator_constrained": false, "mlm:total_parameters": 7500000, @@ -184,7 +184,7 @@ }, "assets": { "checkpoint": { - "href": "https://api-prod.fair.hotosm.org/api/v1/workspace/download/ramp/baseline.zip", + "href": "https://huggingface.co/hotosm/ramp/resolve/74daea54694f2e4924f1222520c614c7f5c029fe/v1-baseline.zip", "title": "RAMP EfficientNetB0 + U-Net baseline weights (zipped TF SavedModel)", "type": "application/zip; framework=tensorflow", "roles": [ diff --git a/models/ramp/tests/conftest.py b/models/ramp/tests/conftest.py index 1a9de9a2..4086e382 100644 --- a/models/ramp/tests/conftest.py +++ b/models/ramp/tests/conftest.py @@ -80,8 +80,9 @@ def create_toy_data(root: Path) -> dict[str, Path]: labels_geojson.write_text(json.dumps({"type": "FeatureCollection", "features": features})) stac_path = root / "dataset-stac-item.json" - stac_path.write_text(json.dumps(_build_dataset_stac_item(chips_dir, labels_dir), indent=2)) - return {"chips": chips_dir, "labels": labels_dir, "dataset_stac_item": stac_path} + stac_path.write_text(json.dumps(_build_dataset_stac_item(chips_dir, labels_geojson), indent=2)) + # `models/ramp/pipeline.py` expects `dataset_labels` to be a single GeoJSON file. + return {"chips": chips_dir, "labels": labels_geojson, "dataset_stac_item": stac_path} @pytest.fixture(scope="session") @@ -89,7 +90,7 @@ def generate_toy_dataset(tmp_path_factory: pytest.TempPathFactory) -> dict[str, return create_toy_data(tmp_path_factory.mktemp("toy_ramp")) -def _build_dataset_stac_item(chips_dir: Path, labels_dir: Path) -> dict[str, Any]: +def _build_dataset_stac_item(chips_dir: Path, labels_geojson: Path) -> dict[str, Any]: return { "type": "Feature", "stac_version": "1.1.0", @@ -122,7 +123,7 @@ def _build_dataset_stac_item(chips_dir: Path, labels_dir: Path) -> dict[str, Any }, "assets": { "chips": {"href": str(chips_dir), "type": "image/tiff", "roles": ["data"]}, - "labels": {"href": str(labels_dir), "type": "application/geo+json", "roles": ["labels"]}, + "labels": {"href": str(labels_geojson), "type": "application/geo+json", "roles": ["labels"]}, }, "links": [], } From a88076fee34ed748ec253455b0eb0f7722192d32 Mon Sep 17 00:00:00 2001 From: Abdelrahman Katkat Date: Thu, 30 Apr 2026 17:10:24 +0300 Subject: [PATCH 11/30] refactor(ramp): enhance type hinting and improve model handling in pipeline.py - Updated type hints for several functions to use Optional and Union for better clarity. - Introduced a new function `_normalize_to_savedmodel_dir` to streamline model path normalization. - Improved error handling and readability in model resolution and checkpoint restoration logic. - Consolidated ZIP handling and model loading processes for better maintainability. --- models/ramp/pipeline.py | 78 ++++++++++++++++++++++++++--------------- 1 file changed, 50 insertions(+), 28 deletions(-) diff --git a/models/ramp/pipeline.py b/models/ramp/pipeline.py index 33cd2d90..c5c8fade 100644 --- a/models/ramp/pipeline.py +++ b/models/ramp/pipeline.py @@ -1,7 +1,5 @@ """ZenML pipeline for RAMP (EfficientNetB0 + U-Net) building semantic segmentation.""" -from __future__ import annotations - import hashlib import io import json @@ -11,7 +9,7 @@ import tempfile import zipfile from pathlib import Path -from typing import Annotated, Any, cast +from typing import Annotated, Any, Optional, Union, cast from urllib.request import urlretrieve from zenml import log_metadata, pipeline, step @@ -70,7 +68,7 @@ def _extract_zip(zip_path: Path, dest_dir: Path) -> None: zf.extractall(dest_dir) -def resolve_model_href(model_uri: str, cache_dir: Path | None = None) -> str: +def resolve_model_href(model_uri: str, cache_dir: Optional[Path] = None) -> str: """Resolve only .onnx or .zip model URIs to local paths. - .onnx -> local .onnx file path - .zip -> extracted SavedModel directory path (folder containing saved_model.pb) @@ -122,7 +120,7 @@ def resolve_model_href(model_uri: str, cache_dir: Path | None = None) -> str: raise ValueError(f"Unsupported model format for {model_uri!r}. Only .onnx and .zip are accepted.") -def _ensure_ramp_baseline(base_model_weights: str, data_base_path: str | Path) -> Path: +def _ensure_ramp_baseline(base_model_weights: str, data_base_path: Union[str, Path]) -> Path: """Return a local SavedModel directory for fine-tuning, downloading if necessary. ``base_model_weights`` may be an HTTP(S) .zip URL (preferred), a local .zip, @@ -136,9 +134,7 @@ def _ensure_ramp_baseline(base_model_weights: str, data_base_path: str | Path) - return image_ck if not base_model_weights: - raise ValueError( - "RAMP baseline weights are required but were not provided. " - ) + raise ValueError("RAMP baseline weights are required but were not provided. ") return Path(resolve_model_href(base_model_weights, cache_dir=Path(data_base_path) / ".baseline_cache")) @@ -373,7 +369,7 @@ def train_ramp_model( ramp_train_dir: str, base_model_weights: str, hyperparameters: dict[str, Any], - data_base_path: str | None = None, + data_base_path: Optional[str] = None, ) -> Path: """Fine-tune EfficientNetB0 + U-Net and return the best SavedModel directory path.""" # run_training reads RAMP_HOME at import time; set it first so saved_model lookups resolve. @@ -464,30 +460,58 @@ def _unzip_savedmodel_bytes(blob: bytes) -> Path: raise RuntimeError("Zipped bytes do not contain a SavedModel (no saved_model.pb found).") +def _normalize_to_savedmodel_dir(model_path: Union[str, Path], context: str = "model") -> Path: + """Return a directory containing `saved_model.pb` for a produced/loaded model path.""" + candidate = Path(str(model_path)) + if candidate.is_dir(): + if (candidate / "saved_model.pb").exists(): + return candidate + for nested in candidate.rglob("saved_model.pb"): + return nested.parent + elif candidate.is_file() and candidate.suffix.lower() in {".h5", ".keras"}: + import tensorflow as tf + + model = tf.keras.models.load_model(str(candidate), compile=False) + exported_dir = Path(tempfile.mkdtemp(prefix="ramp_savedmodel_export_")) / "saved_model" + model.save(str(exported_dir), save_format="tf") + if (exported_dir / "saved_model.pb").exists(): + return exported_dir + raise RuntimeError(f"{context} must resolve to a SavedModel directory containing saved_model.pb; got: {candidate}") + + def _restore_checkpoint(trained_model: Any) -> Path: """Restore a trained RAMP SavedModel from bytes, a SavedModel directory, or a .zip file.""" if isinstance(trained_model, bytes): return _unzip_savedmodel_bytes(trained_model) if isinstance(trained_model, (str, Path)): p = Path(str(trained_model)) - if p.is_dir() and (p / "saved_model.pb").exists(): - return p if p.is_file() and p.suffix.lower() == ".zip": return _unzip_savedmodel_bytes(p.read_bytes()) + return _normalize_to_savedmodel_dir(p, context="trained_model") raise TypeError(f"Cannot restore RAMP checkpoint from {type(trained_model).__name__}") def _convert_savedmodel_to_onnx_bytes(saved_model_dir: Path, opset: int = 13) -> bytes: """Convert a TF SavedModel directory to ONNX bytes via tf2onnx.""" import tf2onnx + import tensorflow as tf with tempfile.TemporaryDirectory() as tmp: onnx_path = Path(tmp) / "model.onnx" - tf2onnx.convert.from_saved_model( - str(saved_model_dir), - output_path=str(onnx_path), - opset=opset, - ) + from_saved_model = getattr(tf2onnx.convert, "from_saved_model", None) + if callable(from_saved_model): + from_saved_model( + str(saved_model_dir), + output_path=str(onnx_path), + opset=opset, + ) + else: + model = tf.keras.models.load_model(str(saved_model_dir), compile=False) + tf2onnx.convert.from_keras( + model, + opset=opset, + output_path=str(onnx_path), + ) return onnx_path.read_bytes() @@ -650,9 +674,9 @@ def train_model( hyperparameters: dict[str, Any], split_info: dict[str, Any], num_classes: int = 4, - model_name: str | None = None, - base_model_id: str | None = None, - dataset_id: str | None = None, + model_name: Optional[str] = None, + base_model_id: Optional[str] = None, + dataset_id: Optional[str] = None, ) -> Annotated[bytes, "trained_model"]: """Fine-tune RAMP EfficientNetB0 U-Net; return the best SavedModel as zipped bytes.""" _ = (num_classes, model_name, base_model_id, dataset_id) @@ -669,9 +693,7 @@ def train_model( hyperparameters=hyperparameters, data_base_path=work_dir, ) - saved_model_dir = final_model_path if final_model_path.is_dir() else final_model_path.parent - if not (saved_model_dir / "saved_model.pb").exists(): - raise RuntimeError(f"Expected SavedModel at {saved_model_dir}; not found.") + saved_model_dir = _normalize_to_savedmodel_dir(final_model_path, context="train_model output") blob = _zip_savedmodel_dir(saved_model_dir) log_metadata(metadata={"saved_model_dir": str(saved_model_dir), "checkpoint_bytes": len(blob)}) @@ -685,7 +707,7 @@ def evaluate_model( dataset_labels: str, hyperparameters: dict[str, Any], split_info: dict[str, Any], - class_names: list[str] | None = None, + class_names: Optional[list[str]] = None, ) -> Annotated[dict[str, Any], "metrics"]: """Compute per-pixel building-class metrics on the validation split.""" _ = class_names @@ -778,12 +800,12 @@ def export_onnx(trained_model: Any) -> Annotated[bytes, "onnx_model"]: @step def run_inference( - model_uri: str | Path | Any, + model_uri: Union[str, Path, Any], input_images: str, prediction_path: str, output_dir: str, confidence: float = 0.5, - model_cache_dir: str | None = None, + model_cache_dir: Optional[str] = None, ) -> Annotated[dict[str, Any], "predictions"]: """Native-TF inference over georeferenced chips → building-footprint GeoJSON.""" return infer_ramp_model( @@ -825,12 +847,12 @@ def _safe_load_keras_model(keras_backend, path: str): def infer_ramp_model( - model_uri: str | Path | Any, + model_uri: Union[str, Path, Any], input_path: str, prediction_path: str, output_dir: str, confidence: float = 0.5, - model_cache_dir: str | None = None, + model_cache_dir: Optional[str] = None, ) -> dict[str, Any]: """Run fairpredictor-style TF inference and return the merged GeoJSON content.""" _patch_predictor_savedmodel_loader_for_tf215() @@ -918,7 +940,7 @@ def training_pipeline( def inference_pipeline( model_uri: str, input_images: str, - inference_params: dict[str, Any] | None = None, + inference_params: Optional[dict[str, Any]] = None, output_dir: str = "", chip_size: int = 256, num_classes: int = 4, From 79314fd45cb5646bb2a4d20239a2a7046f42b5ab Mon Sep 17 00:00:00 2001 From: Abdelrahman Katkat Date: Tue, 5 May 2026 12:02:38 +0300 Subject: [PATCH 12/30] chore(ci): update just version to 1.39.0 in workflows --- .github/workflows/k8s-integration.yml | 2 ++ .github/workflows/validate-stac.yml | 2 ++ models/ramp/pipeline.py | 30 +++++++++++++-------------- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/.github/workflows/k8s-integration.yml b/.github/workflows/k8s-integration.yml index a2c25b86..5833fba8 100644 --- a/.github/workflows/k8s-integration.yml +++ b/.github/workflows/k8s-integration.yml @@ -34,6 +34,8 @@ jobs: enable-cache: true - uses: extractions/setup-just@v2 + with: + just-version: "1.39.0" - uses: helm/kind-action@v1 with: diff --git a/.github/workflows/validate-stac.yml b/.github/workflows/validate-stac.yml index 899e71d3..ace9332a 100644 --- a/.github/workflows/validate-stac.yml +++ b/.github/workflows/validate-stac.yml @@ -36,6 +36,8 @@ jobs: enable-cache: true - name: Install just uses: extractions/setup-just@v2 + with: + just-version: "1.39.0" - name: Install dependencies run: uv sync --group dev - name: Validate STAC items diff --git a/models/ramp/pipeline.py b/models/ramp/pipeline.py index c5c8fade..6460d6e9 100644 --- a/models/ramp/pipeline.py +++ b/models/ramp/pipeline.py @@ -9,7 +9,7 @@ import tempfile import zipfile from pathlib import Path -from typing import Annotated, Any, Optional, Union, cast +from typing import Annotated, Any, cast from urllib.request import urlretrieve from zenml import log_metadata, pipeline, step @@ -68,7 +68,7 @@ def _extract_zip(zip_path: Path, dest_dir: Path) -> None: zf.extractall(dest_dir) -def resolve_model_href(model_uri: str, cache_dir: Optional[Path] = None) -> str: +def resolve_model_href(model_uri: str, cache_dir: Path | None = None) -> str: """Resolve only .onnx or .zip model URIs to local paths. - .onnx -> local .onnx file path - .zip -> extracted SavedModel directory path (folder containing saved_model.pb) @@ -120,7 +120,7 @@ def resolve_model_href(model_uri: str, cache_dir: Optional[Path] = None) -> str: raise ValueError(f"Unsupported model format for {model_uri!r}. Only .onnx and .zip are accepted.") -def _ensure_ramp_baseline(base_model_weights: str, data_base_path: Union[str, Path]) -> Path: +def _ensure_ramp_baseline(base_model_weights: str, data_base_path: str | Path) -> Path: """Return a local SavedModel directory for fine-tuning, downloading if necessary. ``base_model_weights`` may be an HTTP(S) .zip URL (preferred), a local .zip, @@ -369,7 +369,7 @@ def train_ramp_model( ramp_train_dir: str, base_model_weights: str, hyperparameters: dict[str, Any], - data_base_path: Optional[str] = None, + data_base_path: str | None = None, ) -> Path: """Fine-tune EfficientNetB0 + U-Net and return the best SavedModel directory path.""" # run_training reads RAMP_HOME at import time; set it first so saved_model lookups resolve. @@ -460,7 +460,7 @@ def _unzip_savedmodel_bytes(blob: bytes) -> Path: raise RuntimeError("Zipped bytes do not contain a SavedModel (no saved_model.pb found).") -def _normalize_to_savedmodel_dir(model_path: Union[str, Path], context: str = "model") -> Path: +def _normalize_to_savedmodel_dir(model_path: str | Path, context: str = "model") -> Path: """Return a directory containing `saved_model.pb` for a produced/loaded model path.""" candidate = Path(str(model_path)) if candidate.is_dir(): @@ -493,8 +493,8 @@ def _restore_checkpoint(trained_model: Any) -> Path: def _convert_savedmodel_to_onnx_bytes(saved_model_dir: Path, opset: int = 13) -> bytes: """Convert a TF SavedModel directory to ONNX bytes via tf2onnx.""" - import tf2onnx import tensorflow as tf + import tf2onnx with tempfile.TemporaryDirectory() as tmp: onnx_path = Path(tmp) / "model.onnx" @@ -674,9 +674,9 @@ def train_model( hyperparameters: dict[str, Any], split_info: dict[str, Any], num_classes: int = 4, - model_name: Optional[str] = None, - base_model_id: Optional[str] = None, - dataset_id: Optional[str] = None, + model_name: str | None = None, + base_model_id: str | None = None, + dataset_id: str | None = None, ) -> Annotated[bytes, "trained_model"]: """Fine-tune RAMP EfficientNetB0 U-Net; return the best SavedModel as zipped bytes.""" _ = (num_classes, model_name, base_model_id, dataset_id) @@ -707,7 +707,7 @@ def evaluate_model( dataset_labels: str, hyperparameters: dict[str, Any], split_info: dict[str, Any], - class_names: Optional[list[str]] = None, + class_names: list[str] | None = None, ) -> Annotated[dict[str, Any], "metrics"]: """Compute per-pixel building-class metrics on the validation split.""" _ = class_names @@ -800,12 +800,12 @@ def export_onnx(trained_model: Any) -> Annotated[bytes, "onnx_model"]: @step def run_inference( - model_uri: Union[str, Path, Any], + model_uri: str | Path | Any, input_images: str, prediction_path: str, output_dir: str, confidence: float = 0.5, - model_cache_dir: Optional[str] = None, + model_cache_dir: str | None = None, ) -> Annotated[dict[str, Any], "predictions"]: """Native-TF inference over georeferenced chips → building-footprint GeoJSON.""" return infer_ramp_model( @@ -847,12 +847,12 @@ def _safe_load_keras_model(keras_backend, path: str): def infer_ramp_model( - model_uri: Union[str, Path, Any], + model_uri: str | Path | Any, input_path: str, prediction_path: str, output_dir: str, confidence: float = 0.5, - model_cache_dir: Optional[str] = None, + model_cache_dir: str | None = None, ) -> dict[str, Any]: """Run fairpredictor-style TF inference and return the merged GeoJSON content.""" _patch_predictor_savedmodel_loader_for_tf215() @@ -940,7 +940,7 @@ def training_pipeline( def inference_pipeline( model_uri: str, input_images: str, - inference_params: Optional[dict[str, Any]] = None, + inference_params: dict[str, Any] | None = None, output_dir: str = "", chip_size: int = 256, num_classes: int = 4, From 24b7e951bef8b4fe974f7a8893241f75eb876a7d Mon Sep 17 00:00:00 2001 From: Abdelrahman Katkat Date: Tue, 5 May 2026 14:09:30 +0300 Subject: [PATCH 13/30] refactor(pipeline): streamline training directory creation and remove unused cache function; update hyperparameters specification in stac-item.json --- models/ramp/pipeline.py | 20 ++----- models/ramp/stac-item.json | 110 +++++++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 15 deletions(-) diff --git a/models/ramp/pipeline.py b/models/ramp/pipeline.py index 6460d6e9..6c7e8605 100644 --- a/models/ramp/pipeline.py +++ b/models/ramp/pipeline.py @@ -1,6 +1,5 @@ """ZenML pipeline for RAMP (EfficientNetB0 + U-Net) building semantic segmentation.""" -import hashlib import io import json import os @@ -216,11 +215,6 @@ def _materialize_training_input(dataset_chips: str, dataset_labels: str, work_di return input_dir -def _training_cache_dir(dataset_chips: str, dataset_labels: str) -> Path: - cache_key = hashlib.sha256(f"{dataset_chips}|{dataset_labels}".encode()).hexdigest()[:16] - return Path(tempfile.gettempdir()) / f"ramp_training_{cache_key}" - - def preprocess( input_path: str, output_path: str, @@ -336,18 +330,14 @@ def _prepare_training_split( contact_spacing = int(hyperparameters.get("training.contact_spacing", hyperparameters.get("contact_spacing", 8))) seed = int(hyperparameters.get("training.split_seed", hyperparameters.get("split_seed", 42))) - work_dir = _training_cache_dir(dataset_chips, dataset_labels) + work_dir = Path(tempfile.mkdtemp(prefix="ramp_training_")) preprocessed_dir = work_dir / "preprocessed" ramp_train_dir = work_dir / "ramp_training_work" - if force_rebuild and work_dir.exists(): - shutil.rmtree(work_dir) - - if not ramp_train_dir.exists(): - work_dir.mkdir(parents=True, exist_ok=True) - input_dir = _materialize_training_input(dataset_chips, dataset_labels, work_dir) - preprocess(str(input_dir), str(preprocessed_dir), boundary_width, contact_spacing) - split_training_2_validation(str(preprocessed_dir), str(ramp_train_dir), multimasks=True) + work_dir.mkdir(parents=True, exist_ok=True) + input_dir = _materialize_training_input(dataset_chips, dataset_labels, work_dir) + preprocess(str(input_dir), str(preprocessed_dir), boundary_width, contact_spacing) + split_training_2_validation(str(preprocessed_dir), str(ramp_train_dir), multimasks=True) train_count = len(list((ramp_train_dir / "chips").glob("*.tif"))) val_count = len(list((ramp_train_dir / "val-chips").glob("*.tif"))) diff --git a/models/ramp/stac-item.json b/models/ramp/stac-item.json index 60a593f1..16fed4b8 100644 --- a/models/ramp/stac-item.json +++ b/models/ramp/stac-item.json @@ -97,6 +97,116 @@ "seed": 42, "description": "Preprocess chips + labels into 4-class multimasks, then seeded random train/validation split via hot_fair_utilities.split_training_2_validation." }, + "fair:hyperparameters_spec": [ + { + "key": "backbone", + "type": "str", + "default": "efficientnetb0", + "description": "Encoder backbone for U-Net segmentation model" + }, + { + "key": "epochs", + "type": "int", + "default": 20, + "min": 1, + "max": 20, + "description": "Number of training epochs" + }, + { + "key": "batch_size", + "type": "int", + "default": 8, + "min": 1, + "max": 16, + "description": "Samples per training batch" + }, + { + "key": "learning_rate", + "type": "float", + "default": 0.0003, + "min": 1e-7, + "max": 1.0, + "description": "Optimizer learning rate" + }, + { + "key": "val_ratio", + "type": "float", + "default": 0.15, + "min": 0.05, + "max": 0.5, + "description": "Fraction of data held out for validation" + }, + { + "key": "split_seed", + "type": "int", + "default": 42, + "min": 0, + "max": 2147483647, + "description": "Random seed for reproducible train/val split" + }, + { + "key": "boundary_width", + "type": "int", + "default": 3, + "min": 1, + "max": 10, + "description": "Width of building boundary class in multimasks" + }, + { + "key": "contact_spacing", + "type": "int", + "default": 8, + "min": 1, + "max": 20, + "description": "Spacing for contact points between neighbouring buildings" + }, + { + "key": "early_stopping_patience", + "type": "int", + "default": 10, + "min": 1, + "max": 50, + "description": "Epochs to wait before stopping if validation metric does not improve" + }, + { + "key": "augmentation", + "type": "bool", + "default": false, + "description": "Enable data augmentation during training" + }, + { + "key": "imgsz", + "type": "int", + "default": 256, + "min": 128, + "max": 512, + "description": "Input image size (square chips)" + }, + { + "key": "num_classes", + "type": "int", + "default": 4, + "min": 2, + "max": 10, + "description": "Number of output classes (background, building, boundary, contact)" + }, + { + "key": "confidence_threshold", + "type": "float", + "default": 0.5, + "min": 0.0, + "max": 1.0, + "description": "Minimum per-pixel confidence to retain a predicted building class at inference" + }, + { + "key": "min_class_value", + "type": "int", + "default": 1, + "min": 0, + "max": 255, + "description": "Minimum class index value to retain as a foreground prediction at inference" + } + ], "inference_time_per_256x256": { "value": 45, "unit": "ms" From 4a2956cf21326da28f36d39515b0b66988e6eae8 Mon Sep 17 00:00:00 2001 From: Abdelrahman Katkat Date: Tue, 5 May 2026 14:52:40 +0300 Subject: [PATCH 14/30] feat(pipeline): add output materializers for model training and ONNX export steps --- models/ramp/pipeline.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/models/ramp/pipeline.py b/models/ramp/pipeline.py index 6c7e8605..4ce1bf99 100644 --- a/models/ramp/pipeline.py +++ b/models/ramp/pipeline.py @@ -13,6 +13,7 @@ from zenml import log_metadata, pipeline, step +from fair.zenml.materializers import CheckpointBytesMaterializer, ONNXMaterializer from fair.zenml.steps import load_model _DEFAULT_MODEL_CACHE = Path("/workspace/.ramp_model_cache") @@ -656,7 +657,7 @@ def split_dataset( return split_info -@step +@step(output_materializers={"trained_model": CheckpointBytesMaterializer}) def train_model( dataset_chips: str, dataset_labels: str, @@ -776,7 +777,7 @@ def evaluate_model( return metrics_dict -@step +@step(output_materializers={"onnx_model": ONNXMaterializer}) def export_onnx(trained_model: Any) -> Annotated[bytes, "onnx_model"]: """Convert the trained RAMP SavedModel to ONNX bytes and validate.""" import onnx From 13076858cf49caffc005da9d9c53a0e87abdc95e Mon Sep 17 00:00:00 2001 From: Abdelrahman Katkat Date: Wed, 6 May 2026 14:14:42 +0300 Subject: [PATCH 15/30] feat(pipeline): support Keras model formats in model resolution and training functions --- models/ramp/pipeline.py | 225 ++++++++++------------------------------ 1 file changed, 54 insertions(+), 171 deletions(-) diff --git a/models/ramp/pipeline.py b/models/ramp/pipeline.py index 4ce1bf99..7e0d37b2 100644 --- a/models/ramp/pipeline.py +++ b/models/ramp/pipeline.py @@ -1,6 +1,5 @@ """ZenML pipeline for RAMP (EfficientNetB0 + U-Net) building semantic segmentation.""" -import io import json import os import re @@ -8,13 +7,12 @@ import tempfile import zipfile from pathlib import Path -from typing import Annotated, Any, cast +from typing import Annotated, Any from urllib.request import urlretrieve from zenml import log_metadata, pipeline, step from fair.zenml.materializers import CheckpointBytesMaterializer, ONNXMaterializer -from fair.zenml.steps import load_model _DEFAULT_MODEL_CACHE = Path("/workspace/.ramp_model_cache") _QUBVEL_EFFICIENTNET_RELEASE = "https://github.com/qubvel/efficientnet/releases/download/v0.0.1/" @@ -69,8 +67,9 @@ def _extract_zip(zip_path: Path, dest_dir: Path) -> None: def resolve_model_href(model_uri: str, cache_dir: Path | None = None) -> str: - """Resolve only .onnx or .zip model URIs to local paths. + """Resolve .onnx, .keras, or .zip model URIs to local paths. - .onnx -> local .onnx file path + - .keras -> local .keras file path - .zip -> extracted SavedModel directory path (folder containing saved_model.pb) """ if not isinstance(model_uri, str): @@ -94,6 +93,18 @@ def resolve_model_href(model_uri: str, cache_dir: Path | None = None) -> str: if p.is_file(): return str(p) raise FileNotFoundError(f"ONNX model not found: {p}") + # Keras single-file checkpoint + if suffix == ".keras": + if is_http: + dest = cache_dir / (Path(clean_uri).name or "model.keras") + if not dest.is_file(): + urlretrieve(model_uri, dest) + return str(dest) + p = Path(model_uri).resolve() + if p.is_file(): + return str(p) + raise FileNotFoundError(f"Keras model not found: {p}") + # ZIP path -> must contain saved_model.pb somewhere inside if suffix == ".zip": stem = Path(clean_uri).stem or "ramp_model" @@ -117,7 +128,7 @@ def resolve_model_href(model_uri: str, cache_dir: Path | None = None) -> str: for existing in dest_dir.rglob("saved_model.pb"): return str(existing.parent) raise RuntimeError(f"Zip from {model_uri} did not contain a valid SavedModel") - raise ValueError(f"Unsupported model format for {model_uri!r}. Only .onnx and .zip are accepted.") + raise ValueError(f"Unsupported model format for {model_uri!r}. Only .onnx, .keras and .zip are accepted.") def _ensure_ramp_baseline(base_model_weights: str, data_base_path: str | Path) -> Path: @@ -408,12 +419,6 @@ def train_ramp_model( ), ) ) - if not 1 <= epochs <= 200: - raise ValueError(f"Resolved epochs={epochs} is outside [1, 200]") - if not 1 <= batch_size <= 64: - raise ValueError(f"Resolved batch_size={batch_size} is outside [1, 64]") - if not 1 <= patience <= 50: - raise ValueError(f"Resolved early_stopping_patience={patience} is outside [1, 50]") cfg = manage_fine_tuning_config(ramp_train_dir, epochs, batch_size, freeze_layers=False, multimasks=True) cfg["model"]["model_fn_parms"]["backbone"] = backbone @@ -429,56 +434,22 @@ def train_ramp_model( return Path(final_model_path) -def _zip_savedmodel_dir(saved_model_dir: Path) -> bytes: - """Zip a SavedModel directory into bytes for ZenML artifact persistence.""" - buf = io.BytesIO() - with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: - for p in saved_model_dir.rglob("*"): - if p.is_file(): - zf.write(p, arcname=p.relative_to(saved_model_dir)) - return buf.getvalue() - - -def _unzip_savedmodel_bytes(blob: bytes) -> Path: - """Extract a SavedModel zipped as bytes to a temp directory and return its path.""" - dest = Path(tempfile.mkdtemp(prefix="ramp_savedmodel_")) - with zipfile.ZipFile(io.BytesIO(blob), "r") as zf: - zf.extractall(dest) - if (dest / "saved_model.pb").exists(): - return dest - for candidate in dest.rglob("saved_model.pb"): - return candidate.parent - raise RuntimeError("Zipped bytes do not contain a SavedModel (no saved_model.pb found).") - - -def _normalize_to_savedmodel_dir(model_path: str | Path, context: str = "model") -> Path: - """Return a directory containing `saved_model.pb` for a produced/loaded model path.""" - candidate = Path(str(model_path)) - if candidate.is_dir(): - if (candidate / "saved_model.pb").exists(): - return candidate - for nested in candidate.rglob("saved_model.pb"): - return nested.parent - elif candidate.is_file() and candidate.suffix.lower() in {".h5", ".keras"}: - import tensorflow as tf - - model = tf.keras.models.load_model(str(candidate), compile=False) - exported_dir = Path(tempfile.mkdtemp(prefix="ramp_savedmodel_export_")) / "saved_model" - model.save(str(exported_dir), save_format="tf") - if (exported_dir / "saved_model.pb").exists(): - return exported_dir - raise RuntimeError(f"{context} must resolve to a SavedModel directory containing saved_model.pb; got: {candidate}") +def _materialize_keras_bytes(blob: bytes) -> Path: + """Write Keras `.keras` bytes to a temp file and return its path.""" + dest = Path(tempfile.mkdtemp(prefix="ramp_keras_")) / "model.keras" + dest.write_bytes(blob) + return dest def _restore_checkpoint(trained_model: Any) -> Path: - """Restore a trained RAMP SavedModel from bytes, a SavedModel directory, or a .zip file.""" + """Restore a trained RAMP Keras checkpoint as a local `.keras` file path.""" if isinstance(trained_model, bytes): - return _unzip_savedmodel_bytes(trained_model) + return _materialize_keras_bytes(trained_model) if isinstance(trained_model, (str, Path)): p = Path(str(trained_model)) - if p.is_file() and p.suffix.lower() == ".zip": - return _unzip_savedmodel_bytes(p.read_bytes()) - return _normalize_to_savedmodel_dir(p, context="trained_model") + if p.is_file() and p.suffix.lower() == ".keras": + return p + return Path(resolve_model_href(str(p))) raise TypeError(f"Cannot restore RAMP checkpoint from {type(trained_model).__name__}") @@ -669,7 +640,7 @@ def train_model( base_model_id: str | None = None, dataset_id: str | None = None, ) -> Annotated[bytes, "trained_model"]: - """Fine-tune RAMP EfficientNetB0 U-Net; return the best SavedModel as zipped bytes.""" + """Fine-tune RAMP EfficientNetB0 U-Net; return the best model as `.keras` bytes.""" _ = (num_classes, model_name, base_model_id, dataset_id) ramp_train_dir = Path(split_info["_ramp_train_dir"]) @@ -684,10 +655,13 @@ def train_model( hyperparameters=hyperparameters, data_base_path=work_dir, ) - saved_model_dir = _normalize_to_savedmodel_dir(final_model_path, context="train_model output") + import tensorflow as tf - blob = _zip_savedmodel_dir(saved_model_dir) - log_metadata(metadata={"saved_model_dir": str(saved_model_dir), "checkpoint_bytes": len(blob)}) + model = tf.keras.models.load_model(str(final_model_path), compile=False) + exported_path = Path(tempfile.mkdtemp(prefix="ramp_keras_export_")) / "model.keras" + model.save(str(exported_path)) + blob = exported_path.read_bytes() + log_metadata(metadata={"keras_path": str(exported_path), "checkpoint_bytes": len(blob)}) return blob @@ -719,7 +693,7 @@ def evaluate_model( if mask.is_file(): pairs.append((chip, mask)) - saved_model_dir = _restore_checkpoint(trained_model) + restored = _restore_checkpoint(trained_model) if not pairs: # No val data to evaluate against (e.g. CI mocks); return zeroed metrics with the @@ -735,7 +709,7 @@ def evaluate_model( import tensorflow as tf - model = tf.keras.models.load_model(str(saved_model_dir), compile=False) + model = tf.keras.models.load_model(str(restored), compile=False) tp = fp = fn = 0 correct = total = 0 @@ -779,11 +753,17 @@ def evaluate_model( @step(output_materializers={"onnx_model": ONNXMaterializer}) def export_onnx(trained_model: Any) -> Annotated[bytes, "onnx_model"]: - """Convert the trained RAMP SavedModel to ONNX bytes and validate.""" + """Convert the trained RAMP Keras checkpoint to ONNX bytes and validate.""" import onnx + import tensorflow as tf + import tf2onnx - saved_model_dir = _restore_checkpoint(trained_model) - onnx_bytes = _convert_savedmodel_to_onnx_bytes(saved_model_dir) + restored = _restore_checkpoint(trained_model) + model = tf.keras.models.load_model(str(restored), compile=False) + with tempfile.TemporaryDirectory() as tmp: + onnx_path = Path(tmp) / "model.onnx" + tf2onnx.convert.from_keras(model, opset=13, output_path=str(onnx_path)) + onnx_bytes = onnx_path.read_bytes() onnx.checker.check_model(onnx.load_from_string(onnx_bytes)) log_metadata(metadata={"onnx_bytes": len(onnx_bytes)}) return onnx_bytes @@ -791,91 +771,15 @@ def export_onnx(trained_model: Any) -> Annotated[bytes, "onnx_model"]: @step def run_inference( - model_uri: str | Path | Any, + model_uri: str, input_images: str, - prediction_path: str, - output_dir: str, - confidence: float = 0.5, - model_cache_dir: str | None = None, + inference_params: dict[str, Any], ) -> Annotated[dict[str, Any], "predictions"]: - """Native-TF inference over georeferenced chips → building-footprint GeoJSON.""" - return infer_ramp_model( - model_uri=model_uri, - input_path=input_images, - prediction_path=prediction_path, - output_dir=output_dir, - confidence=confidence, - model_cache_dir=model_cache_dir, - ) - + """RAMP batch inference using the promoted ONNX `assets.model`.""" + from fair.serve.base import load_session -def _patch_predictor_savedmodel_loader_for_tf215() -> None: - """Patch fairpredictor's SavedModel directory loader for TF 2.15 compatibility. - - ``TFSMLayer`` is absent in TF 2.15's ``tf.keras.layers``; fall back to ``load_model``. - """ - import importlib - - import tensorflow as tf - - pred = importlib.import_module("predictor.prediction") - pred_mod = cast(Any, pred) - if getattr(pred_mod, "_fair_models_tf215_savedmodel_patch_applied", False): - return - - original_loader = getattr(pred_mod, "_load_keras_model", None) - if original_loader is None: - raise RuntimeError("predictor.prediction._load_keras_model not found; fairpredictor API changed.") - - def _safe_load_keras_model(keras_backend, path: str): - if os.path.isdir(path) and (Path(path) / "saved_model.pb").exists(): - return tf.keras.models.load_model(path, compile=False) - return original_loader(keras_backend, path) - - _safe_load_keras_model._fair_models_tf215_savedmodel_loader = True # type: ignore[attr-defined] - pred_mod._load_keras_model = _safe_load_keras_model - pred_mod._fair_models_tf215_savedmodel_patch_applied = True - - -def infer_ramp_model( - model_uri: str | Path | Any, - input_path: str, - prediction_path: str, - output_dir: str, - confidence: float = 0.5, - model_cache_dir: str | None = None, -) -> dict[str, Any]: - """Run fairpredictor-style TF inference and return the merged GeoJSON content.""" - _patch_predictor_savedmodel_loader_for_tf215() - from predictor.prediction import run_prediction - - cache = Path(model_cache_dir) if model_cache_dir else None - if isinstance(model_uri, bytes): - model_dir = str(_unzip_savedmodel_bytes(model_uri)) - elif isinstance(model_uri, (str, Path)): - model_dir = resolve_model_href(str(model_uri), cache_dir=cache) - else: - raise TypeError("model_uri must be a str, Path, or zipped SavedModel bytes.") - - input_dir = _resolve_input_directory(input_path, "input_path") - out_dir = _to_local_path(prediction_path, "prediction_path") - out_dir.mkdir(parents=True, exist_ok=True) - - if not any(input_dir.glob("**/*.tif")): - raise RuntimeError( - f"No GeoTIFF chips (*.tif) found in {input_dir}. RAMP inference expects georeferenced chips." - ) - - georef_dir = run_prediction( - checkpoint_path=model_dir, - input_path=str(input_dir), - prediction_path=str(out_dir), - confidence=confidence, - crs="3857", - ) - final_output_dir = Path(output_dir or tempfile.mkdtemp(prefix="ramp_postprocess_")) - final_output_dir.mkdir(parents=True, exist_ok=True) - return postprocess(str(georef_dir), str(final_output_dir)) + session = load_session(model_uri) + return predict(session, input_images, inference_params) @step @@ -931,28 +835,7 @@ def training_pipeline( def inference_pipeline( model_uri: str, input_images: str, - inference_params: dict[str, Any] | None = None, - output_dir: str = "", - chip_size: int = 256, - num_classes: int = 4, - confidence: float = 0.5, - zenml_artifact_version_id: str = "", - prediction_path: str = "", -) -> dict[str, Any]: - """RAMP inference pipeline: load model → predict → postprocess → FeatureCollection.""" - _ = (chip_size, num_classes) - resolved_output_dir = output_dir or str(Path(tempfile.mkdtemp(prefix="ramp_inference_"))) - resolved_confidence = float((inference_params or {}).get("confidence_threshold", confidence)) - prediction_dir = prediction_path or str(Path(resolved_output_dir) / "predictions") - model = ( - load_model(model_uri=model_uri, zenml_artifact_version_id=zenml_artifact_version_id) - if zenml_artifact_version_id - else model_uri - ) - return run_inference( - model_uri=model, - input_images=input_images, - prediction_path=prediction_dir, - output_dir=resolved_output_dir, - confidence=resolved_confidence, - ) + inference_params: dict[str, Any], +) -> None: + """RAMP inference pipeline: load ONNX session → predict → FeatureCollection.""" + run_inference(model_uri=model_uri, input_images=input_images, inference_params=inference_params) From 4cebc65f4f1eb9bc6858424480191653a9a302b2 Mon Sep 17 00:00:00 2001 From: Abdelrahman Katkat Date: Wed, 6 May 2026 15:18:25 +0300 Subject: [PATCH 16/30] feat(tests): enhance model training tests to support Keras model serialization and update assertions --- models/ramp/tests/test_steps.py | 35 ++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/models/ramp/tests/test_steps.py b/models/ramp/tests/test_steps.py index 25e21bca..4d210cff 100644 --- a/models/ramp/tests/test_steps.py +++ b/models/ramp/tests/test_steps.py @@ -18,6 +18,14 @@ def _noop_mlflow_ctx(*_args: Any, **_kwargs: Any): yield +class _FakeKerasModel: + def __init__(self, payload: bytes): + self._payload = payload + + def save(self, path: str) -> None: + Path(path).write_bytes(self._payload) + + def test_split_dataset(toy_chips: Path, toy_labels: Path, base_hyperparameters: dict[str, Any]) -> None: from models.ramp.pipeline import split_dataset @@ -52,16 +60,14 @@ def test_split_dataset(toy_chips: Path, toy_labels: Path, base_hyperparameters: def test_train_model(toy_chips: Path, toy_labels: Path, base_hyperparameters: dict[str, Any], tmp_path: Path) -> None: + import tensorflow as tf + from models.ramp.pipeline import train_model ramp_train_dir = tmp_path / "ramp_training_work" (ramp_train_dir / "chips").mkdir(parents=True) (ramp_train_dir / "val-chips").mkdir(parents=True) - fake_saved_model_dir = tmp_path / "saved_model" - fake_saved_model_dir.mkdir() - (fake_saved_model_dir / "saved_model.pb").write_bytes(b"\x08\x01") # magic-enough stub - split_info = { "_work_dir": str(tmp_path), "_preprocessed_dir": str(tmp_path / "preprocessed"), @@ -76,10 +82,14 @@ def test_train_model(toy_chips: Path, toy_labels: Path, base_hyperparameters: di hyperparameters = dict(base_hyperparameters) hyperparameters.update({"epochs": 1, "batch_size": 1}) + expected = b"fake-keras-bytes" + fake_model_path = tmp_path / "best_model.keras" + fake_model_path.write_bytes(b"stub") + with ( patch("models.ramp.pipeline.mlflow_training_context", _noop_mlflow_ctx, create=True), - patch("models.ramp.pipeline.train_ramp_model", return_value=fake_saved_model_dir), - patch("models.ramp.pipeline._zip_savedmodel_dir", return_value=b"fake-savedmodel-zip"), + patch("models.ramp.pipeline.train_ramp_model", return_value=fake_model_path), + patch.object(tf.keras.models, "load_model", return_value=_FakeKerasModel(expected)), patch("models.ramp.pipeline.log_metadata"), ): model_bytes = train_model.entrypoint( @@ -92,7 +102,7 @@ def test_train_model(toy_chips: Path, toy_labels: Path, base_hyperparameters: di ) assert isinstance(model_bytes, bytes) - assert model_bytes == b"fake-savedmodel-zip" + assert model_bytes == expected def test_evaluate_model( @@ -142,6 +152,7 @@ def test_evaluate_model( def test_export_onnx(tmp_path: Path) -> None: import onnx + import tensorflow as tf from onnx import TensorProto, helper from models.ramp.pipeline import export_onnx @@ -154,13 +165,13 @@ def test_export_onnx(tmp_path: Path) -> None: toy_model = helper.make_model(graph, producer_name="test") toy_bytes = toy_model.SerializeToString() - fake_saved_model_dir = tmp_path / "saved_model" - fake_saved_model_dir.mkdir() - (fake_saved_model_dir / "saved_model.pb").write_bytes(b"\x08\x01") + fake_keras_path = tmp_path / "model.keras" + fake_keras_path.write_bytes(b"stub") with ( - patch("models.ramp.pipeline._restore_checkpoint", return_value=fake_saved_model_dir), - patch("models.ramp.pipeline._convert_savedmodel_to_onnx_bytes", return_value=toy_bytes), + patch("models.ramp.pipeline._restore_checkpoint", return_value=fake_keras_path), + patch.object(tf.keras.models, "load_model", return_value=_FakeKerasModel(b"ignored")), + patch("tf2onnx.convert.from_keras", side_effect=lambda _m, opset, output_path: Path(output_path).write_bytes(toy_bytes)), patch("models.ramp.pipeline.log_metadata"), ): exported = export_onnx.entrypoint(trained_model=b"fake") From 1a8f7065db6ef497c353d75308eeece974137a04 Mon Sep 17 00:00:00 2001 From: Abdelrahman Katkat Date: Mon, 11 May 2026 10:13:19 +0300 Subject: [PATCH 17/30] refactor(ramp): update Dockerfile for inference stages and modify entrypoint in stac-item.json --- models/ramp/Dockerfile | 39 ++++++++++++++++++++++++--------- models/ramp/stac-item.json | 4 ++-- models/ramp/tests/test_steps.py | 5 ++++- 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/models/ramp/Dockerfile b/models/ramp/Dockerfile index 338fd42b..fa507c35 100644 --- a/models/ramp/Dockerfile +++ b/models/ramp/Dockerfile @@ -70,17 +70,36 @@ RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --python /app/.venv/bin/python /tmp/fair-src[test] # --------------------------------------------------------------------------- -# Inference stage: runtime + serving deps for the smoke test / live API +# Inference builder stage: distroless-ready venv with serving deps # --------------------------------------------------------------------------- -FROM runtime AS inference - -COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv -COPY --from=builder /tmp/fair-src /tmp/fair-src - +FROM ghcr.io/astral-sh/uv:bookworm-slim AS inference-builder +ENV UV_PYTHON_INSTALL_DIR=/python UV_PYTHON_PREFERENCE=only-managed +RUN uv python install 3.13 +WORKDIR /app +COPY pyproject.toml README.md fair_zenml_patch.pth /tmp/fair-src/ +COPY fair /tmp/fair-src/fair RUN --mount=type=cache,target=/root/.cache/uv \ - SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 \ - uv pip install --python /app/.venv/bin/python /tmp/fair-src[serve] + export SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 && \ + uv venv && uv pip install \ + /tmp/fair-src[serve] \ + rasterio \ + pyproj \ + numpy \ + Pillow \ + onnx \ + onnxruntime +COPY models/ramp/pipeline.py /app/models/ramp/pipeline.py -ENV PYTHONPATH=/workspace +# --------------------------------------------------------------------------- +# Inference stage: distroless runtime for live serving +# --------------------------------------------------------------------------- +FROM gcr.io/distroless/python3-debian12:nonroot AS inference +WORKDIR /app +COPY --from=inference-builder /python /python +COPY --from=inference-builder /app/.venv /app/.venv +COPY --from=inference-builder /app/models /app/models +ENV PATH="/app/.venv/bin:$PATH" PYTHONPATH=/app \ + ZENML_CONFIG_PATH=/tmp/zenml EXPOSE 8080 -CMD ["/app/.venv/bin/uvicorn", "fair.serve.base:create_app", "--factory", "--host", "0.0.0.0", "--port", "8080"] +ENTRYPOINT ["/app/.venv/bin/python", "-m", "uvicorn"] +CMD ["fair.serve.base:create_app", "--factory", "--host", "0.0.0.0", "--port", "8080"] diff --git a/models/ramp/stac-item.json b/models/ramp/stac-item.json index 16fed4b8..62830ff8 100644 --- a/models/ramp/stac-item.json +++ b/models/ramp/stac-item.json @@ -330,7 +330,7 @@ "roles": [ "code" ], - "mlm:entrypoint": "models.ramp.pipeline:training_pipeline" + "mlm:entrypoint": "models.ramp.pipeline:predict" }, "mlm:training": { "href": "ghcr.io/hotosm/fair-models/ramp:v1", @@ -341,7 +341,7 @@ ] }, "mlm:inference": { - "href": "ghcr.io/hotosm/fair-models/ramp:v1", + "href": "ghcr.io/hotosm/fair-models/ramp:latest-inference", "type": "application/vnd.oci.image.index.v1+json", "title": "Inference Docker Image", "roles": [ diff --git a/models/ramp/tests/test_steps.py b/models/ramp/tests/test_steps.py index 4d210cff..e9122e69 100644 --- a/models/ramp/tests/test_steps.py +++ b/models/ramp/tests/test_steps.py @@ -171,7 +171,10 @@ def test_export_onnx(tmp_path: Path) -> None: with ( patch("models.ramp.pipeline._restore_checkpoint", return_value=fake_keras_path), patch.object(tf.keras.models, "load_model", return_value=_FakeKerasModel(b"ignored")), - patch("tf2onnx.convert.from_keras", side_effect=lambda _m, opset, output_path: Path(output_path).write_bytes(toy_bytes)), + patch( + "tf2onnx.convert.from_keras", + side_effect=lambda _m, opset, output_path: Path(output_path).write_bytes(toy_bytes), + ), patch("models.ramp.pipeline.log_metadata"), ): exported = export_onnx.entrypoint(trained_model=b"fake") From 1c7afe5acc0e287d9357895e82e76124dd4a3bd5 Mon Sep 17 00:00:00 2001 From: Abdelrahman Katkat Date: Mon, 11 May 2026 12:23:46 +0300 Subject: [PATCH 18/30] feat(pipeline): enhance postprocessing function to validate CRS and streamline GeoJSON conversion --- models/ramp/pipeline.py | 58 +++++++---------------------------------- 1 file changed, 10 insertions(+), 48 deletions(-) diff --git a/models/ramp/pipeline.py b/models/ramp/pipeline.py index 7e0d37b2..c7343961 100644 --- a/models/ramp/pipeline.py +++ b/models/ramp/pipeline.py @@ -258,6 +258,7 @@ def preprocess( def postprocess(prediction_masks_dir: str, output_dir: str) -> dict[str, Any]: """Merge prediction TIFF tiles into a building-footprint GeoJSON (EPSG:4326).""" + import rasterio from geomltoolkits.geometry.validate import validate_polygon_geometries from geomltoolkits.raster.merge import merge_rasters from geomltoolkits.raster.morphology import morphological_cleaning @@ -276,6 +277,11 @@ def postprocess(prediction_masks_dir: str, output_dir: str) -> dict[str, Any]: merge_rasters(str(pred_dir), str(merged_mask_path)) morphological_cleaning(str(merged_mask_path)) + + with rasterio.open(merged_mask_path) as src: + if src.crs is None: + raise ValueError(f"Merged prediction raster is missing CRS: {merged_mask_path}") + gdf = vectorize_mask( input_tiff=str(merged_mask_path), output_geojson=str(merged_geojson_path), @@ -292,21 +298,6 @@ def postprocess(prediction_masks_dir: str, output_dir: str) -> dict[str, Any]: merged_geojson_path.write_text(json.dumps(geojson_dict), encoding="utf-8") return geojson_dict - if gdf.crs and gdf.crs != "EPSG:4326": - gdf = gdf.to_crs("EPSG:4326") - elif not gdf.crs: - gdf.set_crs("EPSG:3857", inplace=True) - gdf = gdf.to_crs("EPSG:4326") - - import pandas as pd - - for col in gdf.columns: - if col == "geometry": - continue - if pd.api.types.is_extension_array_dtype(gdf[col].dtype): - gdf[col] = gdf[col].astype(object).where(gdf[col].notna(), None) - gdf.to_file(merged_geojson_path, driver="GeoJSON") - validated_geojson = validate_polygon_geometries( geojson_dict, output_path=str(merged_geojson_path), @@ -330,17 +321,12 @@ def _prepare_training_split( """Preprocess + split chips/labels into RAMP train/val layout; return split_info.""" from hot_fair_utilities.training.ramp.prepare_data import split_training_2_validation - val_fraction = float( - hyperparameters.get( - "training.val_ratio", - hyperparameters.get("val_fraction", hyperparameters.get("val_ratio", 0.15)), - ) - ) + val_fraction = float(hyperparameters.get("training.val_ratio", 0.15)) if not 0.0 < val_fraction < 1.0: raise ValueError("val_fraction must be in (0.0, 1.0)") - boundary_width = int(hyperparameters.get("training.boundary_width", hyperparameters.get("boundary_width", 3))) - contact_spacing = int(hyperparameters.get("training.contact_spacing", hyperparameters.get("contact_spacing", 8))) - seed = int(hyperparameters.get("training.split_seed", hyperparameters.get("split_seed", 42))) + boundary_width = int(hyperparameters.get("training.boundary_width", 3)) + contact_spacing = int(hyperparameters.get("training.contact_spacing", 8)) + seed = int(hyperparameters.get("training.split_seed", 42)) work_dir = Path(tempfile.mkdtemp(prefix="ramp_training_")) preprocessed_dir = work_dir / "preprocessed" @@ -453,30 +439,6 @@ def _restore_checkpoint(trained_model: Any) -> Path: raise TypeError(f"Cannot restore RAMP checkpoint from {type(trained_model).__name__}") -def _convert_savedmodel_to_onnx_bytes(saved_model_dir: Path, opset: int = 13) -> bytes: - """Convert a TF SavedModel directory to ONNX bytes via tf2onnx.""" - import tensorflow as tf - import tf2onnx - - with tempfile.TemporaryDirectory() as tmp: - onnx_path = Path(tmp) / "model.onnx" - from_saved_model = getattr(tf2onnx.convert, "from_saved_model", None) - if callable(from_saved_model): - from_saved_model( - str(saved_model_dir), - output_path=str(onnx_path), - opset=opset, - ) - else: - model = tf.keras.models.load_model(str(saved_model_dir), compile=False) - tf2onnx.convert.from_keras( - model, - opset=opset, - output_path=str(onnx_path), - ) - return onnx_path.read_bytes() - - def _build_feature_collection(features: list[dict[str, Any]]) -> dict[str, Any]: return {"type": "FeatureCollection", "features": features} From 73137db9a46a7c42c00a6dcef3f7506ffd39a816 Mon Sep 17 00:00:00 2001 From: Abdelrahman Katkat Date: Mon, 11 May 2026 13:06:30 +0300 Subject: [PATCH 19/30] refactor(ramp): update Dockerfile to use GPU base image and enhance test steps for model training --- models/ramp/Dockerfile | 10 +- models/ramp/tests/test_steps.py | 258 ++++++++++++++++---------------- 2 files changed, 136 insertions(+), 132 deletions(-) diff --git a/models/ramp/Dockerfile b/models/ramp/Dockerfile index fa507c35..5d9c2fb2 100644 --- a/models/ramp/Dockerfile +++ b/models/ramp/Dockerfile @@ -1,11 +1,13 @@ # syntax=docker/dockerfile:1.7 -# Base image: ghcr.io/hotosm/fair-utilities-ramp:cpu-latest (or :gpu-latest via BASE_IMAGE build arg). +# Base image: ghcr.io/hotosm/fair-utilities-ramp:gpu-latest (override BASE_IMAGE for CPU builds). # Build from the fAIr-models repo root: # docker build -f models/ramp/Dockerfile --target test -t ramp:test . +# docker build -f models/ramp/Dockerfile --build-arg BASE_IMAGE=ghcr.io/hotosm/fair-utilities-ramp:gpu-latest --target test -t ramp:test . +# docker build -f models/ramp/Dockerfile --build-arg BASE_IMAGE=ghcr.io/hotosm/fair-utilities-ramp:cpu-latest --target test -t ramp:test . # docker build -f models/ramp/Dockerfile --target runtime -t ramp:runtime . # docker build -f models/ramp/Dockerfile --target inference -t ramp:inference . -ARG BASE_IMAGE=ghcr.io/hotosm/fair-utilities-ramp:cpu-latest +ARG BASE_IMAGE=ghcr.io/hotosm/fair-utilities-ramp:gpu-latest # --------------------------------------------------------------------------- # Builder stage: install model-pack deps into the base image venv @@ -47,7 +49,9 @@ ENV PATH="/app/.venv/bin:$PATH" \ PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ RAMP_HOME=/app \ - SM_FRAMEWORK=tf.keras + SM_FRAMEWORK=tf.keras \ + NVIDIA_VISIBLE_DEVICES=all \ + NVIDIA_DRIVER_CAPABILITIES=compute,utility COPY --from=builder /app/.venv /app/.venv diff --git a/models/ramp/tests/test_steps.py b/models/ramp/tests/test_steps.py index e9122e69..a0e0f066 100644 --- a/models/ramp/tests/test_steps.py +++ b/models/ramp/tests/test_steps.py @@ -1,29 +1,27 @@ -"""Step tests for the RAMP pipeline. +"""Step tests for RAMP building segmentation. -Each test calls ``step.entrypoint(...)`` directly. Heavy RAMP/TF operations -(``train_ramp_model``, SavedModel loading, tf2onnx conversion) are patched so -tests run quickly and do not require GPU resources. +Each test runs real @step entrypoints against the toy OAM chips/labels fixture. +No pipeline-internal mocks; telemetry sinks are already no-ops via +models/conftest.py::mock_instrumentation. """ from __future__ import annotations -from contextlib import contextmanager from pathlib import Path from typing import Any -from unittest.mock import patch +import pytest -@contextmanager -def _noop_mlflow_ctx(*_args: Any, **_kwargs: Any): - yield +_PRETRAINED_URL = "https://huggingface.co/hotosm/ramp/resolve/74daea54694f2e4924f1222520c614c7f5c029fe/v1-baseline.zip" -class _FakeKerasModel: - def __init__(self, payload: bytes): - self._payload = payload +@pytest.fixture(scope="session") +def pretrained_weights(tmp_path_factory: pytest.TempPathFactory) -> str: + from upath import UPath - def save(self, path: str) -> None: - Path(path).write_bytes(self._payload) + cache = tmp_path_factory.mktemp("ramp_weights") / "baseline.zip" + cache.write_bytes(UPath(_PRETRAINED_URL).read_bytes()) + return str(cache) def test_split_dataset(toy_chips: Path, toy_labels: Path, base_hyperparameters: dict[str, Any]) -> None: @@ -34,19 +32,18 @@ def test_split_dataset(toy_chips: Path, toy_labels: Path, base_hyperparameters: { "epochs": 1, "batch_size": 1, - "val_fraction": 0.25, - "split_seed": 42, - "boundary_width": 1, - "contact_spacing": 2, + "training.val_ratio": 0.25, + "training.split_seed": 42, + "training.boundary_width": 1, + "training.contact_spacing": 2, } ) - with patch("models.ramp.pipeline.log_metadata"): - result = split_dataset.entrypoint( - dataset_chips=str(toy_chips), - dataset_labels=str(toy_labels), - hyperparameters=hyperparameters, - ) + result = split_dataset.entrypoint( + dataset_chips=str(toy_chips), + dataset_labels=str(toy_labels), + hyperparameters=hyperparameters, + ) assert result["strategy"] == "random" assert result["train_count"] > 0 @@ -59,125 +56,128 @@ def test_split_dataset(toy_chips: Path, toy_labels: Path, base_hyperparameters: assert (ramp_dir / "val-chips").is_dir() -def test_train_model(toy_chips: Path, toy_labels: Path, base_hyperparameters: dict[str, Any], tmp_path: Path) -> None: - import tensorflow as tf - - from models.ramp.pipeline import train_model - - ramp_train_dir = tmp_path / "ramp_training_work" - (ramp_train_dir / "chips").mkdir(parents=True) - (ramp_train_dir / "val-chips").mkdir(parents=True) +def test_train_model( + toy_chips: Path, + toy_labels: Path, + base_hyperparameters: dict[str, Any], + pretrained_weights: str, +) -> None: + from models.ramp.pipeline import split_dataset, train_model - split_info = { - "_work_dir": str(tmp_path), - "_preprocessed_dir": str(tmp_path / "preprocessed"), - "_ramp_train_dir": str(ramp_train_dir), - "strategy": "random", - "val_ratio": 0.25, - "seed": 42, - "train_count": 3, - "val_count": 1, - "description": "test split", - } hyperparameters = dict(base_hyperparameters) - hyperparameters.update({"epochs": 1, "batch_size": 1}) - - expected = b"fake-keras-bytes" - fake_model_path = tmp_path / "best_model.keras" - fake_model_path.write_bytes(b"stub") - - with ( - patch("models.ramp.pipeline.mlflow_training_context", _noop_mlflow_ctx, create=True), - patch("models.ramp.pipeline.train_ramp_model", return_value=fake_model_path), - patch.object(tf.keras.models, "load_model", return_value=_FakeKerasModel(expected)), - patch("models.ramp.pipeline.log_metadata"), - ): - model_bytes = train_model.entrypoint( - dataset_chips=str(toy_chips), - dataset_labels=str(toy_labels), - base_model_weights="https://example.com/baseline.zip", - hyperparameters=hyperparameters, - split_info=split_info, - num_classes=4, - ) + hyperparameters.update( + { + "epochs": 1, + "batch_size": 1, + "training.val_ratio": 0.25, + "training.split_seed": 42, + "training.boundary_width": 1, + "training.contact_spacing": 2, + } + ) + + split_info = split_dataset.entrypoint( + dataset_chips=str(toy_chips), + dataset_labels=str(toy_labels), + hyperparameters=hyperparameters, + ) + model_bytes = train_model.entrypoint( + dataset_chips=str(toy_chips), + dataset_labels=str(toy_labels), + base_model_weights=pretrained_weights, + hyperparameters=hyperparameters, + split_info=split_info, + num_classes=4, + ) assert isinstance(model_bytes, bytes) - assert model_bytes == expected + assert len(model_bytes) > 0 def test_evaluate_model( - toy_chips: Path, toy_labels: Path, base_hyperparameters: dict[str, Any], tmp_path: Path + toy_chips: Path, + toy_labels: Path, + base_hyperparameters: dict[str, Any], + pretrained_weights: str, ) -> None: - from models.ramp.pipeline import evaluate_model - - ramp_train_dir = tmp_path / "ramp_training_work" - (ramp_train_dir / "chips").mkdir(parents=True) - (ramp_train_dir / "val-chips").mkdir(parents=True) - (ramp_train_dir / "val-multimasks").mkdir(parents=True) - - split_info = { - "_work_dir": str(tmp_path), - "_preprocessed_dir": str(tmp_path / "preprocessed"), - "_ramp_train_dir": str(ramp_train_dir), - "strategy": "random", - "val_ratio": 0.25, - "seed": 42, - "train_count": 3, - "val_count": 1, - "description": "test split", - } - - fake_saved_model_dir = tmp_path / "saved_model" - fake_saved_model_dir.mkdir() - (fake_saved_model_dir / "saved_model.pb").write_bytes(b"\x08\x01") - - with ( - patch("models.ramp.pipeline.mlflow_training_context", _noop_mlflow_ctx, create=True), - patch("models.ramp.pipeline._restore_checkpoint", return_value=fake_saved_model_dir), - patch("models.ramp.pipeline.log_metadata"), - ): - metrics = evaluate_model.entrypoint( - trained_model=b"fake", - dataset_chips=str(toy_chips), - dataset_labels=str(toy_labels), - hyperparameters=base_hyperparameters, - split_info=split_info, - ) + from models.ramp.pipeline import evaluate_model, split_dataset, train_model + + hyperparameters = dict(base_hyperparameters) + hyperparameters.update( + { + "epochs": 1, + "batch_size": 1, + "training.val_ratio": 0.25, + "training.split_seed": 42, + "training.boundary_width": 1, + "training.contact_spacing": 2, + } + ) + + split_info = split_dataset.entrypoint( + dataset_chips=str(toy_chips), + dataset_labels=str(toy_labels), + hyperparameters=hyperparameters, + ) + model_bytes = train_model.entrypoint( + dataset_chips=str(toy_chips), + dataset_labels=str(toy_labels), + base_model_weights=pretrained_weights, + hyperparameters=hyperparameters, + split_info=split_info, + num_classes=4, + ) + metrics = evaluate_model.entrypoint( + trained_model=model_bytes, + dataset_chips=str(toy_chips), + dataset_labels=str(toy_labels), + hyperparameters=hyperparameters, + split_info=split_info, + ) expected = {"fair:accuracy", "fair:mean_iou", "fair:precision", "fair:recall"} assert set(metrics.keys()) == expected - for key in expected: - assert isinstance(metrics[key], float) + for value in metrics.values(): + assert isinstance(value, float) + assert 0.0 <= value <= 1.0 -def test_export_onnx(tmp_path: Path) -> None: +def test_export_onnx( + toy_chips: Path, + toy_labels: Path, + base_hyperparameters: dict[str, Any], + pretrained_weights: str, +) -> None: import onnx - import tensorflow as tf - from onnx import TensorProto, helper - - from models.ramp.pipeline import export_onnx - - # Build a toy ONNX model and capture its bytes. - x = helper.make_tensor_value_info("input", TensorProto.FLOAT, [1, 1]) - y = helper.make_tensor_value_info("output", TensorProto.FLOAT, [1, 1]) - node = helper.make_node("Identity", ["input"], ["output"]) - graph = helper.make_graph([node], "toy", [x], [y]) - toy_model = helper.make_model(graph, producer_name="test") - toy_bytes = toy_model.SerializeToString() - - fake_keras_path = tmp_path / "model.keras" - fake_keras_path.write_bytes(b"stub") - - with ( - patch("models.ramp.pipeline._restore_checkpoint", return_value=fake_keras_path), - patch.object(tf.keras.models, "load_model", return_value=_FakeKerasModel(b"ignored")), - patch( - "tf2onnx.convert.from_keras", - side_effect=lambda _m, opset, output_path: Path(output_path).write_bytes(toy_bytes), - ), - patch("models.ramp.pipeline.log_metadata"), - ): - exported = export_onnx.entrypoint(trained_model=b"fake") + + from models.ramp.pipeline import export_onnx, split_dataset, train_model + + hyperparameters = dict(base_hyperparameters) + hyperparameters.update( + { + "epochs": 1, + "batch_size": 1, + "training.val_ratio": 0.25, + "training.split_seed": 42, + "training.boundary_width": 1, + "training.contact_spacing": 2, + } + ) + + split_info = split_dataset.entrypoint( + dataset_chips=str(toy_chips), + dataset_labels=str(toy_labels), + hyperparameters=hyperparameters, + ) + model_bytes = train_model.entrypoint( + dataset_chips=str(toy_chips), + dataset_labels=str(toy_labels), + base_model_weights=pretrained_weights, + hyperparameters=hyperparameters, + split_info=split_info, + num_classes=4, + ) + exported = export_onnx.entrypoint(trained_model=model_bytes) assert isinstance(exported, bytes) loaded = onnx.load_from_string(exported) From 491a4759624ba18e35048b92da21b0596f55d45c Mon Sep 17 00:00:00 2001 From: Abdelrahman Katkat Date: Mon, 18 May 2026 13:27:25 +0300 Subject: [PATCH 20/30] feat(pipeline): implement label resolution for GeoJSON/JSON files Added a new function `_resolve_labels_geojson` to handle the resolution of dataset labels, supporting both direct file paths and directories containing a single labels file. Updated `_materialize_training_input` to utilize the new label resolution function, enhancing input handling for training datasets. --- models/ramp/pipeline.py | 70 +++++++++++++++++++++++++++++++++++------ 1 file changed, 61 insertions(+), 9 deletions(-) diff --git a/models/ramp/pipeline.py b/models/ramp/pipeline.py index c7343961..9ea920b2 100644 --- a/models/ramp/pipeline.py +++ b/models/ramp/pipeline.py @@ -40,13 +40,67 @@ def _resolve_input_directory(path_value: str, purpose: str) -> Path: return _to_local_path(path_value, purpose) -def _resolve_input_file(path_value: str, purpose: str) -> Path: - """Resolve local/remote file paths to a local path.""" - from fair.utils.data import resolve_path +def _resolve_labels_geojson(dataset_labels: str) -> Path: + """Resolve labels input to exactly one GeoJSON/JSON file. - if "://" in str(path_value): - return resolve_path(path_value) - return _to_local_path(path_value, purpose) + Supports: + - direct file path/URI + - directory/prefix containing exactly one labels file + """ + from fair.utils.data import resolve_directory, resolve_path + + label_patterns = ("*.geojson", "*.json") + labels_value = str(dataset_labels) + + # First attempt: resolve as a file path/URI. + try: + file_candidate = resolve_path(labels_value) + except Exception: # pragma: no cover - fallback to directory/prefix resolution + file_candidate = None + + if file_candidate and file_candidate.is_file(): + if file_candidate.suffix.lower() not in (".geojson", ".json"): + raise ValueError(f"dataset_labels must point to a .geojson or .json file, got: {file_candidate}") + return file_candidate + + def _pick_single_candidate(search_dir: Path) -> Path: + for pattern in label_patterns: + matches = sorted(p for p in search_dir.glob(pattern) if p.is_file()) + if not matches: + continue + if len(matches) > 1: + listed = ", ".join(str(p) for p in matches) + raise ValueError( + f"dataset_labels must resolve to exactly one labels file, found {len(matches)} " + f"matching {pattern} in {search_dir}: {listed}" + ) + return matches[0] + raise FileNotFoundError( + f"No labels file found in {search_dir}. Expected exactly one '*.geojson' or '*.json' file." + ) + + # Remote directory/prefix fallback. + if "://" in labels_value: + for pattern in label_patterns: + try: + local_dir = resolve_directory(labels_value, pattern=pattern) + except FileNotFoundError: + continue + return _pick_single_candidate(local_dir) + raise FileNotFoundError( + f"No labels file found at {labels_value}. Expected exactly one '*.geojson' or '*.json' file." + ) + + # Local directory/file fallback. + local_path = Path(labels_value) + if local_path.is_file(): + if local_path.suffix.lower() not in (".geojson", ".json"): + raise ValueError(f"dataset_labels must be a .geojson or .json file, got: {local_path}") + return local_path + if local_path.is_dir(): + return _pick_single_candidate(local_path) + + raise FileNotFoundError(f"dataset_labels path not found: {local_path}. Provide a labels file or directory/prefix.") def _download_and_extract_zip(zip_url: str, dest_dir: Path) -> None: @@ -190,7 +244,7 @@ def _materialize_training_input(dataset_chips: str, dataset_labels: str, work_di can parse tile ids. """ chips_dir = _resolve_input_directory(dataset_chips, "dataset_chips") - labels_path = _resolve_input_file(dataset_labels, "dataset_labels") + labels_path = _resolve_labels_geojson(dataset_labels) input_dir = work_dir / "input" if input_dir.exists(): @@ -221,8 +275,6 @@ def _materialize_training_input(dataset_chips: str, dataset_labels: str, work_di if not list(input_dir.glob("*.png")): raise FileNotFoundError(f"No train chips (.tif/.tiff/.png) found in {chips_dir}") - if not labels_path.is_file(): - raise FileNotFoundError(f"dataset_labels file not found: {labels_path}") shutil.copy2(labels_path, input_dir / "labels.geojson") return input_dir From 9c56e3d4aef868526372f7bdf430dd134abf91f1 Mon Sep 17 00:00:00 2001 From: Abdelrahman Katkat Date: Thu, 21 May 2026 14:58:37 +0300 Subject: [PATCH 21/30] refactor(pipeline): streamline label resolution and add model file path handling Refactored the `_resolve_labels_geojson` function to simplify label resolution for GeoJSON/JSON files, removing redundant file path checks. Introduced a new helper function `_resolve_model_file_path` to handle the resolution of model file paths from various sources, improving code organization and maintainability. Updated the STAC item JSON to reflect a new citation URL. --- models/ramp/pipeline.py | 70 ++++++++++++++++---------------------- models/ramp/stac-item.json | 2 +- 2 files changed, 30 insertions(+), 42 deletions(-) diff --git a/models/ramp/pipeline.py b/models/ramp/pipeline.py index 9ea920b2..7bd9b9bc 100644 --- a/models/ramp/pipeline.py +++ b/models/ramp/pipeline.py @@ -50,19 +50,9 @@ def _resolve_labels_geojson(dataset_labels: str) -> Path: from fair.utils.data import resolve_directory, resolve_path label_patterns = ("*.geojson", "*.json") + label_suffixes = (".geojson", ".json") labels_value = str(dataset_labels) - # First attempt: resolve as a file path/URI. - try: - file_candidate = resolve_path(labels_value) - except Exception: # pragma: no cover - fallback to directory/prefix resolution - file_candidate = None - - if file_candidate and file_candidate.is_file(): - if file_candidate.suffix.lower() not in (".geojson", ".json"): - raise ValueError(f"dataset_labels must point to a .geojson or .json file, got: {file_candidate}") - return file_candidate - def _pick_single_candidate(search_dir: Path) -> Path: for pattern in label_patterns: matches = sorted(p for p in search_dir.glob(pattern) if p.is_file()) @@ -79,8 +69,13 @@ def _pick_single_candidate(search_dir: Path) -> Path: f"No labels file found in {search_dir}. Expected exactly one '*.geojson' or '*.json' file." ) - # Remote directory/prefix fallback. + # Remote file or directory/prefix. if "://" in labels_value: + if labels_value.lower().endswith(label_suffixes): + file_candidate = resolve_path(labels_value) + if file_candidate.suffix.lower() not in label_suffixes: + raise ValueError(f"dataset_labels must point to a .geojson or .json file, got: {file_candidate}") + return file_candidate for pattern in label_patterns: try: local_dir = resolve_directory(labels_value, pattern=pattern) @@ -94,7 +89,7 @@ def _pick_single_candidate(search_dir: Path) -> Path: # Local directory/file fallback. local_path = Path(labels_value) if local_path.is_file(): - if local_path.suffix.lower() not in (".geojson", ".json"): + if local_path.suffix.lower() not in label_suffixes: raise ValueError(f"dataset_labels must be a .geojson or .json file, got: {local_path}") return local_path if local_path.is_dir(): @@ -120,6 +115,24 @@ def _extract_zip(zip_path: Path, dest_dir: Path) -> None: zf.extractall(dest_dir) +def _resolve_model_file_path(model_uri: str, cache_dir: Path, default_name: str) -> Path: + """Resolve a local, HTTP(S), or remote (s3://) model file to a local Path.""" + from fair.utils.data import resolve_path + + clean_uri = model_uri.split("?", 1)[0] + if model_uri.startswith(("http://", "https://")): + dest = cache_dir / (Path(clean_uri).name or default_name) + if not dest.is_file(): + urlretrieve(model_uri, dest) + return dest + if "://" in model_uri: + return resolve_path(model_uri, local_dir=cache_dir) + p = Path(model_uri).resolve() + if p.is_file(): + return p + raise FileNotFoundError(f"Model file not found: {p}") + + def resolve_model_href(model_uri: str, cache_dir: Path | None = None) -> str: """Resolve .onnx, .keras, or .zip model URIs to local paths. - .onnx -> local .onnx file path @@ -132,32 +145,14 @@ def resolve_model_href(model_uri: str, cache_dir: Path | None = None) -> str: raise TypeError("cache_dir must be a pathlib.Path or None") cache_dir = cache_dir or _DEFAULT_MODEL_CACHE cache_dir.mkdir(parents=True, exist_ok=True) - is_http = model_uri.startswith(("http://", "https://")) clean_uri = model_uri.split("?", 1)[0] suffix = Path(clean_uri).suffix.lower() # ONNX path if suffix == ".onnx": - if is_http: - dest = cache_dir / (Path(clean_uri).name or "model.onnx") - if not dest.is_file(): - urlretrieve(model_uri, dest) - return str(dest) - # local onnx path - p = Path(model_uri).resolve() - if p.is_file(): - return str(p) - raise FileNotFoundError(f"ONNX model not found: {p}") + return str(_resolve_model_file_path(model_uri, cache_dir, "model.onnx")) # Keras single-file checkpoint if suffix == ".keras": - if is_http: - dest = cache_dir / (Path(clean_uri).name or "model.keras") - if not dest.is_file(): - urlretrieve(model_uri, dest) - return str(dest) - p = Path(model_uri).resolve() - if p.is_file(): - return str(p) - raise FileNotFoundError(f"Keras model not found: {p}") + return str(_resolve_model_file_path(model_uri, cache_dir, "model.keras")) # ZIP path -> must contain saved_model.pb somewhere inside if suffix == ".zip": @@ -166,14 +161,7 @@ def resolve_model_href(model_uri: str, cache_dir: Path | None = None) -> str: # cache hit for existing in dest_dir.rglob("saved_model.pb"): return str(existing.parent) - if is_http: - zip_path = cache_dir / (Path(clean_uri).name or "model.zip") - if not zip_path.is_file(): - urlretrieve(model_uri, zip_path) - else: - zip_path = Path(model_uri).resolve() - if not zip_path.is_file(): - raise FileNotFoundError(f"ZIP model not found: {zip_path}") + zip_path = _resolve_model_file_path(model_uri, cache_dir, "model.zip") dest_dir.mkdir(parents=True, exist_ok=True) import zipfile diff --git a/models/ramp/stac-item.json b/models/ramp/stac-item.json index 62830ff8..9bf1e8ff 100644 --- a/models/ramp/stac-item.json +++ b/models/ramp/stac-item.json @@ -366,7 +366,7 @@ }, { "rel": "cite-as", - "href": "https://github.com/radiantearth/ramp-code", + "href": "https://github.com/devglobalpartners/ramp-code", "type": "text/html", "title": "RAMP — Replicable AI for Microplanning" } From 8c751d377c70d1becbc324fbadfe9ee83eb2e20b Mon Sep 17 00:00:00 2001 From: Abdelrahman Katkat Date: Sat, 23 May 2026 17:42:27 +0300 Subject: [PATCH 22/30] chore(config): update kind configuration and adjust STAC item parameters Added new labels for training and inference to the kind configuration. Updated the default values for training epochs and batch size in the STAC item JSON, reflecting a more suitable configuration for model training. --- infra/kind-config.yaml | 2 ++ models/ramp/stac-item.json | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/infra/kind-config.yaml b/infra/kind-config.yaml index cbff373c..b4256b6b 100644 --- a/infra/kind-config.yaml +++ b/infra/kind-config.yaml @@ -5,3 +5,5 @@ nodes: labels: fair/role: infra fair/workload: ml + fair.dev/training: "true" + fair.dev/inference: "true" diff --git a/models/ramp/stac-item.json b/models/ramp/stac-item.json index 9bf1e8ff..8a65b197 100644 --- a/models/ramp/stac-item.json +++ b/models/ramp/stac-item.json @@ -107,17 +107,17 @@ { "key": "epochs", "type": "int", - "default": 20, + "default": 5, "min": 1, - "max": 20, + "max": 16, "description": "Number of training epochs" }, { "key": "batch_size", "type": "int", - "default": 8, + "default": 1, "min": 1, - "max": 16, + "max": 8, "description": "Samples per training batch" }, { From 8b3e5274ed62b7c367b06be6a7af2155f7b52d70 Mon Sep 17 00:00:00 2001 From: Abdelrahman Katkat Date: Sun, 24 May 2026 16:25:13 +0300 Subject: [PATCH 23/30] chore(config): remove training and inference labels from kind configuration Deleted the training and inference labels from the kind configuration file to streamline the setup and reduce unnecessary complexity. --- infra/kind-config.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/infra/kind-config.yaml b/infra/kind-config.yaml index b4256b6b..cbff373c 100644 --- a/infra/kind-config.yaml +++ b/infra/kind-config.yaml @@ -5,5 +5,3 @@ nodes: labels: fair/role: infra fair/workload: ml - fair.dev/training: "true" - fair.dev/inference: "true" From fb9bde221d94a564602bd08b47872aea84dfc0ab Mon Sep 17 00:00:00 2001 From: Abdelrahman Katkat Date: Sun, 24 May 2026 16:37:26 +0300 Subject: [PATCH 24/30] chore(pipeline): integrate MLflow context and enhance model evaluation logging Added MLflow training context to the model training process for better tracking. Enhanced model evaluation by logging evaluation results for both zero metrics and computed metrics, improving observability of model performance. --- infra/kind-config.yaml | 2 ++ models/ramp/pipeline.py | 30 +++++++++++++++++------------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/infra/kind-config.yaml b/infra/kind-config.yaml index cbff373c..b4256b6b 100644 --- a/infra/kind-config.yaml +++ b/infra/kind-config.yaml @@ -5,3 +5,5 @@ nodes: labels: fair/role: infra fair/workload: ml + fair.dev/training: "true" + fair.dev/inference: "true" diff --git a/models/ramp/pipeline.py b/models/ramp/pipeline.py index 7bd9b9bc..bce2a223 100644 --- a/models/ramp/pipeline.py +++ b/models/ramp/pipeline.py @@ -12,6 +12,7 @@ from zenml import log_metadata, pipeline, step +from fair.zenml.instrumentation import log_evaluation_results, mlflow_training_context from fair.zenml.materializers import CheckpointBytesMaterializer, ONNXMaterializer _DEFAULT_MODEL_CACHE = Path("/workspace/.ramp_model_cache") @@ -643,7 +644,7 @@ def train_model( dataset_id: str | None = None, ) -> Annotated[bytes, "trained_model"]: """Fine-tune RAMP EfficientNetB0 U-Net; return the best model as `.keras` bytes.""" - _ = (num_classes, model_name, base_model_id, dataset_id) + _ = num_classes ramp_train_dir = Path(split_info["_ramp_train_dir"]) if not ramp_train_dir.exists(): @@ -651,19 +652,20 @@ def train_model( ramp_train_dir = Path(split_info["_ramp_train_dir"]) work_dir = split_info.get("_work_dir") or str(ramp_train_dir.parent) - final_model_path = train_ramp_model( - ramp_train_dir=str(ramp_train_dir), - base_model_weights=base_model_weights, - hyperparameters=hyperparameters, - data_base_path=work_dir, - ) - import tensorflow as tf + with mlflow_training_context(hyperparameters, model_name, base_model_id, dataset_id): + final_model_path = train_ramp_model( + ramp_train_dir=str(ramp_train_dir), + base_model_weights=base_model_weights, + hyperparameters=hyperparameters, + data_base_path=work_dir, + ) + import tensorflow as tf - model = tf.keras.models.load_model(str(final_model_path), compile=False) - exported_path = Path(tempfile.mkdtemp(prefix="ramp_keras_export_")) / "model.keras" - model.save(str(exported_path)) - blob = exported_path.read_bytes() - log_metadata(metadata={"keras_path": str(exported_path), "checkpoint_bytes": len(blob)}) + model = tf.keras.models.load_model(str(final_model_path), compile=False) + exported_path = Path(tempfile.mkdtemp(prefix="ramp_keras_export_")) / "model.keras" + model.save(str(exported_path)) + blob = exported_path.read_bytes() + log_metadata(metadata={"keras_path": str(exported_path), "checkpoint_bytes": len(blob)}) return blob @@ -706,6 +708,7 @@ def evaluate_model( "fair:precision": 0.0, "fair:recall": 0.0, } + log_evaluation_results(zero_metrics) log_metadata(metadata=zero_metrics) return zero_metrics @@ -749,6 +752,7 @@ def evaluate_model( "fair:precision": float(precision), "fair:recall": float(recall), } + log_evaluation_results(metrics_dict) log_metadata(metadata=metrics_dict) return metrics_dict From 323652a127e2f508abb1d9ac205e248cfc129113 Mon Sep 17 00:00:00 2001 From: Abdelrahman Katkat Date: Sun, 24 May 2026 16:56:02 +0300 Subject: [PATCH 25/30] fix(pipeline): update output materializer label in train_model function Changed the output materializer label from "trained_model" to "trained_model_artifact" in the train_model function to better reflect the returned artifact type. --- models/ramp/pipeline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/ramp/pipeline.py b/models/ramp/pipeline.py index bce2a223..874eae6a 100644 --- a/models/ramp/pipeline.py +++ b/models/ramp/pipeline.py @@ -631,7 +631,7 @@ def split_dataset( return split_info -@step(output_materializers={"trained_model": CheckpointBytesMaterializer}) +@step(output_materializers={"trained_model_artifact": CheckpointBytesMaterializer}) def train_model( dataset_chips: str, dataset_labels: str, @@ -642,7 +642,7 @@ def train_model( model_name: str | None = None, base_model_id: str | None = None, dataset_id: str | None = None, -) -> Annotated[bytes, "trained_model"]: +) -> Annotated[bytes, "trained_model_artifact"]: """Fine-tune RAMP EfficientNetB0 U-Net; return the best model as `.keras` bytes.""" _ = num_classes From 4b3cdad79b5a0c90859dc794f0baef16f792e5ff Mon Sep 17 00:00:00 2001 From: Abdelrahman Katkat Date: Sun, 24 May 2026 22:50:55 +0300 Subject: [PATCH 26/30] feat(pipeline): enhance model loading and evaluation metrics Updated the model loading process to support both `.keras` and `.h5` formats, improving flexibility. Adjusted evaluation metrics to remove the "fair:" prefix for consistency. Additionally, modified the STAC item JSON to reduce training epochs and batch size for better initial training performance. --- fair/zenml/config.py | 1 + models/ramp/pipeline.py | 116 ++++++++++++++++++++++---------- models/ramp/stac-item.json | 4 +- models/ramp/tests/test_steps.py | 3 +- 4 files changed, 85 insertions(+), 39 deletions(-) diff --git a/fair/zenml/config.py b/fair/zenml/config.py index f726a3e5..97802919 100644 --- a/fair/zenml/config.py +++ b/fair/zenml/config.py @@ -230,6 +230,7 @@ def generate_training_config( if k8s: config.setdefault("steps", {}).setdefault("train_model", {}).setdefault("settings", {}).update(k8s) config.setdefault("steps", {}).setdefault("evaluate_model", {}).setdefault("settings", {}).update(k8s) + config.setdefault("steps", {}).setdefault("export_onnx", {}).setdefault("settings", {}).update(k8s) return config diff --git a/models/ramp/pipeline.py b/models/ramp/pipeline.py index 874eae6a..78ca0513 100644 --- a/models/ramp/pipeline.py +++ b/models/ramp/pipeline.py @@ -55,19 +55,24 @@ def _resolve_labels_geojson(dataset_labels: str) -> Path: labels_value = str(dataset_labels) def _pick_single_candidate(search_dir: Path) -> Path: + for name in ("labels.geojson", "labels.json", "label.geojson", "label.json"): + candidate = search_dir / name + if candidate.is_file(): + return candidate + for pattern in label_patterns: matches = sorted(p for p in search_dir.glob(pattern) if p.is_file()) - if not matches: - continue + if len(matches) == 1: + return matches[0] if len(matches) > 1: listed = ", ".join(str(p) for p in matches) raise ValueError( f"dataset_labels must resolve to exactly one labels file, found {len(matches)} " - f"matching {pattern} in {search_dir}: {listed}" + f"matching {pattern} in {search_dir}: {listed}. " + "Rename your labels file to labels.geojson (or pass an explicit file path)." ) - return matches[0] raise FileNotFoundError( - f"No labels file found in {search_dir}. Expected exactly one '*.geojson' or '*.json' file." + f"No labels file found in {search_dir}. Expected labels.geojson or exactly one '*.geojson' file." ) # Remote file or directory/prefix. @@ -461,22 +466,69 @@ def train_ramp_model( return Path(final_model_path) -def _materialize_keras_bytes(blob: bytes) -> Path: - """Write Keras `.keras` bytes to a temp file and return its path.""" - dest = Path(tempfile.mkdtemp(prefix="ramp_keras_")) / "model.keras" +def _load_ramp_keras_model(model_path: str | Path) -> Any: + """Load a RAMP Keras checkpoint with segmentation_models custom layers registered.""" + import tensorflow as tf + + os.environ.setdefault("SM_FRAMEWORK", "tf.keras") + import segmentation_models as sm + + sm.set_framework("tf.keras") + custom_objects: dict[str, Any] = {} + try: + from segmentation_models.utils import get_custom_objects + + custom_objects.update(get_custom_objects()) + except ImportError: + pass + try: + from efficientnet.tfkeras import FixedDropout + + custom_objects.setdefault("FixedDropout", FixedDropout) + except ImportError: + pass + + return tf.keras.models.load_model( + str(model_path), + compile=False, + custom_objects=custom_objects or None, + safe_mode=False, + ) + + +def _materialize_h5_bytes(blob: bytes) -> Path: + """Write Keras `.h5` bytes to a temp file and return its path.""" + dest = Path(tempfile.mkdtemp(prefix="ramp_h5_")) / "model.h5" dest.write_bytes(blob) return dest +def _resolve_h5_checkpoint_path(model_path: Path) -> Path: + """Resolve a RAMP training output path to a loadable `.h5` file.""" + if model_path.is_file() and (model_path.suffix.lower() == ".h5" or model_path.name.endswith(".h5")): + return model_path + h5_sibling = Path(f"{model_path}.h5") if not str(model_path).endswith(".h5") else model_path + if h5_sibling.is_file(): + return h5_sibling + if model_path.parent.is_dir(): + matches = sorted(model_path.parent.glob("*.h5")) + if len(matches) == 1: + return matches[0] + return model_path + + def _restore_checkpoint(trained_model: Any) -> Path: - """Restore a trained RAMP Keras checkpoint as a local `.keras` file path.""" + """Restore a trained RAMP checkpoint as a local `.h5` file path.""" if isinstance(trained_model, bytes): - return _materialize_keras_bytes(trained_model) + return _materialize_h5_bytes(trained_model) if isinstance(trained_model, (str, Path)): p = Path(str(trained_model)) + if p.is_file() and (p.suffix.lower() == ".h5" or p.name.endswith(".h5")): + return p if p.is_file() and p.suffix.lower() == ".keras": return p - return Path(resolve_model_href(str(p))) + resolved = Path(resolve_model_href(str(p))) + return _resolve_h5_checkpoint_path(resolved) raise TypeError(f"Cannot restore RAMP checkpoint from {type(trained_model).__name__}") @@ -643,7 +695,7 @@ def train_model( base_model_id: str | None = None, dataset_id: str | None = None, ) -> Annotated[bytes, "trained_model_artifact"]: - """Fine-tune RAMP EfficientNetB0 U-Net; return the best model as `.keras` bytes.""" + """Fine-tune RAMP EfficientNetB0 U-Net; return the best model as `.h5` bytes.""" _ = num_classes ramp_train_dir = Path(split_info["_ramp_train_dir"]) @@ -659,13 +711,13 @@ def train_model( hyperparameters=hyperparameters, data_base_path=work_dir, ) - import tensorflow as tf - - model = tf.keras.models.load_model(str(final_model_path), compile=False) - exported_path = Path(tempfile.mkdtemp(prefix="ramp_keras_export_")) / "model.keras" - model.save(str(exported_path)) - blob = exported_path.read_bytes() - log_metadata(metadata={"keras_path": str(exported_path), "checkpoint_bytes": len(blob)}) + h5_path = _resolve_h5_checkpoint_path(Path(final_model_path)) + if not h5_path.is_file(): + model = _load_ramp_keras_model(final_model_path) + h5_path = Path(tempfile.mkdtemp(prefix="ramp_h5_export_")) / "model.h5" + model.save(str(h5_path), save_format="h5") + blob = h5_path.read_bytes() + log_metadata(metadata={"h5_path": str(h5_path), "checkpoint_bytes": len(blob)}) return blob @@ -700,21 +752,16 @@ def evaluate_model( restored = _restore_checkpoint(trained_model) if not pairs: - # No val data to evaluate against (e.g. CI mocks); return zeroed metrics with the - # required fair:* keys so downstream validation still sees the expected schema. zero_metrics: dict[str, Any] = { - "fair:accuracy": 0.0, - "fair:mean_iou": 0.0, - "fair:precision": 0.0, - "fair:recall": 0.0, + "accuracy": 0.0, + "mean_iou": 0.0, + "precision": 0.0, + "recall": 0.0, } log_evaluation_results(zero_metrics) - log_metadata(metadata=zero_metrics) return zero_metrics - import tensorflow as tf - - model = tf.keras.models.load_model(str(restored), compile=False) + model = _load_ramp_keras_model(restored) tp = fp = fn = 0 correct = total = 0 @@ -747,13 +794,12 @@ def evaluate_model( accuracy = correct / total if total > 0 else 0.0 metrics_dict: dict[str, Any] = { - "fair:accuracy": float(accuracy), - "fair:mean_iou": float(iou), - "fair:precision": float(precision), - "fair:recall": float(recall), + "accuracy": float(accuracy), + "mean_iou": float(iou), + "precision": float(precision), + "recall": float(recall), } log_evaluation_results(metrics_dict) - log_metadata(metadata=metrics_dict) return metrics_dict @@ -765,7 +811,7 @@ def export_onnx(trained_model: Any) -> Annotated[bytes, "onnx_model"]: import tf2onnx restored = _restore_checkpoint(trained_model) - model = tf.keras.models.load_model(str(restored), compile=False) + model = _load_ramp_keras_model(restored) with tempfile.TemporaryDirectory() as tmp: onnx_path = Path(tmp) / "model.onnx" tf2onnx.convert.from_keras(model, opset=13, output_path=str(onnx_path)) diff --git a/models/ramp/stac-item.json b/models/ramp/stac-item.json index 8a65b197..5f1cc5ee 100644 --- a/models/ramp/stac-item.json +++ b/models/ramp/stac-item.json @@ -272,8 +272,8 @@ ], "mlm:hyperparameters": { "training.backbone": "efficientnetb0", - "training.epochs": 20, - "training.batch_size": 8, + "training.epochs": 1, + "training.batch_size": 1, "training.learning_rate": 0.0003, "training.val_ratio": 0.15, "training.split_seed": 42, diff --git a/models/ramp/tests/test_steps.py b/models/ramp/tests/test_steps.py index a0e0f066..2c5b05e0 100644 --- a/models/ramp/tests/test_steps.py +++ b/models/ramp/tests/test_steps.py @@ -135,8 +135,7 @@ def test_evaluate_model( split_info=split_info, ) - expected = {"fair:accuracy", "fair:mean_iou", "fair:precision", "fair:recall"} - assert set(metrics.keys()) == expected + assert set(metrics) == {"accuracy", "mean_iou", "precision", "recall"} for value in metrics.values(): assert isinstance(value, float) assert 0.0 <= value <= 1.0 From e9f4d487a1d64bf03296700e02352a21c2caa013 Mon Sep 17 00:00:00 2001 From: Abdelrahman Katkat Date: Sun, 24 May 2026 23:23:57 +0300 Subject: [PATCH 27/30] refactor(pipeline): improve path resolution and model loading logic Renamed and refactored functions for clarity and consistency in handling local and remote paths. Updated the model loading process to exclusively support ZIP files containing SavedModel directories, enhancing error handling and simplifying the extraction logic. Adjusted related function calls to reflect these changes, improving overall code maintainability. --- models/ramp/pipeline.py | 125 +++++++++++++--------------------------- 1 file changed, 40 insertions(+), 85 deletions(-) diff --git a/models/ramp/pipeline.py b/models/ramp/pipeline.py index 78ca0513..59855f44 100644 --- a/models/ramp/pipeline.py +++ b/models/ramp/pipeline.py @@ -8,7 +8,6 @@ import zipfile from pathlib import Path from typing import Annotated, Any -from urllib.request import urlretrieve from zenml import log_metadata, pipeline, step @@ -19,26 +18,25 @@ _QUBVEL_EFFICIENTNET_RELEASE = "https://github.com/qubvel/efficientnet/releases/download/v0.0.1/" -def _to_local_path(path_value: str, purpose: str) -> Path: - """Resolve a path with UPath and ensure local filesystem semantics.""" - from upath import UPath - - upath_obj = UPath(path_value) - protocol = getattr(upath_obj, "protocol", "") or "" - if protocol not in ("", "file"): +def _require_local_path(path_value: str, purpose: str) -> Path: + """Ensure path_value is local and return a Path object.""" + if "://" in path_value and not path_value.startswith("file://"): + protocol = path_value.split("://", 1)[0] raise NotImplementedError( f"{purpose} requires a local filesystem path. Received protocol={protocol!r} for {path_value!r}." ) - return Path(str(upath_obj)) + if path_value.startswith("file://"): + return Path(path_value.removeprefix("file://")) + return Path(path_value) -def _resolve_input_directory(path_value: str, purpose: str) -> Path: +def _resolve_input(path_value: str, purpose: str) -> Path: """Resolve local/remote dataset directories to a local path.""" from fair.utils.data import resolve_directory if "://" in str(path_value): return resolve_directory(path_value, pattern="*") - return _to_local_path(path_value, purpose) + return _require_local_path(path_value, purpose) def _resolve_labels_geojson(dataset_labels: str) -> Path: @@ -104,79 +102,36 @@ def _pick_single_candidate(search_dir: Path) -> Path: raise FileNotFoundError(f"dataset_labels path not found: {local_path}. Provide a labels file or directory/prefix.") -def _download_and_extract_zip(zip_url: str, dest_dir: Path) -> None: - """Download a ZIP URL and extract in dest_dir.""" - dest_dir.mkdir(parents=True, exist_ok=True) - zip_name = Path(zip_url.split("/")[-1]).name or "ramp_v1.zip" - zip_path = dest_dir / zip_name - urlretrieve(zip_url, zip_path) - with zipfile.ZipFile(zip_path, "r") as zf: - zf.extractall(dest_dir) - zip_path.unlink(missing_ok=True) - - -def _extract_zip(zip_path: Path, dest_dir: Path) -> None: - dest_dir.mkdir(parents=True, exist_ok=True) - with zipfile.ZipFile(zip_path, "r") as zf: - zf.extractall(dest_dir) - - -def _resolve_model_file_path(model_uri: str, cache_dir: Path, default_name: str) -> Path: - """Resolve a local, HTTP(S), or remote (s3://) model file to a local Path.""" +def _resolve_savedmodel_dir_from_zip(model_uri: str, cache_dir: Path) -> Path: + """Resolve a local/remote ZIP model URI to an extracted SavedModel directory.""" from fair.utils.data import resolve_path - clean_uri = model_uri.split("?", 1)[0] - if model_uri.startswith(("http://", "https://")): - dest = cache_dir / (Path(clean_uri).name or default_name) - if not dest.is_file(): - urlretrieve(model_uri, dest) - return dest - if "://" in model_uri: - return resolve_path(model_uri, local_dir=cache_dir) - p = Path(model_uri).resolve() - if p.is_file(): - return p - raise FileNotFoundError(f"Model file not found: {p}") - - -def resolve_model_href(model_uri: str, cache_dir: Path | None = None) -> str: - """Resolve .onnx, .keras, or .zip model URIs to local paths. - - .onnx -> local .onnx file path - - .keras -> local .keras file path - - .zip -> extracted SavedModel directory path (folder containing saved_model.pb) - """ if not isinstance(model_uri, str): raise TypeError("model_uri must be a string") - if cache_dir is not None and not isinstance(cache_dir, Path): - raise TypeError("cache_dir must be a pathlib.Path or None") - cache_dir = cache_dir or _DEFAULT_MODEL_CACHE + if not isinstance(cache_dir, Path): + raise TypeError("cache_dir must be a pathlib.Path") cache_dir.mkdir(parents=True, exist_ok=True) + clean_uri = model_uri.split("?", 1)[0] suffix = Path(clean_uri).suffix.lower() - # ONNX path - if suffix == ".onnx": - return str(_resolve_model_file_path(model_uri, cache_dir, "model.onnx")) - # Keras single-file checkpoint - if suffix == ".keras": - return str(_resolve_model_file_path(model_uri, cache_dir, "model.keras")) - - # ZIP path -> must contain saved_model.pb somewhere inside - if suffix == ".zip": - stem = Path(clean_uri).stem or "ramp_model" - dest_dir = cache_dir / stem - # cache hit - for existing in dest_dir.rglob("saved_model.pb"): - return str(existing.parent) - zip_path = _resolve_model_file_path(model_uri, cache_dir, "model.zip") - dest_dir.mkdir(parents=True, exist_ok=True) - import zipfile - - with zipfile.ZipFile(zip_path, "r") as zf: - zf.extractall(dest_dir) - for existing in dest_dir.rglob("saved_model.pb"): - return str(existing.parent) - raise RuntimeError(f"Zip from {model_uri} did not contain a valid SavedModel") - raise ValueError(f"Unsupported model format for {model_uri!r}. Only .onnx, .keras and .zip are accepted.") + if suffix != ".zip": + raise ValueError(f"Unsupported model format for {model_uri!r}. Only .zip is accepted for RAMP baselines.") + + stem = Path(clean_uri).stem or "ramp_model" + dest_dir = cache_dir / stem + for existing in dest_dir.rglob("saved_model.pb"): + return existing.parent + + zip_path = resolve_path(model_uri, local_dir=cache_dir) if "://" in model_uri else Path(model_uri).resolve() + if not zip_path.is_file(): + raise FileNotFoundError(f"Model zip not found: {zip_path}") + + dest_dir.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(zip_path, "r") as zf: + zf.extractall(dest_dir) + for existing in dest_dir.rglob("saved_model.pb"): + return existing.parent + raise RuntimeError(f"Zip from {model_uri} did not contain a valid SavedModel") def _ensure_ramp_baseline(base_model_weights: str, data_base_path: str | Path) -> Path: @@ -195,7 +150,7 @@ def _ensure_ramp_baseline(base_model_weights: str, data_base_path: str | Path) - if not base_model_weights: raise ValueError("RAMP baseline weights are required but were not provided. ") - return Path(resolve_model_href(base_model_weights, cache_dir=Path(data_base_path) / ".baseline_cache")) + return _resolve_savedmodel_dir_from_zip(base_model_weights, cache_dir=Path(data_base_path) / ".baseline_cache") def _patch_keras_get_file_for_efficientnet_weights() -> None: @@ -237,7 +192,7 @@ def _materialize_training_input(dataset_chips: str, dataset_labels: str, work_di preserving filename stem (e.g. OAM-{x}-{y}-{z}.png) so hot_fair_utilities label clipping can parse tile ids. """ - chips_dir = _resolve_input_directory(dataset_chips, "dataset_chips") + chips_dir = _resolve_input(dataset_chips, "dataset_chips") labels_path = _resolve_labels_geojson(dataset_labels) input_dir = work_dir / "input" @@ -288,7 +243,7 @@ def preprocess( """ from hot_fair_utilities import preprocess as _preprocess - local_input = _resolve_input_directory(input_path, "input_path") + local_input = _resolve_input(input_path, "input_path") _preprocess( input_path=str(local_input), output_path=output_path, @@ -310,8 +265,8 @@ def postprocess(prediction_masks_dir: str, output_dir: str) -> dict[str, Any]: from geomltoolkits.raster.morphology import morphological_cleaning from geomltoolkits.raster.vectorize import vectorize_mask - pred_dir = _to_local_path(prediction_masks_dir, "prediction_masks_dir") - out_dir = _to_local_path(output_dir, "output_dir") + pred_dir = _require_local_path(prediction_masks_dir, "prediction_masks_dir") + out_dir = _require_local_path(output_dir, "output_dir") out_dir.mkdir(parents=True, exist_ok=True) pred_tifs = sorted(pred_dir.glob("*.tif")) @@ -527,8 +482,9 @@ def _restore_checkpoint(trained_model: Any) -> Path: return p if p.is_file() and p.suffix.lower() == ".keras": return p - resolved = Path(resolve_model_href(str(p))) - return _resolve_h5_checkpoint_path(resolved) + if p.exists(): + return _resolve_h5_checkpoint_path(p) + raise FileNotFoundError(f"Checkpoint path not found: {p}") raise TypeError(f"Cannot restore RAMP checkpoint from {type(trained_model).__name__}") @@ -807,7 +763,6 @@ def evaluate_model( def export_onnx(trained_model: Any) -> Annotated[bytes, "onnx_model"]: """Convert the trained RAMP Keras checkpoint to ONNX bytes and validate.""" import onnx - import tensorflow as tf import tf2onnx restored = _restore_checkpoint(trained_model) From 0c9780a03867fea860a4f9290471b5622310893b Mon Sep 17 00:00:00 2001 From: Abdelrahman Katkat Date: Sun, 24 May 2026 23:54:23 +0300 Subject: [PATCH 28/30] fix(ramp): update STAC item ID and name for consistency; change Dockerfile user to root Modified the STAC item JSON to change the ID and name from "ramp-v1" to "ramp" for better alignment with naming conventions. Updated the Dockerfile to set the user to root, ensuring the k8s ZenML orchestrator can access the in-cluster service account token without authentication issues. --- models/ramp/Dockerfile | 5 +++++ models/ramp/stac-item.json | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/models/ramp/Dockerfile b/models/ramp/Dockerfile index 5d9c2fb2..6695d213 100644 --- a/models/ramp/Dockerfile +++ b/models/ramp/Dockerfile @@ -53,6 +53,11 @@ ENV PATH="/app/.venv/bin:$PATH" \ NVIDIA_VISIBLE_DEVICES=all \ NVIDIA_DRIVER_CAPABILITIES=compute,utility +# The k8s ZenML orchestrator entrypoint needs to read the in-cluster service +# account token. Some base images run as a non-root user which can cause the +# k8s Python client to fall back to anonymous auth (403 on pod reads) in CI. +USER root + COPY --from=builder /app/.venv /app/.venv COPY models/ramp models/ramp diff --git a/models/ramp/stac-item.json b/models/ramp/stac-item.json index 5f1cc5ee..4b5eb366 100644 --- a/models/ramp/stac-item.json +++ b/models/ramp/stac-item.json @@ -9,7 +9,7 @@ "https://stac-extensions.github.io/raster/v1.1.0/schema.json", "https://hotosm.github.io/fAIr-models/schemas/v1.0.0/base-model/schema.json" ], - "id": "ramp-v1", + "id": "ramp", "geometry": { "type": "Polygon", "coordinates": [ @@ -29,7 +29,7 @@ "updated": "2024-01-01T00:00:00Z", "title": "RAMP EfficientNetB0 + U-Net Building Segmentation", "description": "RAMP semantic-segmentation base model: EfficientNetB0 encoder with U-Net decoder outputs 4-class sparse-categorical masks (background, building, boundary, contact) from RGB aerial imagery. Packaged for fAIr finetuning and ONNX-based inference.", - "mlm:name": "ramp-v1", + "mlm:name": "ramp", "mlm:architecture": "EffUnet", "mlm:tasks": [ "semantic-segmentation" From e9189ca06331b93d117b14ec6e59ef30224822ce Mon Sep 17 00:00:00 2001 From: Abdelrahman Katkat Date: Mon, 25 May 2026 02:49:27 +0300 Subject: [PATCH 29/30] refactor(ramp): remove unnecessary root user setting in Dockerfile Eliminated the USER root directive from the Dockerfile, as it was no longer needed for the k8s ZenML orchestrator to access the in-cluster service account token. This change simplifies the Dockerfile and enhances security by avoiding unnecessary root privileges. --- models/ramp/Dockerfile | 5 ----- 1 file changed, 5 deletions(-) diff --git a/models/ramp/Dockerfile b/models/ramp/Dockerfile index 6695d213..5d9c2fb2 100644 --- a/models/ramp/Dockerfile +++ b/models/ramp/Dockerfile @@ -53,11 +53,6 @@ ENV PATH="/app/.venv/bin:$PATH" \ NVIDIA_VISIBLE_DEVICES=all \ NVIDIA_DRIVER_CAPABILITIES=compute,utility -# The k8s ZenML orchestrator entrypoint needs to read the in-cluster service -# account token. Some base images run as a non-root user which can cause the -# k8s Python client to fall back to anonymous auth (403 on pod reads) in CI. -USER root - COPY --from=builder /app/.venv /app/.venv COPY models/ramp models/ramp From 7366dc9dc4c56de322773fbd21712e27fdabad3d Mon Sep 17 00:00:00 2001 From: Abdelrahman Katkat Date: Mon, 25 May 2026 12:42:49 +0300 Subject: [PATCH 30/30] docs(ramp): update README for model overview and usage instructions Enhanced the README.md to provide a clearer overview of the RAMP EfficientNetB0 + U-Net model, including detailed architecture, input/output specifications, pretrained artifacts, and usage instructions for inference and fine-tuning. Added limitations and citation information for better context and usability. --- models/ramp/README.md | 85 +++++++++++++++++++++++++++---------------- 1 file changed, 54 insertions(+), 31 deletions(-) diff --git a/models/ramp/README.md b/models/ramp/README.md index e55d1893..4f997dae 100644 --- a/models/ramp/README.md +++ b/models/ramp/README.md @@ -1,60 +1,83 @@ # RAMP EfficientNetB0 + U-Net Building Segmentation -Semantic segmentation model for building footprint extraction from RGB aerial imagery, derived from the RAMP (Replicable AI for Microplanning) project. +## Overview + +RAMP is a semantic-segmentation model that extracts **building footprints** from **RGB very-high-resolution aerial/satellite imagery**. In this repository it is packaged as a **global** base model (worldwide STAC footprint) intended for **fine-tuning on a target area** and for **ONNX-based inference** within the fAIr pipeline. ## Architecture -- **Model**: EfficientNetB0 encoder + U-Net decoder (`EffUnet`) -- **Framework**: TensorFlow 2.15 / `tf.keras` (via `segmentation_models` with `SM_FRAMEWORK=tf.keras`) -- **Task**: Semantic segmentation (sparse categorical crossentropy) -- **Input**: RGB chips (256x256, float32, channels-last) -- **Classes**: 4 (background=0, building=1, boundary=2, contact=3) +- **Model**: `EffUnet` (EfficientNetB0 encoder + U-Net decoder) +- **Task**: semantic segmentation +- **Input**: \([-1, 256, 256, 3]\) float32 RGB chips (channels-last) +- **Output**: \([-1, 256, 256, 4]\) float32 per-class scores for 4 classes + - **0**: background + - **1**: building + - **2**: boundary (helps separate adjacent buildings) + - **3**: contact (helps separate close neighbouring buildings) The boundary (class 2) and contact (class 3) channels help the model cleanly separate adjacent buildings at inference time, even when they share a wall. The `predict()` helper collapses the 4-class softmax to a binary building mask before vectorization. ## Pretrained Source -Baseline RAMP weights (TensorFlow SavedModel) hosted by HOTOSM: +Pretrained artifacts for this model pack are declared in `models/ramp/stac-item.json`: + +- **Checkpoint (zipped TF SavedModel)**: `https://huggingface.co/hotosm/ramp/resolve/74daea54694f2e4924f1222520c614c7f5c029fe/v1-baseline.zip` +- **ONNX model**: `https://huggingface.co/hotosm/ramp/resolve/83c77a7e5feb3af62e3604d7bb96c6c6e9ff1a96/ramp-v1.onnx` -- Checkpoint: `https://api-prod.fair.hotosm.org/api/v1/workspace/download/ramp/baseline.zip` -- ONNX model: `https://api-prod.fair.hotosm.org/api/v1/workspace/download/ramp/ramp-v1.onnx` +Upstream RAMP resources: -## Pipeline +- **Project documentation**: `https://rampml.global/` +- **Upstream codebase**: `https://github.com/devglobalpartners/ramp-code` +- **Training data**: RAMP datasets on Radiant MLHub (see `https://rampml.global/training-data/`). Per upstream docs, labels are **manually annotated** building footprints; the published RAMP training datasets are released under **CC BY-NC 4.0**. -Training pipeline steps (ZenML) defined in `pipeline.py`: +Published paper reference: none is included in this repository; cite the upstream project documentation/repository. -- `split_dataset` - preprocesses chips + labels into 4-class multimasks and produces a seeded random train/validation split via `hot_fair_utilities.split_training_2_validation` -- `train_model` - fine-tunes RAMP on the split, returning the best SavedModel serialized as a zipped byte stream -- `evaluate_model` - computes `fair:accuracy`, `fair:mean_iou` (building IoU), `fair:precision`, and `fair:recall` on the validation split -- `export_onnx` - converts the trained SavedModel to an ONNX byte stream via `tf2onnx` +## Limitations -Inference is served through `fair.serve.base`, which calls the module-level `predict(session, input_images, params) -> FeatureCollection`: each chip is preprocessed, run through an `onnxruntime` session, decoded to a binary building mask, and vectorized to georeferenced polygons. +- **Domain shift**: performance may degrade on imagery with different sensors, resolutions, or preprocessing than the training distribution. +- **Challenging imagery**: haze, blur, strong color casts, or off-nadir views can reduce quality. +- **Dense/attached roofs**: polygonization may still merge neighbouring buildings in dense urban areas. -## Base Image +## Usage -Training, test, and inference Docker stages all build on -`ghcr.io/hotosm/fair-utilities-ramp:cpu-latest` (or `:gpu-latest` via the -`BASE_IMAGE` build arg), which provides TensorFlow, GDAL, `hot_fair_utilities` -RAMP extras, and the RAMP runtime under `/app/.venv`. +### Inference (Docker) ```bash -# Build targets -docker build -f models/ramp/Dockerfile --target runtime -t ramp:runtime . -docker build -f models/ramp/Dockerfile --target test -t ramp:test . docker build -f models/ramp/Dockerfile --target inference -t ramp:inference . +docker run --rm -p 8080:8080 -e MODEL_MODULE=models.ramp.pipeline ramp:inference + +curl -s http://localhost:8080/health +``` + +Example request (server downloads tiles into chips, runs ONNX, returns GeoJSON FeatureCollection): + +```bash +curl -s http://localhost:8080/predict \ + -H "Content-Type: application/json" \ + -d '{ + "model_uri": "https://huggingface.co/hotosm/ramp/resolve/83c77a7e5feb3af62e3604d7bb96c6c6e9ff1a96/ramp-v1.onnx", + "image_uri": "https://example.com/tiles/{z}/{x}/{y}.png", + "bbox": [0.0, 0.0, 0.01, 0.01], + "zoom": 18, + "params": { "confidence_threshold": 0.5, "min_class_value": 1 } + }' ``` -## Limitations and Bias +### Fine-tuning (Python) -- Training data and baseline weights are derived from the RAMP corpus (primarily humanitarian-mapping contexts); performance on dense urban scenes with complex roof structures may be lower than on sparser rural settlements. -- The model is sensitive to imagery with strong color casts, motion blur, or significant off-nadir angle; preprocess inputs to approximately nadir RGB at the target zoom before inference. -- Binary building output from `predict()` discards the boundary/contact auxiliary classes after decoding; downstream polygonization may still merge neighbouring buildings that share a footprint edge. +Fine-tuning is implemented as a ZenML pipeline in `models/ramp/pipeline.py` and is exercised end-to-end in `models/test_integration.py` via `fair.client.FairClient` (register → finetune → promote → predict). ## Citation -RAMP - Replicable AI for Microplanning. Upstream source: https://github.com/radiantearth/ramp-code +```bibtex +@misc{ramp_docs, + title = {Replicable AI for Microplanning (ramp)}, + author = {{DevGlobal Partners}}, + howpublished = {\\url{https://rampml.global/}}, + note = {Accessed 2026-05-25} +} +``` ## License -- Model weights and code: Apache-2.0 -- Training data: ODbL-1.0 (OpenStreetMap-derived labels) +Apache-2.0