diff --git a/docs/pipeline-setup.md b/docs/pipeline-setup.md index d720ddf..c0b9aae 100644 --- a/docs/pipeline-setup.md +++ b/docs/pipeline-setup.md @@ -138,6 +138,51 @@ staging receipts and [paid recovery](paid-recovery.md) before deliberately stagi The doctor does not initiate staging or repair. Local reference-frame models may still need their separately reviewed Torch/Hugging Face cache preparation before processing a source. +### Stage missing Modal model caches + +These commands start one paid CPU staging call per invocation. Select the saved local profile +alias for the intended workspace and use a fresh absolute Unix deadline in seconds and a unique +local receipt path for each authorized phase. Run phases serially. Set `STAGING_PROFILE` and +`STAGING_RECEIPT_DIR`, and set `DEADLINE_EPOCH` immediately before each call. For example, +`DEADLINE_EPOCH=$(( $(date +%s) + 1800 ))` allows 30 minutes from launch. Defining the function +below is local and does not start staging. + +```sh +stage_cache() { + uv run --locked modal --profile "$STAGING_PROFILE" run worker/modal_stage_weights.py::main \ + --phase "$1" --deadline-epoch "$2" --out "$STAGING_RECEIPT_DIR/$1.json" +} +``` + +| Phase command | Volume mounts and effect | +| --- | --- | +| `stage_cache clean-only "$DEADLINE_EPOCH"` | Cleaning cache only. Stages LaMa and pinned SegFormer in `wander-clean-video-cache`. The motion cache is outside the function's mounts. | +| `stage_cache lhm "$DEADLINE_EPOCH"` | LHM cache only. Stages LHM priors, model snapshot and Mask R-CNN in `wander-overnight-lhm-cache`. | +| `stage_cache dino "$DEADLINE_EPOCH"` | LHM cache only. Stages the DINO checkpoint. | +| `stage_cache gfpgan "$DEADLINE_EPOCH"` | LHM cache only. Stages the GFPGAN checkpoint. | +| `stage_cache lhm-resume "$DEADLINE_EPOCH"` | LHM cache only. Range recovery requires a matching failed `lhm` receipt and incomplete extraction. | +| `stage_cache clean-instance "$DEADLINE_EPOCH"` | Cleaning and LHM caches. Copies the checksum-verified Mask R-CNN checkpoint from the LHM cache into the cleaning cache. | +| `stage_cache clean-motion "$DEADLINE_EPOCH"` | Existing `wander-weights` source, cleaning and motion caches. Copies Pi3X into a fresh motion destination, stages SegFormer in both destination caches and stages LaMa in the cleaning cache. | +| `stage_cache clean-motion-repair "$DEADLINE_EPOCH"` | Same mounts as `clean-motion`. Repairs the diagnosed legacy Pi3X symlink failure using its receipt and incomplete copy. | +| `stage_cache clean-motion-repair-v2 "$DEADLINE_EPOCH"` | Same repair checks and mounts, with its own legacy attempt receipt. | + +The bare script path also selects `main` in Modal 1.5.5. Use the local entrypoint to select +mounts. `::stage` is the underlying function and has no static mounts. Only the three combined +cleaning and motion phases look up `wander-weights`, which must already exist. `clean-only` +records `clean-only-attempt.json`, `clean-only-segformer.json` and `clean-only-lama.json` under +the cleaning volume's `setup-receipts/`. Legacy attempt and step receipt paths are preserved. +An incomplete attempt requires inspection and reconciliation. A complete attempt returns its +saved report without downloading again, though a new CLI invocation can still launch a container. + +Staging is outside the paid-stage ledger. The function requests one CPU and 1 GiB of memory, +with limits of two CPUs and 4 GiB, one container per function configuration, zero function retries +and a two-second scaledown window. Modal's execution timeout is 1,800 seconds. On function entry, +the worker uses the smaller of 1,750 seconds and the remaining time to `--deadline-epoch`, +rejecting an exhausted deadline. Queueing and startup can consume the absolute time budget before +the worker enters. Neither timeout bounds total CLI wall time or spend. The receipt's +`computeUpperEstimateUSD` uses elapsed worker time at the resource limits (about $0.00003508/s). +Review provider charges for startup, build, transfer and storage separately. + Single-person placement now measures an undilated Mask R-CNN instance at every packaged source ordinal, using the cached COCO_V1 checkpoint also used by the default reference-frame preparation. This local pass never downloads weights. It records the exact source, decoder, checkpoint and diff --git a/scripts/tests/test_stage_weights.py b/scripts/tests/test_stage_weights.py index f030162..7fcdbd6 100644 --- a/scripts/tests/test_stage_weights.py +++ b/scripts/tests/test_stage_weights.py @@ -1,16 +1,23 @@ """Offline tests for streamed cache extraction and bounded HTTP range recovery.""" import io +import json import sys import tarfile import tempfile +import time import unittest from pathlib import Path -from unittest.mock import patch +from types import SimpleNamespace +from unittest.mock import Mock, call, patch + +import pytest +from modal._functions import synchronizer sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "worker")) sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "worker" / "scripts")) from modal_stage_weights import extract_missing_priors, extract_priors +import modal_stage_weights as staging def archive_bytes(files): @@ -30,6 +37,190 @@ def __init__(self, body, headers, status=200): self.status = status +@pytest.mark.parametrize( + "phase, mount_paths", + [ + ("clean-only", ("/clean",)), + ("clean-motion", ("/source", "/clean", "/motion")), + ("clean-motion-repair", ("/source", "/clean", "/motion")), + ("clean-motion-repair-v2", ("/source", "/clean", "/motion")), + ("clean-instance", ("/clean", "/lhm")), + ("lhm", ("/lhm",)), + ("lhm-resume", ("/lhm",)), + ("dino", ("/lhm",)), + ("gfpgan", ("/lhm",)), + ], +) +def test_local_entrypoint_uses_real_sdk_variant_with_only_phase_dependencies( + tmp_path, phase, mount_paths +): + base = synchronizer._translate_in(staging.stage) + volume_type = type(synchronizer._translate_in(staging.VOLUMES["/clean"])) + assert not any(isinstance(dep, volume_type) for dep in base._deps_()) + report = {"phase": phase, "status": "complete", "steps": []} + deadline = time.time() + 1200 + out = tmp_path / "receipt.json" + invocations = [] + + def remote(function, selected_phase, deadline_epoch): + variant = synchronizer._translate_in(function) + assert variant._base_function is base + assert tuple(path for path, _ in variant._options.validated_volumes) == mount_paths + assert variant._deps_() == [ + synchronizer._translate_in(staging.VOLUMES[p]) for p in mount_paths + ] + invocations.append((selected_phase, deadline_epoch)) + return report + + with patch.object(type(staging.stage), "remote", remote): + staging.main(phase, deadline, str(out)) + assert invocations == [(phase, deadline)] + assert json.loads(out.read_text()) == report + + +def test_local_entrypoint_rejects_unknown_phase_without_invoking(tmp_path): + out = tmp_path / "receipt.json" + with patch.object(staging, "stage") as function, pytest.raises(ValueError, match="unknown"): + staging.main("unknown", time.time() + 1200, str(out)) + function.with_options.assert_not_called() + function.remote.assert_not_called() + assert not out.exists() + + +@pytest.mark.parametrize( + "phase", ["clean-only", "clean-motion", "clean-motion-repair", "clean-motion-repair-v2"] +) +def test_clean_staging_outputs_and_receipts(tmp_path, phase): + combined = phase != "clean-only" + clean = tmp_path / "clean" + motion = tmp_path / "motion" + clean.mkdir() + motion.mkdir() + receipts = clean / "setup-receipts" + receipts.mkdir() + pi = motion / "huggingface/hub/models--yyfz233--Pi3X" + segformer_name = "models--nvidia--segformer-b0-finetuned-ade-512-512" + segformer = clean / "huggingface/hub" / segformer_name + snapshot = segformer / "snapshots" / staging.SEGFORMER_REV + + if combined: + source = tmp_path / "source/hub/models--yyfz233--Pi3X/blobs" + source.mkdir(parents=True) + if phase == "clean-motion": + (source / "blob").write_bytes(b"pi3x") + else: + (source.parent.parent / "shared").write_bytes(b"pi3x") + (source / "blob").symlink_to("../../shared") + pending = pi.with_name(pi.name + ".incomplete") / "blobs" + pending.mkdir(parents=True) + (pending / "blob").symlink_to("../../shared") + (receipts / "clean-motion-attempt.json").write_text( + json.dumps( + { + "status": "failed", + "error": "cache has a broken or escaping symlink", + "steps": [], + } + ) + ) + else: + (pi / "blobs").mkdir(parents=True) + (pi / "blobs/blob").write_bytes(b"existing inference cache") + (receipts / "segformer.json").write_text("legacy receipt") + motion_before = sorted(motion.rglob("*")) + + def container_path(value): + path = Path(value) + if path.is_absolute() and path.parts[1] in {"source", "clean", "motion", "lhm"}: + if not combined: + assert path.parts[1] == "clean", f"clean-only accessed {path}" + return tmp_path.joinpath(*path.parts[1:]) + return path + + def download_snapshot(*args, **kwargs): + snapshot.mkdir(parents=True) + (snapshot / "model.safetensors").write_bytes(b"segformer") + return str(snapshot) + + download = Mock(side_effect=download_snapshot) + volumes = {mount: Mock() for mount in staging.VOLUMES} + raw_stage = synchronizer._translate_in(staging.stage)._raw_f + with ( + patch.object(staging, "Path", side_effect=container_path), + patch.object(staging, "VOLUMES", volumes), + patch.object(staging.modal, "current_function_call_id", return_value="fc-offline"), + patch.dict(sys.modules, {"huggingface_hub": SimpleNamespace(snapshot_download=download)}), + patch("signal.signal"), + patch("signal.alarm") as alarm, + patch( + "urllib.request.urlopen", return_value=Response(b"lama", {"Content-Length": "4"}) + ) as fetch, + ): + deadline = time.time() + 1800 + report = raw_stage(phase, deadline) + assert raw_stage(phase, deadline) == report + + download.assert_called_once_with( + "nvidia/segformer-b0-finetuned-ade-512-512", + revision=staging.SEGFORMER_REV, + cache_dir="/clean/huggingface/hub", + allow_patterns=["config.json", "preprocessor_config.json", "model.safetensors"], + max_workers=1, + ) + fetch.assert_called_once_with(staging.LAMA_URL, timeout=45) + assert alarm.call_args_list == [call(1750), call(0)] + assert report["status"] == "complete" + assert report["functionCallId"] == "fc-offline" + assert json.loads((receipts / f"{phase}-attempt.json").read_text()) == report + assert [step["name"] for step in report["steps"]] == ( + ["pi3x", "segformer", "lama"] if combined else ["segformer", "lama"] + ) + assert (clean / "lama/big-lama.pt").read_bytes() == b"lama" + assert (segformer / "refs/main").read_text() == staging.SEGFORMER_REV + assert volumes["/clean"].commit.call_count == len(report["steps"]) + 2 + volumes["/source"].commit.assert_not_called() + volumes["/lhm"].commit.assert_not_called() + if combined: + assert (pi / "blobs/blob").read_bytes() == b"pi3x" + assert (motion / "huggingface/hub" / segformer_name / "refs/main").is_file() + assert json.loads((receipts / "pi3x.json").read_text())["sourceVolume"] == "wander-weights" + assert (receipts / "segformer.json").is_file() + assert (receipts / "lama.json").is_file() + assert volumes["/motion"].commit.call_count == volumes["/clean"].commit.call_count + if phase != "clean-motion": + assert [repair["blob"] for repair in report["repairs"]] == ["blob"] + else: + assert (pi / "blobs/blob").read_bytes() == b"existing inference cache" + assert sorted(motion.rglob("*")) == motion_before + assert (receipts / "segformer.json").read_text() == "legacy receipt" + assert (receipts / "clean-only-segformer.json").is_file() + assert (receipts / "clean-only-lama.json").is_file() + volumes["/motion"].commit.assert_not_called() + + +def test_clean_only_refuses_an_incomplete_attempt_before_downloading(tmp_path): + receipt = tmp_path / "clean/setup-receipts/clean-only-attempt.json" + receipt.parent.mkdir(parents=True) + previous = '{"status": "failed", "steps": []}' + receipt.write_text(previous) + volumes = {"/clean": Mock()} + download = Mock() + raw_stage = synchronizer._translate_in(staging.stage)._raw_f + with ( + patch.object(staging, "Path", side_effect=lambda path: tmp_path / path.lstrip("/")), + patch.object(staging, "VOLUMES", volumes), + patch.object(staging.modal, "current_function_call_id", return_value="fc-offline"), + patch.dict(sys.modules, {"huggingface_hub": SimpleNamespace(snapshot_download=download)}), + patch("urllib.request.urlopen") as fetch, + pytest.raises(RuntimeError, match="prior incomplete attempt"), + ): + raw_stage("clean-only", time.time() + 1200) + download.assert_not_called() + fetch.assert_not_called() + volumes["/clean"].commit.assert_not_called() + assert receipt.read_text() == previous + + class StageWeightsTest(unittest.TestCase): def setUp(self): self.temporary = tempfile.TemporaryDirectory() diff --git a/worker/modal_stage_weights.py b/worker/modal_stage_weights.py index 56e4864..9c6047b 100644 --- a/worker/modal_stage_weights.py +++ b/worker/modal_stage_weights.py @@ -1,7 +1,8 @@ """Explicit CPU-only model cache staging; never performs inference or starts GPU workers. -Run with MODAL_PROFILE=dtpu and an absolute deadline. Each phase runs once, preserves a failure -receipt, and refuses to restart an incomplete copy/extraction without operator reconciliation. +Run the local entrypoint with the intended Modal profile and an absolute deadline. Each phase +runs once, preserves a failure receipt, and refuses to restart an incomplete copy/extraction +without operator reconciliation. Public upstream URLs/revisions are recorded in receipts; source video is never uploaded. """ @@ -23,10 +24,19 @@ image = modal.Image.debian_slim(python_version="3.12").uv_pip_install("huggingface_hub==0.28.1") app = modal.App("wander-daniel-overnight-cache-setup") -source = modal.Volume.from_name("wander-weights") -clean_cache = modal.Volume.from_name("wander-clean-video-cache", create_if_missing=True) -motion_cache = modal.Volume.from_name("wander-overnight-motion-cache", create_if_missing=True) -lhm_cache = modal.Volume.from_name("wander-overnight-lhm-cache", create_if_missing=True) +VOLUMES = { + "/source": modal.Volume.from_name("wander-weights"), + "/clean": modal.Volume.from_name("wander-clean-video-cache", create_if_missing=True), + "/motion": modal.Volume.from_name("wander-overnight-motion-cache", create_if_missing=True), + "/lhm": modal.Volume.from_name("wander-overnight-lhm-cache", create_if_missing=True), +} +CLEAN_MOTION_PHASES = ("clean-motion", "clean-motion-repair", "clean-motion-repair-v2") +PHASE_MOUNTS = { + **dict.fromkeys(CLEAN_MOTION_PHASES, ("/source", "/clean", "/motion")), + "clean-only": ("/clean",), + "clean-instance": ("/clean", "/lhm"), + **dict.fromkeys(("lhm", "lhm-resume", "dino", "gfpgan"), ("/lhm",)), +} def digest_file(path): @@ -218,6 +228,7 @@ def read(self, size): } +# Modal hydrates static mounts before main executes. Select mounts only on the invoked variant. @app.function( image=image, cpu=(1, 2), @@ -226,7 +237,6 @@ def read(self, size): retries=0, max_containers=1, scaledown_window=2, - volumes={"/source": source, "/clean": clean_cache, "/motion": motion_cache, "/lhm": lhm_cache}, ) def stage(phase: str, deadline_epoch: float): import json @@ -237,17 +247,10 @@ def stage(phase: str, deadline_epoch: float): from huggingface_hub import snapshot_download - if phase not in ( - "clean-motion", - "clean-motion-repair", - "clean-motion-repair-v2", - "lhm", - "lhm-resume", - "dino", - "gfpgan", - "clean-instance", - ): + if phase not in PHASE_MOUNTS: raise ValueError("unknown staging phase") + if not all(Path(mount).is_dir() for mount in PHASE_MOUNTS[phase]): + raise RuntimeError("staging phase mounts missing; invoke the main local entrypoint") remaining = min(1750, int(deadline_epoch - time.time())) if remaining <= 0: raise ValueError("setup deadline already exhausted") @@ -255,8 +258,6 @@ def stage(phase: str, deadline_epoch: float): def deadline(*_): raise TimeoutError("cache staging deadline exhausted") - signal.signal(signal.SIGALRM, deadline) - signal.alarm(remaining) started = time.time() report = { "phase": phase, @@ -268,7 +269,8 @@ def deadline(*_): "functionCallId": modal.current_function_call_id(), "steps": [], } - receipt_root = Path("/clean" if phase.startswith("clean-") else "/lhm") / "setup-receipts" + receipt_mount = "/clean" if phase.startswith("clean-") else "/lhm" + receipt_root = Path(receipt_mount) / "setup-receipts" failure = receipt_root / f"{phase}-attempt.json" if failure.exists(): previous = json.loads(failure.read_text()) @@ -277,22 +279,27 @@ def deadline(*_): "prior incomplete attempt requires reconciliation; no automatic retry" ) return previous + + def commit_changes(): + VOLUMES[receipt_mount].commit() + if phase in CLEAN_MOTION_PHASES: + VOLUMES["/motion"].commit() + + signal.signal(signal.SIGALRM, deadline) + signal.alarm(remaining) write_receipt(failure, report) - clean_cache.commit() - motion_cache.commit() - lhm_cache.commit() + commit_changes() def step_done(name, document): report["steps"].append({"name": name, **document}) - write_receipt(receipt_root / f"{name}.json", document) + receipt_name = f"clean-only-{name}" if phase == "clean-only" else name + write_receipt(receipt_root / f"{receipt_name}.json", document) write_receipt(failure, report) - clean_cache.commit() - motion_cache.commit() - lhm_cache.commit() + commit_changes() print(f"cache step complete: {name}", flush=True) try: - if phase.startswith("clean-motion"): + if phase in CLEAN_MOTION_PHASES: pi_source = Path("/source/hub/models--yyfz233--Pi3X") pi_dest = Path("/motion/huggingface/hub/models--yyfz233--Pi3X") pending = pi_dest.with_name(pi_dest.name + ".incomplete") @@ -349,6 +356,7 @@ def step_done(name, document): }, ) + if phase == "clean-only" or phase in CLEAN_MOTION_PHASES: snapshot = snapshot_download( "nvidia/segformer-b0-finetuned-ade-512-512", revision=SEGFORMER_REV, @@ -360,9 +368,10 @@ def step_done(name, document): # The active workers resolve main; pin its local ref to the staged immutable revision. (model_dir / "refs").mkdir(exist_ok=True) (model_dir / "refs/main").write_text(SEGFORMER_REV) - shutil.copytree( - model_dir, Path("/motion/huggingface/hub") / model_dir.name, symlinks=True - ) + if phase in CLEAN_MOTION_PHASES: + shutil.copytree( + model_dir, Path("/motion/huggingface/hub") / model_dir.name, symlinks=True + ) step_done( "segformer", { @@ -536,9 +545,7 @@ def step_done(name, document): report["elapsedSeconds"] = time.time() - started report["computeUpperEstimateUSD"] = report["elapsedSeconds"] * RATE_CEILING write_receipt(failure, report) - clean_cache.commit() - motion_cache.commit() - lhm_cache.commit() + commit_changes() return report @@ -546,7 +553,10 @@ def step_done(name, document): def main(phase: str, deadline_epoch: float, out: str): import json - report = stage.remote(phase, deadline_epoch) + if phase not in PHASE_MOUNTS: + raise ValueError("unknown staging phase") + volumes = {mount: VOLUMES[mount] for mount in PHASE_MOUNTS[phase]} + report = stage.with_options(volumes=volumes).remote(phase, deadline_epoch) path = Path(out) path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(report, indent=2))