Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions docs/pipeline-setup.md
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,51 @@ staging receipts and [paid recovery](paid-recovery.md) before deliberately stagi
The doctor does not initiate staging or repair. Local reference-frame models may still need
their separately reviewed Torch/Hugging Face cache preparation before processing a source.

### Stage missing Modal model caches

These commands start one paid CPU staging call per invocation. Select the saved local profile
alias for the intended workspace and use a fresh absolute Unix deadline in seconds and a unique
local receipt path for each authorized phase. Run phases serially. Set `STAGING_PROFILE` and
`STAGING_RECEIPT_DIR`, and set `DEADLINE_EPOCH` immediately before each call. For example,
`DEADLINE_EPOCH=$(( $(date +%s) + 1800 ))` allows 30 minutes from launch. Defining the function
below is local and does not start staging.

```sh
stage_cache() {
uv run --locked modal --profile "$STAGING_PROFILE" run worker/modal_stage_weights.py::main \
--phase "$1" --deadline-epoch "$2" --out "$STAGING_RECEIPT_DIR/$1.json"
}
```

| Phase command | Volume mounts and effect |
| --- | --- |
| `stage_cache clean-only "$DEADLINE_EPOCH"` | Cleaning cache only. Stages LaMa and pinned SegFormer in `wander-clean-video-cache`. The motion cache is outside the function's mounts. |
| `stage_cache lhm "$DEADLINE_EPOCH"` | LHM cache only. Stages LHM priors, model snapshot and Mask R-CNN in `wander-overnight-lhm-cache`. |
| `stage_cache dino "$DEADLINE_EPOCH"` | LHM cache only. Stages the DINO checkpoint. |
| `stage_cache gfpgan "$DEADLINE_EPOCH"` | LHM cache only. Stages the GFPGAN checkpoint. |
| `stage_cache lhm-resume "$DEADLINE_EPOCH"` | LHM cache only. Range recovery requires a matching failed `lhm` receipt and incomplete extraction. |
| `stage_cache clean-instance "$DEADLINE_EPOCH"` | Cleaning and LHM caches. Copies the checksum-verified Mask R-CNN checkpoint from the LHM cache into the cleaning cache. |
| `stage_cache clean-motion "$DEADLINE_EPOCH"` | Existing `wander-weights` source, cleaning and motion caches. Copies Pi3X into a fresh motion destination, stages SegFormer in both destination caches and stages LaMa in the cleaning cache. |
| `stage_cache clean-motion-repair "$DEADLINE_EPOCH"` | Same mounts as `clean-motion`. Repairs the diagnosed legacy Pi3X symlink failure using its receipt and incomplete copy. |
| `stage_cache clean-motion-repair-v2 "$DEADLINE_EPOCH"` | Same repair checks and mounts, with its own legacy attempt receipt. |

The bare script path also selects `main` in Modal 1.5.5. Use the local entrypoint to select
mounts. `::stage` is the underlying function and has no static mounts. Only the three combined
cleaning and motion phases look up `wander-weights`, which must already exist. `clean-only`
records `clean-only-attempt.json`, `clean-only-segformer.json` and `clean-only-lama.json` under
the cleaning volume's `setup-receipts/`. Legacy attempt and step receipt paths are preserved.
An incomplete attempt requires inspection and reconciliation. A complete attempt returns its
saved report without downloading again, though a new CLI invocation can still launch a container.

Staging is outside the paid-stage ledger. The function requests one CPU and 1 GiB of memory,
with limits of two CPUs and 4 GiB, one container per function configuration, zero function retries
and a two-second scaledown window. Modal's execution timeout is 1,800 seconds. On function entry,
the worker uses the smaller of 1,750 seconds and the remaining time to `--deadline-epoch`,
rejecting an exhausted deadline. Queueing and startup can consume the absolute time budget before
the worker enters. Neither timeout bounds total CLI wall time or spend. The receipt's
`computeUpperEstimateUSD` uses elapsed worker time at the resource limits (about $0.00003508/s).
Review provider charges for startup, build, transfer and storage separately.

Single-person placement now measures an undilated Mask R-CNN instance at every packaged source
ordinal, using the cached COCO_V1 checkpoint also used by the default reference-frame preparation.
This local pass never downloads weights. It records the exact source, decoder, checkpoint and
Expand Down
193 changes: 192 additions & 1 deletion scripts/tests/test_stage_weights.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,23 @@
"""Offline tests for streamed cache extraction and bounded HTTP range recovery."""

import io
import json
import sys
import tarfile
import tempfile
import time
import unittest
from pathlib import Path
from unittest.mock import patch
from types import SimpleNamespace
from unittest.mock import Mock, call, patch

import pytest
from modal._functions import synchronizer

sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "worker"))
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "worker" / "scripts"))
from modal_stage_weights import extract_missing_priors, extract_priors
import modal_stage_weights as staging


def archive_bytes(files):
Expand All @@ -30,6 +37,190 @@ def __init__(self, body, headers, status=200):
self.status = status


@pytest.mark.parametrize(
"phase, mount_paths",
[
("clean-only", ("/clean",)),
("clean-motion", ("/source", "/clean", "/motion")),
("clean-motion-repair", ("/source", "/clean", "/motion")),
("clean-motion-repair-v2", ("/source", "/clean", "/motion")),
("clean-instance", ("/clean", "/lhm")),
("lhm", ("/lhm",)),
("lhm-resume", ("/lhm",)),
("dino", ("/lhm",)),
("gfpgan", ("/lhm",)),
],
)
def test_local_entrypoint_uses_real_sdk_variant_with_only_phase_dependencies(
tmp_path, phase, mount_paths
):
base = synchronizer._translate_in(staging.stage)
volume_type = type(synchronizer._translate_in(staging.VOLUMES["/clean"]))
assert not any(isinstance(dep, volume_type) for dep in base._deps_())
report = {"phase": phase, "status": "complete", "steps": []}
deadline = time.time() + 1200
out = tmp_path / "receipt.json"
invocations = []

def remote(function, selected_phase, deadline_epoch):
variant = synchronizer._translate_in(function)
assert variant._base_function is base
assert tuple(path for path, _ in variant._options.validated_volumes) == mount_paths
assert variant._deps_() == [
synchronizer._translate_in(staging.VOLUMES[p]) for p in mount_paths
]
invocations.append((selected_phase, deadline_epoch))
return report

with patch.object(type(staging.stage), "remote", remote):
staging.main(phase, deadline, str(out))
assert invocations == [(phase, deadline)]
assert json.loads(out.read_text()) == report


def test_local_entrypoint_rejects_unknown_phase_without_invoking(tmp_path):
out = tmp_path / "receipt.json"
with patch.object(staging, "stage") as function, pytest.raises(ValueError, match="unknown"):
staging.main("unknown", time.time() + 1200, str(out))
function.with_options.assert_not_called()
function.remote.assert_not_called()
assert not out.exists()


@pytest.mark.parametrize(
"phase", ["clean-only", "clean-motion", "clean-motion-repair", "clean-motion-repair-v2"]
)
def test_clean_staging_outputs_and_receipts(tmp_path, phase):
combined = phase != "clean-only"
clean = tmp_path / "clean"
motion = tmp_path / "motion"
clean.mkdir()
motion.mkdir()
receipts = clean / "setup-receipts"
receipts.mkdir()
pi = motion / "huggingface/hub/models--yyfz233--Pi3X"
segformer_name = "models--nvidia--segformer-b0-finetuned-ade-512-512"
segformer = clean / "huggingface/hub" / segformer_name
snapshot = segformer / "snapshots" / staging.SEGFORMER_REV

if combined:
source = tmp_path / "source/hub/models--yyfz233--Pi3X/blobs"
source.mkdir(parents=True)
if phase == "clean-motion":
(source / "blob").write_bytes(b"pi3x")
else:
(source.parent.parent / "shared").write_bytes(b"pi3x")
(source / "blob").symlink_to("../../shared")
pending = pi.with_name(pi.name + ".incomplete") / "blobs"
pending.mkdir(parents=True)
(pending / "blob").symlink_to("../../shared")
(receipts / "clean-motion-attempt.json").write_text(
json.dumps(
{
"status": "failed",
"error": "cache has a broken or escaping symlink",
"steps": [],
}
)
)
else:
(pi / "blobs").mkdir(parents=True)
(pi / "blobs/blob").write_bytes(b"existing inference cache")
(receipts / "segformer.json").write_text("legacy receipt")
motion_before = sorted(motion.rglob("*"))

def container_path(value):
path = Path(value)
if path.is_absolute() and path.parts[1] in {"source", "clean", "motion", "lhm"}:
if not combined:
assert path.parts[1] == "clean", f"clean-only accessed {path}"
return tmp_path.joinpath(*path.parts[1:])
return path

def download_snapshot(*args, **kwargs):
snapshot.mkdir(parents=True)
(snapshot / "model.safetensors").write_bytes(b"segformer")
return str(snapshot)

download = Mock(side_effect=download_snapshot)
volumes = {mount: Mock() for mount in staging.VOLUMES}
raw_stage = synchronizer._translate_in(staging.stage)._raw_f
with (
patch.object(staging, "Path", side_effect=container_path),
patch.object(staging, "VOLUMES", volumes),
patch.object(staging.modal, "current_function_call_id", return_value="fc-offline"),
patch.dict(sys.modules, {"huggingface_hub": SimpleNamespace(snapshot_download=download)}),
patch("signal.signal"),
patch("signal.alarm") as alarm,
patch(
"urllib.request.urlopen", return_value=Response(b"lama", {"Content-Length": "4"})
) as fetch,
):
deadline = time.time() + 1800
report = raw_stage(phase, deadline)
assert raw_stage(phase, deadline) == report

download.assert_called_once_with(
"nvidia/segformer-b0-finetuned-ade-512-512",
revision=staging.SEGFORMER_REV,
cache_dir="/clean/huggingface/hub",
allow_patterns=["config.json", "preprocessor_config.json", "model.safetensors"],
max_workers=1,
)
fetch.assert_called_once_with(staging.LAMA_URL, timeout=45)
assert alarm.call_args_list == [call(1750), call(0)]
assert report["status"] == "complete"
assert report["functionCallId"] == "fc-offline"
assert json.loads((receipts / f"{phase}-attempt.json").read_text()) == report
assert [step["name"] for step in report["steps"]] == (
["pi3x", "segformer", "lama"] if combined else ["segformer", "lama"]
)
assert (clean / "lama/big-lama.pt").read_bytes() == b"lama"
assert (segformer / "refs/main").read_text() == staging.SEGFORMER_REV
assert volumes["/clean"].commit.call_count == len(report["steps"]) + 2
volumes["/source"].commit.assert_not_called()
volumes["/lhm"].commit.assert_not_called()
if combined:
assert (pi / "blobs/blob").read_bytes() == b"pi3x"
assert (motion / "huggingface/hub" / segformer_name / "refs/main").is_file()
assert json.loads((receipts / "pi3x.json").read_text())["sourceVolume"] == "wander-weights"
assert (receipts / "segformer.json").is_file()
assert (receipts / "lama.json").is_file()
assert volumes["/motion"].commit.call_count == volumes["/clean"].commit.call_count
if phase != "clean-motion":
assert [repair["blob"] for repair in report["repairs"]] == ["blob"]
else:
assert (pi / "blobs/blob").read_bytes() == b"existing inference cache"
assert sorted(motion.rglob("*")) == motion_before
assert (receipts / "segformer.json").read_text() == "legacy receipt"
assert (receipts / "clean-only-segformer.json").is_file()
assert (receipts / "clean-only-lama.json").is_file()
volumes["/motion"].commit.assert_not_called()


def test_clean_only_refuses_an_incomplete_attempt_before_downloading(tmp_path):
receipt = tmp_path / "clean/setup-receipts/clean-only-attempt.json"
receipt.parent.mkdir(parents=True)
previous = '{"status": "failed", "steps": []}'
receipt.write_text(previous)
volumes = {"/clean": Mock()}
download = Mock()
raw_stage = synchronizer._translate_in(staging.stage)._raw_f
with (
patch.object(staging, "Path", side_effect=lambda path: tmp_path / path.lstrip("/")),
patch.object(staging, "VOLUMES", volumes),
patch.object(staging.modal, "current_function_call_id", return_value="fc-offline"),
patch.dict(sys.modules, {"huggingface_hub": SimpleNamespace(snapshot_download=download)}),
patch("urllib.request.urlopen") as fetch,
pytest.raises(RuntimeError, match="prior incomplete attempt"),
):
raw_stage("clean-only", time.time() + 1200)
download.assert_not_called()
fetch.assert_not_called()
volumes["/clean"].commit.assert_not_called()
assert receipt.read_text() == previous


class StageWeightsTest(unittest.TestCase):
def setUp(self):
self.temporary = tempfile.TemporaryDirectory()
Expand Down
Loading
Loading