From ee750636952497bcb6c6cd07d97ba3420f486176 Mon Sep 17 00:00:00 2001 From: Austin Jian Date: Fri, 20 Nov 2026 16:13:02 -0500 Subject: [PATCH] fix: add a hash-bound final LHM cache recovery slot --- docs/pipeline-setup.md | 29 ++++ scripts/tests/test_stage_weights.py | 225 ++++++++++++++++++++++++++++ worker/modal_stage_weights.py | 130 +++++++++++----- 3 files changed, 350 insertions(+), 34 deletions(-) diff --git a/docs/pipeline-setup.md b/docs/pipeline-setup.md index 2248c8b..1ff1beb 100644 --- a/docs/pipeline-setup.md +++ b/docs/pipeline-setup.md @@ -184,6 +184,35 @@ fetched during resume, including headers. Any partial `.git` files left by an ol on the volume and are omitted from the final runtime inventory. Existing receipt and recovery checks still apply. +#### Reconcile a failed LHM range recovery + +The final recovery slot is `lhm-resume-reconciled`. Before launching it, verify that the prior +Modal app and call have terminated. Compute `PRIOR_RESUME_SHA256` over the exact saved bytes of +the volume's `lhm-resume-attempt.json`, including whitespace. This phase requires a failed +`lhm-resume` receipt with a call ID, `TimeoutError`, the exact message +`cache staging deadline exhausted`, and an empty step list. The original `lhm` failure guard +still applies, and `data.incomplete` must exist with `data` absent. + +Use one authorized launcher. Before invoking Modal, exclusively create and durably persist a +fixed local launch claim for this workspace, environment and LHM volume. Use exclusive-create +mode (`open(..., "x")`) and retain the claim even if launch or completion is uncertain. An +existing claim blocks another launch. The Modal volume receipt alone cannot enforce an atomic +cap across independent launchers. + +```sh +uv run --locked modal --profile "$STAGING_PROFILE" run worker/modal_stage_weights.py::main \ + --phase lhm-resume-reconciled --deadline-epoch "$DEADLINE_EPOCH" \ + --prior-resume-sha256 "$PRIOR_RESUME_SHA256" \ + --out "$STAGING_RECEIPT_DIR/lhm-resume-2.json" +``` + +The worker commits `setup-receipts/lhm-resume-2-attempt.json` before reading archive ranges. +It records both call IDs, the prior receipt hash, `totalAttempt: 3` and `rangeAttempt: 2`. +The original receipts and their counters are preserved. Any existing final receipt consumes the +slot, including a partial or completed receipt. Inspect it directly after a lost response. This +path allows one initial staging attempt and at most two range recoveries. It uses the same +Git-metadata exclusion and resource limits as ordinary range recovery. + Staging is outside the paid-stage ledger. The function requests one CPU and 1 GiB of memory, with limits of two CPUs and 4 GiB, one container per function configuration, zero function retries and a two-second scaledown window. Modal's execution timeout is 1,800 seconds. On function entry, diff --git a/scripts/tests/test_stage_weights.py b/scripts/tests/test_stage_weights.py index 70573b6..029fc25 100644 --- a/scripts/tests/test_stage_weights.py +++ b/scripts/tests/test_stage_weights.py @@ -1,5 +1,6 @@ """Offline tests for streamed cache extraction and bounded HTTP range recovery.""" +import hashlib import io import json import sys @@ -87,6 +88,40 @@ def test_local_entrypoint_rejects_unknown_phase_without_invoking(tmp_path): assert not out.exists() +@pytest.mark.parametrize( + "phase, supplied_hash", + [ + ("lhm-resume-reconciled", None), + ("lhm-resume-reconciled", ""), + ("lhm-resume-reconciled", "a" * 63), + ("lhm-resume-reconciled", "z" * 64), + ("lhm-resume", "a" * 64), + ("clean-only", ""), + ], +) +def test_reconciliation_hash_args_rejected_at_both_entrypoints(tmp_path, phase, supplied_hash): + raw_stage = synchronizer._translate_in(staging.stage)._raw_f + deadline = time.time() + 1200 + with patch.object(staging, "stage") as function: + with pytest.raises(ValueError, match="--prior-resume-sha256"): + staging.main(phase, deadline, str(tmp_path / "receipt.json"), supplied_hash) + function.with_options.assert_not_called() + with pytest.raises(ValueError, match="--prior-resume-sha256"): + raw_stage(phase, deadline, supplied_hash) + + +def test_reconciled_local_entrypoint_invokes_one_lhm_variant(tmp_path): + deadline = time.time() + 1200 + out = tmp_path / "receipt.json" + with patch.object(staging, "stage") as function: + remote = function.with_options.return_value.remote + remote.return_value = {"status": "complete", "steps": []} + staging.main("lhm-resume-reconciled", deadline, str(out), "A" * 64) + function.with_options.assert_called_once_with(volumes={"/lhm": staging.VOLUMES["/lhm"]}) + remote.assert_called_once_with("lhm-resume-reconciled", deadline, prior_resume_sha256="a" * 64) + assert json.loads(out.read_text()) == remote.return_value + + @pytest.mark.parametrize( "phase", ["clean-only", "clean-motion", "clean-motion-repair", "clean-motion-repair-v2"] ) @@ -245,6 +280,196 @@ def test_lhm_resume_preserves_legacy_attempt_receipt(tmp_path, status): assert receipt.read_text() == original +@pytest.fixture +def reconciliation(tmp_path, monkeypatch): + lhm = tmp_path / "lhm" + pending = lhm / "data.incomplete" + pending.mkdir(parents=True) + receipts = lhm / "setup-receipts" + receipts.mkdir() + initial = receipts / "lhm-attempt.json" + prior = receipts / "lhm-resume-attempt.json" + final = receipts / "lhm-resume-2-attempt.json" + old = { + "phase": "lhm-resume", + "functionCallId": "fc-" + "A" * 26, + "status": "failed", + "errorType": "TimeoutError", + "error": "cache staging deadline exhausted", + "steps": [], + "rangeBytesReceived": 123, + "elapsedSeconds": 1750, + } + initial.write_bytes(json.dumps(old | {"phase": "lhm"}, indent=4).encode() + b"\n") + prior.write_bytes(json.dumps(old, indent=4).encode() + b"\n") + volume = Mock() + snapshot = Mock(side_effect=OSError("offline stop after prior staging")) + call_id = "fc-" + "B" * 26 + raw_stage = synchronizer._translate_in(staging.stage)._raw_f + + def container_path(path): + path = Path(path) + return tmp_path.joinpath(*path.parts[1:]) if path.parts[:2] == ("/", "lhm") else path + + monkeypatch.setattr(staging, "Path", container_path) + monkeypatch.setattr(staging, "VOLUMES", {"/lhm": volume}) + monkeypatch.setattr(staging.modal, "current_function_call_id", lambda: call_id) + monkeypatch.setitem(sys.modules, "huggingface_hub", SimpleNamespace(snapshot_download=snapshot)) + monkeypatch.setattr("signal.signal", Mock()) + monkeypatch.setattr("signal.alarm", Mock()) + + def run(prior_hash=None): + digest = ( + prior_hash if prior_hash is not None else hashlib.sha256(prior.read_bytes()).hexdigest() + ) + return raw_stage("lhm-resume-reconciled", time.time() + 1200, digest) + + return SimpleNamespace( + lhm=lhm, + pending=pending, + initial=initial, + prior=prior, + final=final, + old=old, + volume=volume, + snapshot=snapshot, + call_id=call_id, + run=run, + ) + + +@pytest.mark.parametrize( + "changes", + [ + {"phase": "lhm"}, + {"functionCallId": None}, + {"functionCallId": "ap-" + "A" * 26}, + {"status": "started"}, + {"status": "complete"}, + {"errorType": "ValueError"}, + {"error": "cache staging deadline exhausted "}, + {"steps": [{"name": "lhm-priors"}]}, + {"steps": None}, + ], +) +def test_reconciliation_requires_the_exact_terminal_receipt(reconciliation, changes): + state = reconciliation + state.prior.write_text(json.dumps(state.old | changes)) + with patch.object(staging, "extract_missing_priors") as extract: + with pytest.raises(RuntimeError, match="failed lhm-resume deadline receipt"): + state.run() + extract.assert_not_called() + state.volume.commit.assert_not_called() + assert not state.final.exists() + + +@pytest.mark.parametrize( + "mismatch", ["hash", "initial-error", "initial-steps", "pending", "complete"] +) +def test_reconciliation_requires_reviewed_hash_and_eligible_cache_state(reconciliation, mismatch): + state = reconciliation + digest = None + if mismatch == "hash": + digest = hashlib.sha256(state.prior.read_bytes()).hexdigest() + with state.prior.open("ab") as stream: + stream.write(b" ") + elif mismatch == "initial-error": + state.initial.write_text(json.dumps(state.old | {"error": "other error"})) + elif mismatch == "initial-steps": + state.initial.write_text(json.dumps(state.old | {"steps": ["lhm-priors"]})) + elif mismatch == "pending": + state.pending.rmdir() + else: + (state.lhm / "data").mkdir() + with patch.object(staging, "extract_missing_priors") as extract: + with pytest.raises(RuntimeError, match="SHA-256 mismatch|diagnosed incomplete extraction"): + state.run(digest) + extract.assert_not_called() + state.volume.commit.assert_not_called() + assert not state.final.exists() + + +@pytest.mark.parametrize( + "contents", [b"", b'{"status":"started"}', b'{"status":"failed"}', b'{"status":"complete"}'] +) +def test_final_range_slot_blocks_every_replay(reconciliation, contents): + state = reconciliation + state.final.write_bytes(contents) + with patch.object(staging, "extract_missing_priors") as extract: + with pytest.raises(RuntimeError, match="slot already consumed"): + state.run() + extract.assert_not_called() + state.volume.commit.assert_not_called() + assert state.final.read_bytes() == contents + + +def test_reconciled_resume_persists_ownership_before_range_work(reconciliation, monkeypatch): + state = reconciliation + prior_path = "pretrained_models/gagatracker/model.pt2" + git_path = "pretrained_models/gagatracker/.git/lfs/objects/blob" + data = archive_bytes( + { + prior_path: b"runtime weights", + git_path: b"x" * (1024 * 1024), + "pretrained_models/human_model_files/a": b"a", + "pretrained_models/dense_sample_points/b": b"b", + "pretrained_models/voxel_grid/c": b"c", + } + ) + with tarfile.open(fileobj=io.BytesIO(data), mode="r:") as archive: + skipped = archive.getmember(git_path) + initial_bytes = state.initial.read_bytes() + prior_bytes = state.prior.read_bytes() + digest = hashlib.sha256(prior_bytes).hexdigest() + requests = [] + seen_at_head = [] + + def serve(request, timeout): + assert request.full_url == staging.PRIOR_URL + if request.get_method() == "HEAD": + state.volume.commit.assert_called_once_with() + seen_at_head.append(json.loads(state.final.read_text())) + return Response(b"", {"ETag": '"pinned"', "Content-Length": str(len(data))}) + assert request.get_header("If-match") == '"pinned"' + first, last = map(int, request.get_header("Range").removeprefix("bytes=").split("-")) + requests.append((first, last)) + return Response( + data[first : last + 1], + { + "ETag": '"pinned"', + "Content-Range": f"bytes {first}-{last}/{len(data)}", + }, + 206, + ) + + monkeypatch.setattr(staging, "PRIOR_BYTES", len(data)) + with patch("urllib.request.urlopen", side_effect=serve): + report = state.run(digest) + with pytest.raises(RuntimeError, match="slot already consumed"): + state.run(digest) + assert len(seen_at_head) == 1 + started = seen_at_head[0] + assert started["status"] == "started" and started["steps"] == [] + assert started["functionCallId"] == state.call_id + assert started["priorResumeFunctionCallId"] == state.old["functionCallId"] + assert started["priorResumeReceiptSha256"] == digest + assert (started["totalAttempt"], started["rangeAttempt"]) == (3, 2) + assert requests and not any( + first < skipped.offset_data + skipped.size and last >= skipped.offset_data + for first, last in requests + ) + assert (state.lhm / "data" / prior_path).read_bytes() == b"runtime weights" + assert not (state.lhm / "data" / git_path).exists() + assert report["steps"][0]["skippedGitBytes"] == skipped.size + assert report["steps"][0]["rangeBytesReceived"] == sum( + last - first + 1 for first, last in requests + ) + assert state.initial.read_bytes() == initial_bytes + assert state.prior.read_bytes() == prior_bytes + assert json.loads(state.final.read_text()) == report + state.snapshot.assert_called_once() + + class StageWeightsTest(unittest.TestCase): def setUp(self): self.temporary = tempfile.TemporaryDirectory() diff --git a/worker/modal_stage_weights.py b/worker/modal_stage_weights.py index 0ae1c1d..5938505 100644 --- a/worker/modal_stage_weights.py +++ b/worker/modal_stage_weights.py @@ -6,6 +6,9 @@ Public upstream URLs/revisions are recorded in receipts; source video is never uploaded. """ +import hashlib +import json +import re from pathlib import Path import modal @@ -31,17 +34,17 @@ "/lhm": modal.Volume.from_name("wander-overnight-lhm-cache", create_if_missing=True), } CLEAN_MOTION_PHASES = ("clean-motion", "clean-motion-repair", "clean-motion-repair-v2") +RECONCILED_RESUME = "lhm-resume-reconciled" +LHM_RESUME_PHASES = ("lhm-resume", RECONCILED_RESUME) PHASE_MOUNTS = { **dict.fromkeys(CLEAN_MOTION_PHASES, ("/source", "/clean", "/motion")), "clean-only": ("/clean",), "clean-instance": ("/clean", "/lhm"), - **dict.fromkeys(("lhm", "lhm-resume", "dino", "gfpgan"), ("/lhm",)), + **dict.fromkeys(("lhm", *LHM_RESUME_PHASES, "dino", "gfpgan"), ("/lhm",)), } def digest_file(path): - import hashlib - digest = hashlib.sha256() with Path(path).open("rb") as file: for chunk in iter(lambda: file.read(4 * 1024 * 1024), b""): @@ -73,8 +76,6 @@ def inventory(root, *, skip_git_metadata=False): def write_receipt(path, document): - import json - path = Path(path) path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_suffix(".tmp") @@ -82,9 +83,66 @@ def write_receipt(path, document): temporary.replace(path) +def validate_staging_phase(phase, prior_resume_sha256): + if phase not in PHASE_MOUNTS: + raise ValueError("unknown staging phase") + if phase == RECONCILED_RESUME: + if not isinstance(prior_resume_sha256, str) or not re.fullmatch( + r"[0-9a-fA-F]{64}", prior_resume_sha256 + ): + raise ValueError("lhm-resume-reconciled requires a valid --prior-resume-sha256") + return prior_resume_sha256.lower() + if prior_resume_sha256 is not None: + raise ValueError("--prior-resume-sha256 is only valid for lhm-resume-reconciled") + return None + + +def validate_initial_lhm_resume(receipt_root, pending, destination): + previous = json.loads((receipt_root / "lhm-attempt.json").read_text()) + if ( + (previous.get("errorType"), previous.get("error")) + not in ( + ("TimeoutError", "cache staging deadline exhausted"), + ("ValueError", "unexpected prior archive root"), + ) + or previous.get("steps") + or not pending.is_dir() + or destination.exists() + ): + raise RuntimeError("range recovery requires a diagnosed incomplete extraction") + + +def reconciled_resume_metadata(receipt_root, pending, destination, prior_sha256): + raw = (receipt_root / "lhm-resume-attempt.json").read_bytes() + if hashlib.sha256(raw).hexdigest() != prior_sha256: + raise RuntimeError("prior lhm-resume receipt SHA-256 mismatch") + previous = json.loads(raw) + call_id = previous.get("functionCallId") if isinstance(previous, dict) else None + if ( + not isinstance(call_id, str) + or not re.fullmatch(r"fc-[A-Z0-9]{26}", call_id) + or previous.get("phase") != "lhm-resume" + or previous.get("status") != "failed" + or previous.get("errorType") != "TimeoutError" + or previous.get("error") != "cache staging deadline exhausted" + or previous.get("steps") != [] + ): + raise RuntimeError( + "final range recovery requires a failed lhm-resume deadline receipt with no steps" + ) + if pending.is_symlink() or destination.is_symlink(): + raise RuntimeError("final range recovery requires ordinary prior cache paths") + validate_initial_lhm_resume(receipt_root, pending, destination) + return { + "priorResumeReceiptSha256": prior_sha256, + "priorResumeFunctionCallId": call_id, + "totalAttempt": 3, + "rangeAttempt": 2, + } + + def extract_priors(response, destination, expected_bytes): """Hash and extract one streamed public tar with Python3.12's safe data filter.""" - import hashlib import tarfile class Reader: @@ -252,8 +310,8 @@ def read(self, size): max_containers=1, scaledown_window=2, ) -def stage(phase: str, deadline_epoch: float): - import json +def stage(phase: str, deadline_epoch: float, prior_resume_sha256: str | None = None): + prior_resume_sha256 = validate_staging_phase(phase, prior_resume_sha256) import shutil import signal import time @@ -261,8 +319,6 @@ def stage(phase: str, deadline_epoch: float): from huggingface_hub import snapshot_download - if phase not in PHASE_MOUNTS: - raise ValueError("unknown staging phase") if not all(Path(mount).is_dir() for mount in PHASE_MOUNTS[phase]): raise RuntimeError("staging phase mounts missing; invoke the main local entrypoint") remaining = min(1750, int(deadline_epoch - time.time())) @@ -285,7 +341,13 @@ def deadline(*_): } receipt_mount = "/clean" if phase.startswith("clean-") else "/lhm" receipt_root = Path(receipt_mount) / "setup-receipts" - failure = receipt_root / f"{phase}-attempt.json" + failure = receipt_root / ( + "lhm-resume-2-attempt.json" if phase == RECONCILED_RESUME else f"{phase}-attempt.json" + ) + if phase == RECONCILED_RESUME and (failure.exists() or failure.is_symlink()): + raise RuntimeError( + "final range recovery slot already consumed; inspect the existing receipt" + ) if failure.exists(): previous = json.loads(failure.read_text()) if previous.get("status") != "complete": @@ -294,6 +356,14 @@ def deadline(*_): ) return previous + if phase == "lhm" or phase in LHM_RESUME_PHASES: + destination = Path("/lhm/data") + pending = Path("/lhm/data.incomplete") + if phase == RECONCILED_RESUME: + report.update( + reconciled_resume_metadata(receipt_root, pending, destination, prior_resume_sha256) + ) + def commit_changes(): VOLUMES[receipt_mount].commit() if phase in CLEAN_MOTION_PHASES: @@ -301,7 +371,11 @@ def commit_changes(): signal.signal(signal.SIGALRM, deadline) signal.alarm(remaining) - write_receipt(failure, report) + if phase == RECONCILED_RESUME: + with failure.open("x") as stream: + json.dump(report, stream, indent=2) + else: + write_receipt(failure, report) commit_changes() def step_done(name, document): @@ -443,22 +517,10 @@ def step_done(name, document): "sha256": expected_sha, }, ) - elif phase in ("lhm", "lhm-resume"): - destination = Path("/lhm/data") - pending = Path("/lhm/data.incomplete") - if phase == "lhm-resume": - previous = json.loads((receipt_root / "lhm-attempt.json").read_text()) - if ( - (previous.get("errorType"), previous.get("error")) - not in ( - ("TimeoutError", "cache staging deadline exhausted"), - ("ValueError", "unexpected prior archive root"), - ) - or previous.get("steps") - or not pending.is_dir() - or destination.exists() - ): - raise RuntimeError("range recovery requires a diagnosed incomplete extraction") + elif phase == "lhm" or phase in LHM_RESUME_PHASES: + if phase in LHM_RESUME_PHASES: + if phase == "lhm-resume": + validate_initial_lhm_resume(receipt_root, pending, destination) receipt = extract_missing_priors(PRIOR_URL, pending, PRIOR_BYTES) else: if destination.exists() or pending.exists(): @@ -564,13 +626,13 @@ def step_done(name, document): @app.local_entrypoint() -def main(phase: str, deadline_epoch: float, out: str): - import json - - if phase not in PHASE_MOUNTS: - raise ValueError("unknown staging phase") +def main(phase: str, deadline_epoch: float, out: str, prior_resume_sha256: str | None = None): + prior_resume_sha256 = validate_staging_phase(phase, prior_resume_sha256) volumes = {mount: VOLUMES[mount] for mount in PHASE_MOUNTS[phase]} - report = stage.with_options(volumes=volumes).remote(phase, deadline_epoch) + options = ( + {"prior_resume_sha256": prior_resume_sha256} if prior_resume_sha256 is not None else {} + ) + report = stage.with_options(volumes=volumes).remote(phase, deadline_epoch, **options) path = Path(out) path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(report, indent=2))