From 002517f74bb9a3eb73d507958b04f12adfc327a7 Mon Sep 17 00:00:00 2001 From: Austin Jian Date: Fri, 20 Nov 2026 15:45:31 -0500 Subject: [PATCH] fix: persist bounded Modal tracking call receipts --- docs/paid-recovery.md | 25 +++ scripts/tests/test_tracking_submission.py | 197 ++++++++++++++++++++++ worker/modal_motion.py | 22 +-- worker/modal_multiperson.py | 56 +++++- worker/stages/modal_profile.py | 23 +++ 5 files changed, 301 insertions(+), 22 deletions(-) create mode 100644 scripts/tests/test_tracking_submission.py create mode 100644 worker/stages/modal_profile.py diff --git a/docs/paid-recovery.md b/docs/paid-recovery.md index b25d25c..f2a2863 100644 --- a/docs/paid-recovery.md +++ b/docs/paid-recovery.md @@ -625,3 +625,28 @@ under **Source and outputs**, when present, even if reconstruction failed before The links are read-only and uncached so a current cancellation receipt can be inspected. No arbitrary run files or receipts outside the run directory are exposed. Missing historical receipts are not synthesized, and a visible receipt does not authorize another paid attempt. + +## Tracking result waiting + +The tracking entrypoint uses the same arguments as before: + +```sh +WANDER_WORKER_TIMEOUT_SECONDS=600 uv run --locked modal run worker/modal_multiperson.py::main \ + --video /path/to/clip.mp4 --cameras /path/to/cameras.json --out /path/to/tracks +``` + +Tracking writes `tracking-call.json` in `--out` before submitting, with `appId` and the effective +local `modalProfile` alias under the same rules as the motion receipt. The alias is a local +recovery hint. Verify the app and call IDs against provider history to establish ownership. +Tracking adds the returned `functionCallId` before waiting. A submission that does not return +an ID stays `submission_uncertain`, and any existing tracking call receipt blocks submission to +that destination. Use retained evidence and the shared paid ledger to reconcile the original +attempt before considering another execution. + +The result wait covers queueing and uses the sum of the registered startup and execution +timeouts. With the 600-second ceiling, this is 1,200 seconds. With the default registration, +Modal 1.5.5 uses 3,600 seconds each for startup and execution, giving a 7,200-second wait. +Submission, cancellation and extraction are outside that wait. A timeout or interrupted wait +requests cancellation of the owned call and records whether the request outcome is known. +This receipt tracks the provider call. Tracking outputs still arrive through the returned +archive, and `result_received` precedes extraction and checks of the tracking report. diff --git a/scripts/tests/test_tracking_submission.py b/scripts/tests/test_tracking_submission.py new file mode 100644 index 0000000..a6af88f --- /dev/null +++ b/scripts/tests/test_tracking_submission.py @@ -0,0 +1,197 @@ +"""Exercise the tracking local entrypoint with offline Modal call doubles.""" + +import io +import json +import sys +import tarfile +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +from click.testing import CliRunner +from modal import config as modal_config +from modal.cli.entry_point import entrypoint_cli + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) +from worker import modal_multiperson +from worker.stages import lhm_recovery +from worker.stages.worker_resources import TIMEOUT_ENV, inference_resources + + +@pytest.fixture +def launch(tmp_path, monkeypatch): + video = tmp_path / "video.mp4" + video.write_bytes(b"video bytes") + cameras = tmp_path / "cameras.json" + cameras.write_bytes(b"camera bytes") + destination = tmp_path / "tracks" + function = Mock() + function.remote.side_effect = AssertionError("tracking must use spawn") + call = function.spawn.return_value + call.object_id = "fc-track" + monkeypatch.setattr(modal_multiperson, "track", function) + monkeypatch.setattr(modal_multiperson, "app", SimpleNamespace(app_id="ap-track")) + monkeypatch.setattr( + modal_multiperson, + "INFERENCE_RESOURCES", + inference_resources(timeout=3600, memory=65536, environment={TIMEOUT_ENV: "600"}), + ) + monkeypatch.setattr(modal_config, "_user_config", {"local-alias": {}, "cli-alias": {}}) + monkeypatch.setattr(modal_config, "_profile", "local-alias") + monkeypatch.setenv("MODAL_PROFILE", "local-alias") + for name in ( + "MODAL_TOKEN_ID", + "MODAL_TOKEN_SECRET", + "MODAL_OAUTH_REFRESH_TOKEN", + "MODAL_OAUTH_CLIENT_ID", + "MODAL_OAUTH_CLIENT_SECRET", + ): + monkeypatch.delenv(name, raising=False) + + def run(**overrides): + arguments = {"video": str(video), "out": str(destination), "cameras": str(cameras)} + modal_multiperson.main(**(arguments | overrides)) + + def receipt(): + return json.loads((destination / "tracking-call.json").read_text()) + + return SimpleNamespace( + run=run, receipt=receipt, destination=destination, function=function, call=call + ) + + +def tracking_output(error=None): + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w:gz") as archive: + log = b"tracking evidence\n" + entry = tarfile.TarInfo("track.log") + entry.size = len(log) + archive.addfile(entry, io.BytesIO(log)) + return {"archive": buffer.getvalue(), "report": {"error": error}} + + +def test_tracking_receipt_is_durable_before_spawn_and_wait(launch, monkeypatch): + monkeypatch.setitem(sys.modules, "worker.modal_motion", None) + cli = CliRunner().invoke(entrypoint_cli, ["--profile", "cli-alias", "run", "--help"]) + assert cli.exit_code == 0, cli.output + initial = { + "schema": "wander.tracking-call/1", + "status": "submitting", + "appId": "ap-track", + "modalProfile": "cli-alias", + "functionCallId": None, + "resultTimeoutSeconds": 1200, + } + + def spawn(*args): + assert launch.receipt() == initial + return launch.call + + def get(*, timeout): + assert timeout == 1200 + assert launch.receipt() == initial | {"status": "waiting", "functionCallId": "fc-track"} + return tracking_output() + + launch.function.spawn.side_effect = spawn + launch.call.get.side_effect = get + launch.run(fps=15.0, det_thresh=0.3, gate=0.4, max_gap=4, min_track_samples=6, overlay_every=2) + launch.function.spawn.assert_called_once_with( + b"video bytes", b"camera bytes", 15.0, 0.3, 0.4, 4, 6, 2 + ) + launch.call.cancel.assert_not_called() + assert launch.receipt() == initial | {"status": "result_received", "functionCallId": "fc-track"} + assert (launch.destination / "track.log").read_bytes() == b"tracking evidence\n" + assert json.loads((launch.destination / "modal-run.json").read_text()) == {"error": None} + + +def test_tracking_default_wait_and_reported_inference_failure(launch, monkeypatch): + monkeypatch.setattr( + modal_multiperson, + "INFERENCE_RESOURCES", + inference_resources(timeout=3600, memory=65536, environment={}), + ) + monkeypatch.setenv("MODAL_TOKEN_ID", "synthetic-override") + launch.call.get.return_value = tracking_output("inference failed") + with pytest.raises(RuntimeError, match="see the downloaded track.log"): + launch.run(cameras="") + launch.function.spawn.assert_called_once_with(b"video bytes", None, 12, 0.15, 0.22, 8, 8, 0) + launch.call.get.assert_called_once_with(timeout=7200) + launch.call.cancel.assert_not_called() + assert launch.receipt()["status"] == "result_received" + assert launch.receipt()["modalProfile"] is None + assert (launch.destination / "track.log").is_file() + assert json.loads((launch.destination / "modal-run.json").read_text()) == { + "error": "inference failed" + } + + +@pytest.mark.parametrize( + "error, cancel_fails, status, raised", + [ + (TimeoutError(), False, "wait_timeout", RuntimeError), + (TimeoutError(), True, "wait_timeout", RuntimeError), + (KeyboardInterrupt(), False, "wait_interrupted", KeyboardInterrupt), + ], +) +def test_wait_failure_cancels_only_owned_call_and_blocks_repeat( + launch, error, cancel_fails, status, raised +): + launch.call.get.side_effect = error + if cancel_fails: + launch.call.cancel.side_effect = OSError("private cancellation response") + with pytest.raises(raised) as failure: + launch.run() + if isinstance(error, TimeoutError): + assert failure.value.__cause__ is error + assert "do not resubmit" in str(failure.value) + launch.call.cancel.assert_called_once_with() + receipt = launch.receipt() + assert receipt["status"] == status + assert receipt["appId"] == "ap-track" + assert receipt["modalProfile"] == "local-alias" + assert receipt["functionCallId"] == "fc-track" + assert receipt["cancellation"] == ("unknown" if cancel_fails else "requested") + assert "private cancellation response" not in json.dumps(receipt) + with pytest.raises(FileExistsError): + launch.run() + launch.function.spawn.assert_called_once() + assert launch.receipt() == receipt + + +def test_uncertain_submission_blocks_repeat_without_cancelling_unknown_call(launch): + launch.function.spawn.side_effect = OSError("lost submission response") + with pytest.raises(OSError, match="lost submission response"): + launch.run() + receipt = launch.receipt() + assert receipt["status"] == "submission_uncertain" + assert receipt["appId"] == "ap-track" + assert receipt["modalProfile"] == "local-alias" + assert receipt["functionCallId"] is None + with pytest.raises(FileExistsError): + launch.run() + launch.function.spawn.assert_called_once() + launch.call.get.assert_not_called() + launch.call.cancel.assert_not_called() + assert launch.receipt() == receipt + + +def test_receipt_write_failure_after_spawn_cancels_before_waiting(launch, monkeypatch): + atomic_json = lhm_recovery.atomic_json + + def write(path, receipt): + if receipt["status"] == "waiting": + raise OSError("disk write failed") + atomic_json(path, receipt) + + monkeypatch.setattr(lhm_recovery, "atomic_json", write) + with pytest.raises(OSError, match="disk write failed"): + launch.run() + launch.function.spawn.assert_called_once() + launch.call.get.assert_not_called() + launch.call.cancel.assert_called_once_with() + receipt = launch.receipt() + assert receipt["status"] == "wait_interrupted" + assert receipt["functionCallId"] == "fc-track" + assert receipt["cancellation"] == "requested" diff --git a/worker/modal_motion.py b/worker/modal_motion.py index ac58d61..7791f38 100644 --- a/worker/modal_motion.py +++ b/worker/modal_motion.py @@ -9,11 +9,11 @@ from __future__ import annotations -import os from pathlib import Path import modal if modal.is_local(): + from worker.stages.modal_profile import effective_modal_profile as _effective_modal_profile from worker.stages.worker_resources import inference_resources else: from worker_resources import inference_resources @@ -179,26 +179,6 @@ def reconstruct( return dict(report=report, archive=buf.getvalue()) -def _effective_modal_profile() -> str | None: - from modal import config as modal_config - - # Modal 1.5.5 updates this selection for CLI --profile before entering main. - profile = modal_config._profile - if profile not in modal_config.config_profiles(): - return None - # The SDK resolves credential settings from the environment before the profile. - credential_overrides = ( - "MODAL_TOKEN_ID", - "MODAL_TOKEN_SECRET", - "MODAL_OAUTH_REFRESH_TOKEN", - "MODAL_OAUTH_CLIENT_ID", - "MODAL_OAUTH_CLIENT_SECRET", - ) - if any(name in os.environ for name in credential_overrides): - return None - return profile - - @app.local_entrypoint() def main( video: str, out: str, experiment: str = "vdpm", batch: int = 16, anchor_lock: bool = False diff --git a/worker/modal_multiperson.py b/worker/modal_multiperson.py index dabc33e..4851b4d 100644 --- a/worker/modal_multiperson.py +++ b/worker/modal_multiperson.py @@ -228,6 +228,59 @@ def track( return {"report": report, "archive": buf.getvalue()} +def _tracking_result(dest, *inputs): + from worker.stages.lhm_recovery import atomic_json + from worker.stages.modal_profile import effective_modal_profile + + execution_timeout = INFERENCE_RESOURCES["timeout"] + # Modal 1.5.5 uses the execution timeout when startup_timeout is omitted. + timeout = INFERENCE_RESOURCES.get("startup_timeout", execution_timeout) + execution_timeout + path = dest / "tracking-call.json" + receipt = { + "schema": "wander.tracking-call/1", + "status": "submitting", + "appId": app.app_id, + "modalProfile": effective_modal_profile(), + "functionCallId": None, + "resultTimeoutSeconds": timeout, + } + # Reserve this destination exclusively even if submission never returns a call ID. + with path.open("x") as stream: + stream.write("{}\n") + atomic_json(path, receipt) + try: + call = track.spawn(*inputs) + except BaseException: + receipt["status"] = "submission_uncertain" + atomic_json(path, receipt) + raise + + try: + receipt.update(functionCallId=call.object_id, status="waiting") + atomic_json(path, receipt) + result = call.get(timeout=timeout) + except BaseException as error: + timed_out = isinstance(error, TimeoutError) + receipt["status"] = "wait_timeout" if timed_out else "wait_interrupted" + try: + call.cancel() + receipt["cancellation"] = "requested" + except BaseException as cancellation_error: + receipt["cancellation"] = "unknown" + receipt["cancellationErrorType"] = type(cancellation_error).__name__ + atomic_json(path, receipt) + if timed_out: + raise RuntimeError( + f"Tracking result wait timed out after {timeout}s; cancellation " + f"{receipt['cancellation']}. Inspect {path} before any recovery; do not resubmit." + ) from error + raise + + receipt["status"] = "result_received" + atomic_json(path, receipt) + return result + + @app.local_entrypoint() def main( video: str, @@ -246,7 +299,8 @@ def main( dest = Path(out) dest.mkdir(parents=True, exist_ok=True) - result = track.remote( + result = _tracking_result( + dest, Path(video).read_bytes(), Path(cameras).read_bytes() if cameras else None, fps, diff --git a/worker/stages/modal_profile.py b/worker/stages/modal_profile.py new file mode 100644 index 0000000..72c3965 --- /dev/null +++ b/worker/stages/modal_profile.py @@ -0,0 +1,23 @@ +"""Resolve local Modal profile aliases for provider call receipts.""" + +import os + + +def effective_modal_profile() -> str | None: + from modal import config as modal_config + + # Modal 1.5.5 updates this selection for CLI --profile before entering main. + profile = modal_config._profile + if profile not in modal_config.config_profiles(): + return None + # The SDK resolves credential settings from the environment before the profile. + credential_overrides = ( + "MODAL_TOKEN_ID", + "MODAL_TOKEN_SECRET", + "MODAL_OAUTH_REFRESH_TOKEN", + "MODAL_OAUTH_CLIENT_ID", + "MODAL_OAUTH_CLIENT_SECRET", + ) + if any(name in os.environ for name in credential_overrides): + return None + return profile