From 6595b7d191e14eaa5ad2bcb7f6dc3cdbfe393831 Mon Sep 17 00:00:00 2001 From: Austin Jian Date: Fri, 20 Nov 2026 11:25:02 -0500 Subject: [PATCH] fix: record Modal identity in bounded motion receipts --- docs/paid-recovery.md | 22 ++++-- scripts/tests/test_motion_submission.py | 91 ++++++++++++++++++++++--- worker/modal_motion.py | 23 +++++++ worker/stages/motion_submission.py | 6 +- 4 files changed, 125 insertions(+), 17 deletions(-) diff --git a/docs/paid-recovery.md b/docs/paid-recovery.md index 4cbfa7c..b25d25c 100644 --- a/docs/paid-recovery.md +++ b/docs/paid-recovery.md @@ -598,13 +598,21 @@ without that variable retains the existing direct call behavior. This is not a d whole-command deadline: submission/upload, image setup, cancellation and artifact extraction are outside the result wait. -The worker writes `motion-call.json` before submitting and saves the returned function-call ID -before waiting. A timed-out or interrupted wait requests cancellation of only that call, without -terminating shared containers. The receipt distinguishes a cancellation request from an unknown -cancellation outcome; neither proves zero spend. Submission without a returned ID remains -`submission_uncertain`. Inspect the saved ID/provider history before recovery; an existing receipt -cannot submit again. The shared paid-stage claim remains consumed. `result_received` means the -provider returned, not that extraction or visual acceptance passed. +The worker writes `motion-call.json` with the Modal application ID (`appId`) and effective local +credential profile alias (`modalProfile`) before submitting. It adds the returned `functionCallId` +before waiting and retains the metadata across outcomes. The alias reflects the SDK's selected +profile, including CLI `--profile` selection. It is `null` if the SDK has no matching configured +profile or environment variables override credential settings. The alias is local to the launching +machine and provides no proof of Modal workspace ownership. Historical receipts may lack both +fields. Verify the app and call IDs against provider history before recovery, and leave the profile +unknown where evidence is missing. + +A timed-out or interrupted wait requests cancellation of only that call, without terminating shared +containers. The receipt distinguishes a cancellation request from an unknown cancellation outcome; +neither proves zero spend. Submission without a returned ID remains `submission_uncertain`. An +existing receipt cannot submit again, and the shared paid-stage claim remains consumed. +`result_received` records the provider return. Extraction and visual acceptance still require +verification. A fresh city-upload trial on Nov18 exposed this gap: the Pi3X call remained queued for ten minutes with backlog1, zero runners and zero running inputs, then the operator stopped its owned app. diff --git a/scripts/tests/test_motion_submission.py b/scripts/tests/test_motion_submission.py index 3653d43..664c64d 100644 --- a/scripts/tests/test_motion_submission.py +++ b/scripts/tests/test_motion_submission.py @@ -14,6 +14,13 @@ sys.path.insert(0, str(ROOT)) from worker.stages.motion_submission import bounded_motion_result +OWNER = {"app_id": "ap-owner", "modal_profile": "local-owner"} + + +def assert_owner(receipt): + assert receipt["appId"] == OWNER["app_id"] + assert receipt["modalProfile"] == OWNER["modal_profile"] + def test_identity_is_durable_before_wait_and_success_is_not_cancelled(tmp_path): call = Mock(object_id="fc-owned") @@ -22,18 +29,21 @@ def test_identity_is_durable_before_wait_and_success_is_not_cancelled(tmp_path): def result(*, timeout): receipt = json.loads((tmp_path / "motion-call.json").read_text()) + assert_owner(receipt) assert receipt["functionCallId"] == "fc-owned" assert receipt["status"] == "waiting" assert timeout == 1200 return {"archive": b"result"} call.get.side_effect = result - assert bounded_motion_result(function, tmp_path, b"video", timeout=1200, batch=16) == { + assert bounded_motion_result(function, tmp_path, b"video", timeout=1200, batch=16, **OWNER) == { "archive": b"result" } function.spawn.assert_called_once_with(b"video", batch=16) call.cancel.assert_not_called() - assert json.loads((tmp_path / "motion-call.json").read_text())["status"] == "result_received" + receipt = json.loads((tmp_path / "motion-call.json").read_text()) + assert_owner(receipt) + assert receipt["status"] == "result_received" @pytest.mark.parametrize("cancel_fails", [False, True]) @@ -45,11 +55,12 @@ def test_queue_timeout_cancels_only_owned_call_and_never_retries(tmp_path, cance if cancel_fails: call.cancel.side_effect = OSError("sensitive provider response") with pytest.raises(RuntimeError, match="do not resubmit"): - bounded_motion_result(function, tmp_path, b"video", timeout=1200) + bounded_motion_result(function, tmp_path, b"video", timeout=1200, **OWNER) function.spawn.assert_called_once() call.get.assert_called_once_with(timeout=1200) call.cancel.assert_called_once_with() receipt = json.loads((tmp_path / "motion-call.json").read_text()) + assert_owner(receipt) assert receipt["functionCallId"] == "fc-owned" assert receipt["status"] == "wait_timeout" assert receipt["cancellation"] == ("unknown" if cancel_fails else "requested") @@ -57,6 +68,7 @@ def test_queue_timeout_cancels_only_owned_call_and_never_retries(tmp_path, cance with pytest.raises(FileExistsError): bounded_motion_result(function, tmp_path, b"video", timeout=1200) function.spawn.assert_called_once() + assert json.loads((tmp_path / "motion-call.json").read_text()) == receipt @pytest.mark.parametrize("error", [KeyboardInterrupt(), RuntimeError("provider failure")]) @@ -66,17 +78,20 @@ def test_interruption_cancels_without_reclassifying_as_wait_timeout(tmp_path, er call.object_id = "fc-owned" call.get.side_effect = error with pytest.raises(type(error)): - bounded_motion_result(function, tmp_path, b"video", timeout=1200) + bounded_motion_result(function, tmp_path, b"video", timeout=1200, **OWNER) call.cancel.assert_called_once_with() - assert json.loads((tmp_path / "motion-call.json").read_text())["status"] == "wait_interrupted" + receipt = json.loads((tmp_path / "motion-call.json").read_text()) + assert_owner(receipt) + assert receipt["status"] == "wait_interrupted" def test_unknown_submission_is_preserved_and_cannot_be_repeated(tmp_path): function = Mock() function.spawn.side_effect = OSError("lost response") with pytest.raises(OSError): - bounded_motion_result(function, tmp_path, b"video", timeout=1200) + bounded_motion_result(function, tmp_path, b"video", timeout=1200, **OWNER) receipt = json.loads((tmp_path / "motion-call.json").read_text()) + assert_owner(receipt) assert receipt["status"] == "submission_uncertain" assert receipt["functionCallId"] is None with pytest.raises(FileExistsError): @@ -84,6 +99,19 @@ def test_unknown_submission_is_preserved_and_cannot_be_repeated(tmp_path): function.spawn.assert_called_once() +def test_existing_legacy_receipt_prevents_submission(tmp_path): + path = tmp_path / "motion-call.json" + legacy = '{"schema":"wander.motion-call/1","functionCallId":"fc-legacy"}\n' + path.write_text(legacy) + function = Mock() + + with pytest.raises(FileExistsError): + bounded_motion_result(function, tmp_path, b"video", timeout=1200, **OWNER) + + function.spawn.assert_not_called() + assert path.read_text() == legacy + + @pytest.mark.parametrize("timeout", [0, -1, True, 1.5, "600"]) def test_invalid_wait_fails_before_submission(tmp_path, timeout): function = Mock() @@ -97,6 +125,9 @@ def test_motion_entrypoint_wires_bounded_wait_without_changing_default( tmp_path, monkeypatch, bounded ): import modal + from click.testing import CliRunner + from modal import config as modal_config + from modal.cli.entry_point import entrypoint_cli monkeypatch.delenv("WANDER_WORKER_TIMEOUT_SECONDS", raising=False) if bounded: @@ -118,21 +149,63 @@ def test_motion_entrypoint_wires_bounded_wait_without_changing_default( call.get.return_value = result main = namespace["main"] main.__globals__["reconstruct"] = function + main.__globals__["app"] = Mock(app_id="ap-entrypoint") source = tmp_path / "source.mp4" source.write_bytes(b"original") destination = tmp_path / "output" - main(str(source), str(destination), experiment="pi3x") + receipt_path = destination / "motion-call.json" + + def spawn(*args, **kwargs): + receipt = json.loads(receipt_path.read_text()) + assert receipt["status"] == "submitting" + assert receipt["functionCallId"] is None + assert receipt["appId"] == "ap-entrypoint" + assert receipt["modalProfile"] == "cli-selected" + return call + + function.spawn.side_effect = spawn + monkeypatch.setattr(modal_config, "_profile", "environment-selected") + monkeypatch.setattr( + modal_config, "_user_config", {"environment-selected": {}, "cli-selected": {}} + ) + with patch.dict("os.environ", {"MODAL_PROFILE": "environment-selected"}, clear=True): + if bounded: + cli = CliRunner().invoke(entrypoint_cli, ["--profile", "cli-selected", "run", "--help"]) + assert cli.exit_code == 0, cli.output + main(str(source), str(destination), experiment="pi3x") if bounded: function.remote.assert_not_called() function.spawn.assert_called_once_with( b"original", experiment="pi3x", batch=16, anchor_lock=False ) call.get.assert_called_once_with(timeout=1200) - assert (destination / "motion-call.json").exists() + assert json.loads(receipt_path.read_text())["functionCallId"] == "fc-entrypoint" else: function.remote.assert_called_once_with(b"original", "pi3x", 16, False) function.spawn.assert_not_called() - assert not (destination / "motion-call.json").exists() + assert not receipt_path.exists() + + +@pytest.mark.parametrize( + "configured, override, expected", + [ + (True, None, "configured-alias"), + (False, None, None), + (True, "MODAL_TOKEN_ID", None), + (True, "MODAL_OAUTH_REFRESH_TOKEN", None), + ], +) +def test_profile_alias_requires_a_configured_profile_without_credential_overrides( + monkeypatch, configured, override, expected +): + from modal import config as modal_config + from worker.modal_motion import _effective_modal_profile + + profiles = {"configured-alias": {"active": True}} if configured else {} + monkeypatch.setattr(modal_config, "_user_config", profiles) + monkeypatch.setattr(modal_config, "_profile", modal_config._config_active_profile()) + with patch.dict("os.environ", {override: "synthetic"} if override else {}, clear=True): + assert _effective_modal_profile() == expected def test_pinned_sdk_empty_poll_raises_builtin_timeout(): diff --git a/worker/modal_motion.py b/worker/modal_motion.py index fb49ef2..ac58d61 100644 --- a/worker/modal_motion.py +++ b/worker/modal_motion.py @@ -9,6 +9,7 @@ from __future__ import annotations +import os from pathlib import Path import modal @@ -178,6 +179,26 @@ def reconstruct( return dict(report=report, archive=buf.getvalue()) +def _effective_modal_profile() -> str | None: + from modal import config as modal_config + + # Modal 1.5.5 updates this selection for CLI --profile before entering main. + profile = modal_config._profile + if profile not in modal_config.config_profiles(): + return None + # The SDK resolves credential settings from the environment before the profile. + credential_overrides = ( + "MODAL_TOKEN_ID", + "MODAL_TOKEN_SECRET", + "MODAL_OAUTH_REFRESH_TOKEN", + "MODAL_OAUTH_CLIENT_ID", + "MODAL_OAUTH_CLIENT_SECRET", + ) + if any(name in os.environ for name in credential_overrides): + return None + return profile + + @app.local_entrypoint() def main( video: str, out: str, experiment: str = "vdpm", batch: int = 16, anchor_lock: bool = False @@ -199,6 +220,8 @@ def main( destination, inputs, timeout=INFERENCE_RESOURCES["startup_timeout"] + INFERENCE_RESOURCES["timeout"], + app_id=app.app_id, + modal_profile=_effective_modal_profile(), experiment=experiment, batch=batch, anchor_lock=anchor_lock, diff --git a/worker/stages/motion_submission.py b/worker/stages/motion_submission.py index 238d588..ac1eea6 100644 --- a/worker/stages/motion_submission.py +++ b/worker/stages/motion_submission.py @@ -5,7 +5,9 @@ from worker.stages.lhm_recovery import atomic_json -def bounded_motion_result(function, destination, inputs, *, timeout, **kwargs): +def bounded_motion_result( + function, destination, inputs, *, timeout, app_id=None, modal_profile=None, **kwargs +): """Bound result waiting (including queueing), not upload, cancellation, or billing. The caller owns a new output directory. An existing receipt is never resubmitted. @@ -20,6 +22,8 @@ def bounded_motion_result(function, destination, inputs, *, timeout, **kwargs): receipt = { "schema": "wander.motion-call/1", "status": "submitting", + "appId": app_id, + "modalProfile": modal_profile, "functionCallId": None, "resultTimeoutSeconds": timeout, }