Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 15 additions & 7 deletions docs/paid-recovery.md
Original file line number Diff line number Diff line change
Expand Up @@ -598,13 +598,21 @@ without that variable retains the existing direct call behavior. This is not a d
whole-command deadline: submission/upload, image setup, cancellation and artifact extraction
are outside the result wait.

The worker writes `motion-call.json` before submitting and saves the returned function-call ID
before waiting. A timed-out or interrupted wait requests cancellation of only that call, without
terminating shared containers. The receipt distinguishes a cancellation request from an unknown
cancellation outcome; neither proves zero spend. Submission without a returned ID remains
`submission_uncertain`. Inspect the saved ID/provider history before recovery; an existing receipt
cannot submit again. The shared paid-stage claim remains consumed. `result_received` means the
provider returned, not that extraction or visual acceptance passed.
The worker writes `motion-call.json` with the Modal application ID (`appId`) and effective local
credential profile alias (`modalProfile`) before submitting. It adds the returned `functionCallId`
before waiting and retains the metadata across outcomes. The alias reflects the SDK's selected
profile, including CLI `--profile` selection. It is `null` if the SDK has no matching configured
profile or environment variables override credential settings. The alias is local to the launching
machine and provides no proof of Modal workspace ownership. Historical receipts may lack both
fields. Verify the app and call IDs against provider history before recovery, and leave the profile
unknown where evidence is missing.

A timed-out or interrupted wait requests cancellation of only that call, without terminating shared
containers. The receipt distinguishes a cancellation request from an unknown cancellation outcome;
neither proves zero spend. Submission without a returned ID remains `submission_uncertain`. An
existing receipt cannot submit again, and the shared paid-stage claim remains consumed.
`result_received` records the provider return. Extraction and visual acceptance still require
verification.

A fresh city-upload trial on Nov18 exposed this gap: the Pi3X call remained queued for ten minutes
with backlog1, zero runners and zero running inputs, then the operator stopped its owned app.
Expand Down
91 changes: 82 additions & 9 deletions scripts/tests/test_motion_submission.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,13 @@
sys.path.insert(0, str(ROOT))
from worker.stages.motion_submission import bounded_motion_result

OWNER = {"app_id": "ap-owner", "modal_profile": "local-owner"}


def assert_owner(receipt):
assert receipt["appId"] == OWNER["app_id"]
assert receipt["modalProfile"] == OWNER["modal_profile"]


def test_identity_is_durable_before_wait_and_success_is_not_cancelled(tmp_path):
call = Mock(object_id="fc-owned")
Expand All @@ -22,18 +29,21 @@ def test_identity_is_durable_before_wait_and_success_is_not_cancelled(tmp_path):

def result(*, timeout):
receipt = json.loads((tmp_path / "motion-call.json").read_text())
assert_owner(receipt)
assert receipt["functionCallId"] == "fc-owned"
assert receipt["status"] == "waiting"
assert timeout == 1200
return {"archive": b"result"}

call.get.side_effect = result
assert bounded_motion_result(function, tmp_path, b"video", timeout=1200, batch=16) == {
assert bounded_motion_result(function, tmp_path, b"video", timeout=1200, batch=16, **OWNER) == {
"archive": b"result"
}
function.spawn.assert_called_once_with(b"video", batch=16)
call.cancel.assert_not_called()
assert json.loads((tmp_path / "motion-call.json").read_text())["status"] == "result_received"
receipt = json.loads((tmp_path / "motion-call.json").read_text())
assert_owner(receipt)
assert receipt["status"] == "result_received"


@pytest.mark.parametrize("cancel_fails", [False, True])
Expand All @@ -45,18 +55,20 @@ def test_queue_timeout_cancels_only_owned_call_and_never_retries(tmp_path, cance
if cancel_fails:
call.cancel.side_effect = OSError("sensitive provider response")
with pytest.raises(RuntimeError, match="do not resubmit"):
bounded_motion_result(function, tmp_path, b"video", timeout=1200)
bounded_motion_result(function, tmp_path, b"video", timeout=1200, **OWNER)
function.spawn.assert_called_once()
call.get.assert_called_once_with(timeout=1200)
call.cancel.assert_called_once_with()
receipt = json.loads((tmp_path / "motion-call.json").read_text())
assert_owner(receipt)
assert receipt["functionCallId"] == "fc-owned"
assert receipt["status"] == "wait_timeout"
assert receipt["cancellation"] == ("unknown" if cancel_fails else "requested")
assert "sensitive" not in json.dumps(receipt)
with pytest.raises(FileExistsError):
bounded_motion_result(function, tmp_path, b"video", timeout=1200)
function.spawn.assert_called_once()
assert json.loads((tmp_path / "motion-call.json").read_text()) == receipt


@pytest.mark.parametrize("error", [KeyboardInterrupt(), RuntimeError("provider failure")])
Expand All @@ -66,24 +78,40 @@ def test_interruption_cancels_without_reclassifying_as_wait_timeout(tmp_path, er
call.object_id = "fc-owned"
call.get.side_effect = error
with pytest.raises(type(error)):
bounded_motion_result(function, tmp_path, b"video", timeout=1200)
bounded_motion_result(function, tmp_path, b"video", timeout=1200, **OWNER)
call.cancel.assert_called_once_with()
assert json.loads((tmp_path / "motion-call.json").read_text())["status"] == "wait_interrupted"
receipt = json.loads((tmp_path / "motion-call.json").read_text())
assert_owner(receipt)
assert receipt["status"] == "wait_interrupted"


def test_unknown_submission_is_preserved_and_cannot_be_repeated(tmp_path):
function = Mock()
function.spawn.side_effect = OSError("lost response")
with pytest.raises(OSError):
bounded_motion_result(function, tmp_path, b"video", timeout=1200)
bounded_motion_result(function, tmp_path, b"video", timeout=1200, **OWNER)
receipt = json.loads((tmp_path / "motion-call.json").read_text())
assert_owner(receipt)
assert receipt["status"] == "submission_uncertain"
assert receipt["functionCallId"] is None
with pytest.raises(FileExistsError):
bounded_motion_result(function, tmp_path, b"video", timeout=1200)
function.spawn.assert_called_once()


def test_existing_legacy_receipt_prevents_submission(tmp_path):
path = tmp_path / "motion-call.json"
legacy = '{"schema":"wander.motion-call/1","functionCallId":"fc-legacy"}\n'
path.write_text(legacy)
function = Mock()

with pytest.raises(FileExistsError):
bounded_motion_result(function, tmp_path, b"video", timeout=1200, **OWNER)

function.spawn.assert_not_called()
assert path.read_text() == legacy


@pytest.mark.parametrize("timeout", [0, -1, True, 1.5, "600"])
def test_invalid_wait_fails_before_submission(tmp_path, timeout):
function = Mock()
Expand All @@ -97,6 +125,9 @@ def test_motion_entrypoint_wires_bounded_wait_without_changing_default(
tmp_path, monkeypatch, bounded
):
import modal
from click.testing import CliRunner
from modal import config as modal_config
from modal.cli.entry_point import entrypoint_cli

monkeypatch.delenv("WANDER_WORKER_TIMEOUT_SECONDS", raising=False)
if bounded:
Expand All @@ -118,21 +149,63 @@ def test_motion_entrypoint_wires_bounded_wait_without_changing_default(
call.get.return_value = result
main = namespace["main"]
main.__globals__["reconstruct"] = function
main.__globals__["app"] = Mock(app_id="ap-entrypoint")
source = tmp_path / "source.mp4"
source.write_bytes(b"original")
destination = tmp_path / "output"
main(str(source), str(destination), experiment="pi3x")
receipt_path = destination / "motion-call.json"

def spawn(*args, **kwargs):
receipt = json.loads(receipt_path.read_text())
assert receipt["status"] == "submitting"
assert receipt["functionCallId"] is None
assert receipt["appId"] == "ap-entrypoint"
assert receipt["modalProfile"] == "cli-selected"
return call

function.spawn.side_effect = spawn
monkeypatch.setattr(modal_config, "_profile", "environment-selected")
monkeypatch.setattr(
modal_config, "_user_config", {"environment-selected": {}, "cli-selected": {}}
)
with patch.dict("os.environ", {"MODAL_PROFILE": "environment-selected"}, clear=True):
if bounded:
cli = CliRunner().invoke(entrypoint_cli, ["--profile", "cli-selected", "run", "--help"])
assert cli.exit_code == 0, cli.output
main(str(source), str(destination), experiment="pi3x")
if bounded:
function.remote.assert_not_called()
function.spawn.assert_called_once_with(
b"original", experiment="pi3x", batch=16, anchor_lock=False
)
call.get.assert_called_once_with(timeout=1200)
assert (destination / "motion-call.json").exists()
assert json.loads(receipt_path.read_text())["functionCallId"] == "fc-entrypoint"
else:
function.remote.assert_called_once_with(b"original", "pi3x", 16, False)
function.spawn.assert_not_called()
assert not (destination / "motion-call.json").exists()
assert not receipt_path.exists()


@pytest.mark.parametrize(
"configured, override, expected",
[
(True, None, "configured-alias"),
(False, None, None),
(True, "MODAL_TOKEN_ID", None),
(True, "MODAL_OAUTH_REFRESH_TOKEN", None),
],
)
def test_profile_alias_requires_a_configured_profile_without_credential_overrides(
monkeypatch, configured, override, expected
):
from modal import config as modal_config
from worker.modal_motion import _effective_modal_profile

profiles = {"configured-alias": {"active": True}} if configured else {}
monkeypatch.setattr(modal_config, "_user_config", profiles)
monkeypatch.setattr(modal_config, "_profile", modal_config._config_active_profile())
with patch.dict("os.environ", {override: "synthetic"} if override else {}, clear=True):
assert _effective_modal_profile() == expected


def test_pinned_sdk_empty_poll_raises_builtin_timeout():
Expand Down
23 changes: 23 additions & 0 deletions worker/modal_motion.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

from __future__ import annotations

import os
from pathlib import Path
import modal

Expand Down Expand Up @@ -178,6 +179,26 @@ def reconstruct(
return dict(report=report, archive=buf.getvalue())


def _effective_modal_profile() -> str | None:
from modal import config as modal_config

# Modal 1.5.5 updates this selection for CLI --profile before entering main.
profile = modal_config._profile
if profile not in modal_config.config_profiles():
return None
# The SDK resolves credential settings from the environment before the profile.
credential_overrides = (
"MODAL_TOKEN_ID",
"MODAL_TOKEN_SECRET",
"MODAL_OAUTH_REFRESH_TOKEN",
"MODAL_OAUTH_CLIENT_ID",
"MODAL_OAUTH_CLIENT_SECRET",
)
if any(name in os.environ for name in credential_overrides):
return None
return profile


@app.local_entrypoint()
def main(
video: str, out: str, experiment: str = "vdpm", batch: int = 16, anchor_lock: bool = False
Expand All @@ -199,6 +220,8 @@ def main(
destination,
inputs,
timeout=INFERENCE_RESOURCES["startup_timeout"] + INFERENCE_RESOURCES["timeout"],
app_id=app.app_id,
modal_profile=_effective_modal_profile(),
experiment=experiment,
batch=batch,
anchor_lock=anchor_lock,
Expand Down
6 changes: 5 additions & 1 deletion worker/stages/motion_submission.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@
from worker.stages.lhm_recovery import atomic_json


def bounded_motion_result(function, destination, inputs, *, timeout, **kwargs):
def bounded_motion_result(
function, destination, inputs, *, timeout, app_id=None, modal_profile=None, **kwargs
):
"""Bound result waiting (including queueing), not upload, cancellation, or billing.

The caller owns a new output directory. An existing receipt is never resubmitted.
Expand All @@ -20,6 +22,8 @@ def bounded_motion_result(function, destination, inputs, *, timeout, **kwargs):
receipt = {
"schema": "wander.motion-call/1",
"status": "submitting",
"appId": app_id,
"modalProfile": modal_profile,
"functionCallId": None,
"resultTimeoutSeconds": timeout,
}
Expand Down
Loading