Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions docs/paid-recovery.md
Original file line number Diff line number Diff line change
Expand Up @@ -625,3 +625,28 @@ under **Source and outputs**, when present, even if reconstruction failed before
The links are read-only and uncached so a current cancellation receipt can be inspected. No
arbitrary run files or receipts outside the run directory are exposed. Missing historical
receipts are not synthesized, and a visible receipt does not authorize another paid attempt.

## Tracking result waiting

The tracking entrypoint uses the same arguments as before:

```sh
WANDER_WORKER_TIMEOUT_SECONDS=600 uv run --locked modal run worker/modal_multiperson.py::main \
--video /path/to/clip.mp4 --cameras /path/to/cameras.json --out /path/to/tracks
```

Tracking writes `tracking-call.json` in `--out` before submitting, with `appId` and the effective
local `modalProfile` alias under the same rules as the motion receipt. The alias is a local
recovery hint. Verify the app and call IDs against provider history to establish ownership.
Tracking adds the returned `functionCallId` before waiting. A submission that does not return
an ID stays `submission_uncertain`, and any existing tracking call receipt blocks submission to
that destination. Use retained evidence and the shared paid ledger to reconcile the original
attempt before considering another execution.

The result wait covers queueing and uses the sum of the registered startup and execution
timeouts. With the 600-second ceiling, this is 1,200 seconds. With the default registration,
Modal 1.5.5 uses 3,600 seconds each for startup and execution, giving a 7,200-second wait.
Submission, cancellation and extraction are outside that wait. A timeout or interrupted wait
requests cancellation of the owned call and records whether the request outcome is known.
This receipt tracks the provider call. Tracking outputs still arrive through the returned
archive, and `result_received` precedes extraction and checks of the tracking report.
197 changes: 197 additions & 0 deletions scripts/tests/test_tracking_submission.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,197 @@
"""Exercise the tracking local entrypoint with offline Modal call doubles."""

import io
import json
import sys
import tarfile
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import Mock

import pytest
from click.testing import CliRunner
from modal import config as modal_config
from modal.cli.entry_point import entrypoint_cli

ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT))
from worker import modal_multiperson
from worker.stages import lhm_recovery
from worker.stages.worker_resources import TIMEOUT_ENV, inference_resources


@pytest.fixture
def launch(tmp_path, monkeypatch):
video = tmp_path / "video.mp4"
video.write_bytes(b"video bytes")
cameras = tmp_path / "cameras.json"
cameras.write_bytes(b"camera bytes")
destination = tmp_path / "tracks"
function = Mock()
function.remote.side_effect = AssertionError("tracking must use spawn")
call = function.spawn.return_value
call.object_id = "fc-track"
monkeypatch.setattr(modal_multiperson, "track", function)
monkeypatch.setattr(modal_multiperson, "app", SimpleNamespace(app_id="ap-track"))
monkeypatch.setattr(
modal_multiperson,
"INFERENCE_RESOURCES",
inference_resources(timeout=3600, memory=65536, environment={TIMEOUT_ENV: "600"}),
)
monkeypatch.setattr(modal_config, "_user_config", {"local-alias": {}, "cli-alias": {}})
monkeypatch.setattr(modal_config, "_profile", "local-alias")
monkeypatch.setenv("MODAL_PROFILE", "local-alias")
for name in (
"MODAL_TOKEN_ID",
"MODAL_TOKEN_SECRET",
"MODAL_OAUTH_REFRESH_TOKEN",
"MODAL_OAUTH_CLIENT_ID",
"MODAL_OAUTH_CLIENT_SECRET",
):
monkeypatch.delenv(name, raising=False)

def run(**overrides):
arguments = {"video": str(video), "out": str(destination), "cameras": str(cameras)}
modal_multiperson.main(**(arguments | overrides))

def receipt():
return json.loads((destination / "tracking-call.json").read_text())

return SimpleNamespace(
run=run, receipt=receipt, destination=destination, function=function, call=call
)


def tracking_output(error=None):
buffer = io.BytesIO()
with tarfile.open(fileobj=buffer, mode="w:gz") as archive:
log = b"tracking evidence\n"
entry = tarfile.TarInfo("track.log")
entry.size = len(log)
archive.addfile(entry, io.BytesIO(log))
return {"archive": buffer.getvalue(), "report": {"error": error}}


def test_tracking_receipt_is_durable_before_spawn_and_wait(launch, monkeypatch):
monkeypatch.setitem(sys.modules, "worker.modal_motion", None)
cli = CliRunner().invoke(entrypoint_cli, ["--profile", "cli-alias", "run", "--help"])
assert cli.exit_code == 0, cli.output
initial = {
"schema": "wander.tracking-call/1",
"status": "submitting",
"appId": "ap-track",
"modalProfile": "cli-alias",
"functionCallId": None,
"resultTimeoutSeconds": 1200,
}

def spawn(*args):
assert launch.receipt() == initial
return launch.call

def get(*, timeout):
assert timeout == 1200
assert launch.receipt() == initial | {"status": "waiting", "functionCallId": "fc-track"}
return tracking_output()

launch.function.spawn.side_effect = spawn
launch.call.get.side_effect = get
launch.run(fps=15.0, det_thresh=0.3, gate=0.4, max_gap=4, min_track_samples=6, overlay_every=2)
launch.function.spawn.assert_called_once_with(
b"video bytes", b"camera bytes", 15.0, 0.3, 0.4, 4, 6, 2
)
launch.call.cancel.assert_not_called()
assert launch.receipt() == initial | {"status": "result_received", "functionCallId": "fc-track"}
assert (launch.destination / "track.log").read_bytes() == b"tracking evidence\n"
assert json.loads((launch.destination / "modal-run.json").read_text()) == {"error": None}


def test_tracking_default_wait_and_reported_inference_failure(launch, monkeypatch):
monkeypatch.setattr(
modal_multiperson,
"INFERENCE_RESOURCES",
inference_resources(timeout=3600, memory=65536, environment={}),
)
monkeypatch.setenv("MODAL_TOKEN_ID", "synthetic-override")
launch.call.get.return_value = tracking_output("inference failed")
with pytest.raises(RuntimeError, match="see the downloaded track.log"):
launch.run(cameras="")
launch.function.spawn.assert_called_once_with(b"video bytes", None, 12, 0.15, 0.22, 8, 8, 0)
launch.call.get.assert_called_once_with(timeout=7200)
launch.call.cancel.assert_not_called()
assert launch.receipt()["status"] == "result_received"
assert launch.receipt()["modalProfile"] is None
assert (launch.destination / "track.log").is_file()
assert json.loads((launch.destination / "modal-run.json").read_text()) == {
"error": "inference failed"
}


@pytest.mark.parametrize(
"error, cancel_fails, status, raised",
[
(TimeoutError(), False, "wait_timeout", RuntimeError),
(TimeoutError(), True, "wait_timeout", RuntimeError),
(KeyboardInterrupt(), False, "wait_interrupted", KeyboardInterrupt),
],
)
def test_wait_failure_cancels_only_owned_call_and_blocks_repeat(
launch, error, cancel_fails, status, raised
):
launch.call.get.side_effect = error
if cancel_fails:
launch.call.cancel.side_effect = OSError("private cancellation response")
with pytest.raises(raised) as failure:
launch.run()
if isinstance(error, TimeoutError):
assert failure.value.__cause__ is error
assert "do not resubmit" in str(failure.value)
launch.call.cancel.assert_called_once_with()
receipt = launch.receipt()
assert receipt["status"] == status
assert receipt["appId"] == "ap-track"
assert receipt["modalProfile"] == "local-alias"
assert receipt["functionCallId"] == "fc-track"
assert receipt["cancellation"] == ("unknown" if cancel_fails else "requested")
assert "private cancellation response" not in json.dumps(receipt)
with pytest.raises(FileExistsError):
launch.run()
launch.function.spawn.assert_called_once()
assert launch.receipt() == receipt


def test_uncertain_submission_blocks_repeat_without_cancelling_unknown_call(launch):
launch.function.spawn.side_effect = OSError("lost submission response")
with pytest.raises(OSError, match="lost submission response"):
launch.run()
receipt = launch.receipt()
assert receipt["status"] == "submission_uncertain"
assert receipt["appId"] == "ap-track"
assert receipt["modalProfile"] == "local-alias"
assert receipt["functionCallId"] is None
with pytest.raises(FileExistsError):
launch.run()
launch.function.spawn.assert_called_once()
launch.call.get.assert_not_called()
launch.call.cancel.assert_not_called()
assert launch.receipt() == receipt


def test_receipt_write_failure_after_spawn_cancels_before_waiting(launch, monkeypatch):
atomic_json = lhm_recovery.atomic_json

def write(path, receipt):
if receipt["status"] == "waiting":
raise OSError("disk write failed")
atomic_json(path, receipt)

monkeypatch.setattr(lhm_recovery, "atomic_json", write)
with pytest.raises(OSError, match="disk write failed"):
launch.run()
launch.function.spawn.assert_called_once()
launch.call.get.assert_not_called()
launch.call.cancel.assert_called_once_with()
receipt = launch.receipt()
assert receipt["status"] == "wait_interrupted"
assert receipt["functionCallId"] == "fc-track"
assert receipt["cancellation"] == "requested"
22 changes: 1 addition & 21 deletions worker/modal_motion.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@

from __future__ import annotations

import os
from pathlib import Path
import modal

if modal.is_local():
from worker.stages.modal_profile import effective_modal_profile as _effective_modal_profile
from worker.stages.worker_resources import inference_resources
else:
from worker_resources import inference_resources
Expand Down Expand Up @@ -179,26 +179,6 @@ def reconstruct(
return dict(report=report, archive=buf.getvalue())


def _effective_modal_profile() -> str | None:
from modal import config as modal_config

# Modal 1.5.5 updates this selection for CLI --profile before entering main.
profile = modal_config._profile
if profile not in modal_config.config_profiles():
return None
# The SDK resolves credential settings from the environment before the profile.
credential_overrides = (
"MODAL_TOKEN_ID",
"MODAL_TOKEN_SECRET",
"MODAL_OAUTH_REFRESH_TOKEN",
"MODAL_OAUTH_CLIENT_ID",
"MODAL_OAUTH_CLIENT_SECRET",
)
if any(name in os.environ for name in credential_overrides):
return None
return profile


@app.local_entrypoint()
def main(
video: str, out: str, experiment: str = "vdpm", batch: int = 16, anchor_lock: bool = False
Expand Down
56 changes: 55 additions & 1 deletion worker/modal_multiperson.py
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,59 @@ def track(
return {"report": report, "archive": buf.getvalue()}


def _tracking_result(dest, *inputs):
from worker.stages.lhm_recovery import atomic_json
from worker.stages.modal_profile import effective_modal_profile

execution_timeout = INFERENCE_RESOURCES["timeout"]
# Modal 1.5.5 uses the execution timeout when startup_timeout is omitted.
timeout = INFERENCE_RESOURCES.get("startup_timeout", execution_timeout) + execution_timeout
path = dest / "tracking-call.json"
receipt = {
"schema": "wander.tracking-call/1",
"status": "submitting",
"appId": app.app_id,
"modalProfile": effective_modal_profile(),
"functionCallId": None,
"resultTimeoutSeconds": timeout,
}
# Reserve this destination exclusively even if submission never returns a call ID.
with path.open("x") as stream:
stream.write("{}\n")
atomic_json(path, receipt)
try:
call = track.spawn(*inputs)
except BaseException:
receipt["status"] = "submission_uncertain"
atomic_json(path, receipt)
raise

try:
receipt.update(functionCallId=call.object_id, status="waiting")
atomic_json(path, receipt)
result = call.get(timeout=timeout)
except BaseException as error:
timed_out = isinstance(error, TimeoutError)
receipt["status"] = "wait_timeout" if timed_out else "wait_interrupted"
try:
call.cancel()
receipt["cancellation"] = "requested"
except BaseException as cancellation_error:
receipt["cancellation"] = "unknown"
receipt["cancellationErrorType"] = type(cancellation_error).__name__
atomic_json(path, receipt)
if timed_out:
raise RuntimeError(
f"Tracking result wait timed out after {timeout}s; cancellation "
f"{receipt['cancellation']}. Inspect {path} before any recovery; do not resubmit."
) from error
raise

receipt["status"] = "result_received"
atomic_json(path, receipt)
return result


@app.local_entrypoint()
def main(
video: str,
Expand All @@ -246,7 +299,8 @@ def main(

dest = Path(out)
dest.mkdir(parents=True, exist_ok=True)
result = track.remote(
result = _tracking_result(
dest,
Path(video).read_bytes(),
Path(cameras).read_bytes() if cameras else None,
fps,
Expand Down
23 changes: 23 additions & 0 deletions worker/stages/modal_profile.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
"""Resolve local Modal profile aliases for provider call receipts."""

import os


def effective_modal_profile() -> str | None:
from modal import config as modal_config

# Modal 1.5.5 updates this selection for CLI --profile before entering main.
profile = modal_config._profile
if profile not in modal_config.config_profiles():
return None
# The SDK resolves credential settings from the environment before the profile.
credential_overrides = (
"MODAL_TOKEN_ID",
"MODAL_TOKEN_SECRET",
"MODAL_OAUTH_REFRESH_TOKEN",
"MODAL_OAUTH_CLIENT_ID",
"MODAL_OAUTH_CLIENT_SECRET",
)
if any(name in os.environ for name in credential_overrides):
return None
return profile
Loading