Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions docs/paid-recovery.md
Original file line number Diff line number Diff line change
Expand Up @@ -650,3 +650,16 @@ Submission, cancellation and extraction are outside that wait. A timeout or inte
requests cancellation of the owned call and records whether the request outcome is known.
This receipt tracks the provider call. Tracking outputs still arrive through the returned
archive, and `result_received` precedes extraction and checks of the tracking report.

### Bounded avatar result waits

Frozen and tracked-avatar entrypoints record the owning app/profile and a local result deadline
before submission. The wait budget is the effective startup timeout plus execution timeout;
shortened tracked-animation execution deadlines shorten the corresponding wait. This does not
increase the provider deadline, retry allowance or spending budget.

On timeout or interruption the wrapper requests cancellation of only the call it submitted and
retains the recovery receipt. A cancellation request does not certify termination or a complete
checkpoint: inspect provider state and use the existing recovery path. Neither a wait failure
nor a missing checkpoint authorizes resubmission. The result wait still excludes image builds
and final output download; retain an overall operator deadline.
3 changes: 3 additions & 0 deletions scripts/stage_attempts.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,8 @@ def _read_authorization(path):
"worker/wander_worker/source_timing.py",
),
"worker/modal_lhm.py": (
"worker/stages/lhm_recovery.py",
"worker/stages/modal_profile.py",
"worker/stages/worker_resources.py",
"worker/stages/lhm_execution.py",
"worker/stages/lhm_person.py",
Expand All @@ -152,6 +154,7 @@ def _read_authorization(path):
"worker/stages/person_exclusions.py",
),
"worker/modal_multiperson.py::animate": (
"worker/stages/modal_profile.py",
"worker/stages/worker_resources.py",
"worker/stages/lhm_execution.py",
"worker/stages/lhm_recovery.py",
Expand Down
225 changes: 224 additions & 1 deletion scripts/tests/test_lhm_recovery.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,220 @@ def interrupt():
self.assertEqual(recovered["status"], "complete")
function.spawn.assert_called_once()
call.get.assert_called_once()
call.cancel.assert_not_called()

def test_bounded_wait_ownership_is_durable_before_spawn_and_get(self):
function = Mock()
call = Mock(object_id="fc-owned")
result = {
"recoveryId": self.receipt["recoveryId"],
"remotePath": self.receipt["remotePath"],
"manifestSha256": "a" * 64,
}

def spawn(inputs, **kwargs):
receipt = recovery.load_receipt(self.receipt_path)
self.assertEqual(receipt["status"], "submitting")
self.assertIsNone(receipt["functionCallId"])
self.assertEqual(receipt["appId"], "ap-owned")
self.assertEqual(receipt["modalProfile"], "test-profile")
self.assertEqual(receipt["resultTimeoutSeconds"], 120)
self.assertEqual(kwargs, {"recovery_id": self.receipt["recoveryId"], "animate": True})
return call

def get(*, timeout):
receipt = recovery.load_receipt(self.receipt_path)
self.assertEqual(receipt["functionCallId"], "fc-owned")
self.assertEqual(receipt["status"], "waiting")
self.assertEqual(timeout, 120)
return result

function.spawn.side_effect = spawn
call.get.side_effect = get
self.assertEqual(
recovery.submit_once(
function,
self.receipt_path,
{},
result_timeout_seconds=120,
app_id="ap-owned",
modal_profile="test-profile",
animate=True,
),
result,
)
function.spawn.assert_called_once()
call.get.assert_called_once_with(timeout=120)
call.cancel.assert_not_called()

def test_invalid_bounded_timeout_never_spawns(self):
function = Mock()
for value in (True, False, 0, -1, 1.0, 1.5, "120", float("nan"), float("inf")):
with (
self.subTest(value=value),
self.assertRaisesRegex(ValueError, "positive finite integer"),
):
recovery.submit_once(function, self.receipt_path, {}, result_timeout_seconds=value)
self.assertEqual(recovery.load_receipt(self.receipt_path)["status"], "prepared")
function.spawn.assert_not_called()
function.spawn.return_value.cancel.assert_not_called()

def test_bounded_failure_cancels_only_owned_call_and_preserves_recovery(self):
self.write_frozen()
self.finish()
for i, error in enumerate(
(
TimeoutError("secret timeout"),
KeyboardInterrupt("secret interrupt"),
SystemExit("secret exit"),
OSError("secret transfer"),
)
):
with self.subTest(error=type(error).__name__):
receipt_path = self.root / "local" / recovery.RECEIPT_NAME
# Independent test case, restoring the original prepared receipt only in this fixture.
recovery.atomic_json(receipt_path, self.receipt)
function = Mock()
call = function.spawn.return_value
call.object_id = f"fc-owned-{i}"
call.get.side_effect = error
with self.assertRaises(type(error)):
recovery.submit_once(function, receipt_path, {}, result_timeout_seconds=60)
receipt = recovery.load_receipt(receipt_path)
self.assertEqual(
receipt["status"],
"wait_timeout" if isinstance(error, TimeoutError) else "wait_interrupted",
)
self.assertEqual(receipt["cancellation"], "requested")
self.assertEqual(receipt["waitErrorType"], type(error).__name__)
self.assertNotIn("secret", receipt_path.read_text())
call.cancel.assert_called_once_with()
call.get.assert_called_once_with(timeout=60)
with self.assertRaisesRegex(ValueError, "cannot resubmit"):
recovery.submit_once(function, receipt_path, {}, result_timeout_seconds=60)
function.spawn.assert_called_once()
self.assertTrue((self.output / "person-posed.ply").exists())
self.assertEqual(
recovery.recover_outputs(receipt_path, self.volume)["status"], "complete"
)

def test_cancellation_error_records_class_only_and_keeps_original_exception(self):
function = Mock()
call = function.spawn.return_value
call.object_id = "fc-owned"
original = TimeoutError("private wait details")
call.get.side_effect = original
call.cancel.side_effect = OSError("private cancellation details")
with self.assertRaises(TimeoutError) as caught:
recovery.submit_once(function, self.receipt_path, {}, result_timeout_seconds=60)
self.assertIs(caught.exception, original)
receipt = recovery.load_receipt(self.receipt_path)
self.assertEqual(receipt["status"], "wait_timeout")
self.assertEqual(receipt["cancellation"], "error")
self.assertEqual(receipt["cancellationErrorType"], "OSError")
self.assertNotIn("private wait details", self.receipt_path.read_text())
self.assertNotIn("private cancellation details", self.receipt_path.read_text())
call.cancel.assert_called_once()

def test_failure_before_spawn_never_cancels(self):
function = Mock()
with patch.object(recovery, "atomic_json", side_effect=OSError("storage failure")):
with self.assertRaises(OSError):
recovery.submit_once(function, self.receipt_path, {}, result_timeout_seconds=60)
function.spawn.assert_not_called()
function.spawn.side_effect = OSError("uncertain spawn")
with self.assertRaises(OSError):
recovery.submit_once(function, self.receipt_path, {}, result_timeout_seconds=60)
function.spawn.return_value.cancel.assert_not_called()
self.assertEqual(recovery.load_receipt(self.receipt_path)["status"], "submission_uncertain")
with self.assertRaisesRegex(ValueError, "cannot resubmit"):
recovery.submit_once(function, self.receipt_path, {}, result_timeout_seconds=60)
function.spawn.assert_called_once()

def test_post_spawn_receipt_failure_cancels_before_wait_even_without_timeout(self):
function = Mock()
call = function.spawn.return_value
call.object_id = "fc-owned"
real_write = recovery.atomic_json

def write(path, receipt):
if receipt["status"] == "waiting":
raise OSError("receipt write failed")
real_write(path, receipt)

with patch.object(recovery, "atomic_json", side_effect=write), self.assertRaises(OSError):
recovery.submit_once(function, self.receipt_path, {})
call.get.assert_not_called()
call.cancel.assert_called_once()
receipt = recovery.load_receipt(self.receipt_path)
self.assertEqual(receipt["status"], "wait_interrupted")
self.assertEqual(receipt["functionCallId"], "fc-owned")
self.assertEqual(receipt["cancellation"], "requested")
with self.assertRaisesRegex(ValueError, "cannot resubmit"):
recovery.submit_once(function, self.receipt_path, {})

def test_persistent_receipt_failure_still_cancels_and_blocks_resubmit(self):
function = Mock()
call = function.spawn.return_value
call.object_id = "fc-owned"
real_write = recovery.atomic_json

def write(path, receipt):
if receipt["status"] != "submitting":
raise OSError("storage unavailable")
real_write(path, receipt)

with patch.object(recovery, "atomic_json", side_effect=write), self.assertRaises(OSError):
recovery.submit_once(function, self.receipt_path, {}, result_timeout_seconds=60)
call.cancel.assert_called_once()
call.get.assert_not_called()
self.assertEqual(recovery.load_receipt(self.receipt_path)["status"], "submitting")
with self.assertRaisesRegex(ValueError, "cannot resubmit"):
recovery.submit_once(function, self.receipt_path, {})

def test_legacy_success_has_unbounded_get_and_no_new_receipt_fields(self):
function = Mock()
call = function.spawn.return_value
call.object_id = "fc-legacy"
call.get.return_value = {
"recoveryId": self.receipt["recoveryId"],
"remotePath": self.receipt["remotePath"],
"manifestSha256": "a" * 64,
}
recovery.submit_once(function, self.receipt_path, {})
call.get.assert_called_once_with()
call.cancel.assert_not_called()
receipt = recovery.load_receipt(self.receipt_path)
for field in ("resultTimeoutSeconds", "appId", "modalProfile", "cancellation"):
self.assertNotIn(field, receipt)

def test_result_receipt_failure_keeps_outputs_and_cancels_owned_call(self):
self.write_frozen()
self.finish()
function = Mock()
call = function.spawn.return_value
call.object_id = "fc-owned"
call.get.return_value = {
"recoveryId": self.receipt["recoveryId"],
"remotePath": self.receipt["remotePath"],
"manifestSha256": recovery.file_hash(self.checkpoint / "manifest.json"),
}
real_write = recovery.atomic_json

def write(path, receipt):
if receipt["status"] == "checkpoint_available":
raise OSError("receipt storage failed")
real_write(path, receipt)

with patch.object(recovery, "atomic_json", side_effect=write), self.assertRaises(OSError):
recovery.submit_once(function, self.receipt_path, {}, result_timeout_seconds=60)
call.cancel.assert_called_once()
call.get.assert_called_once_with(timeout=60)
self.assertEqual(recovery.load_receipt(self.receipt_path)["status"], "wait_interrupted")
self.assertEqual(
recovery.recover_outputs(self.receipt_path, self.volume)["status"], "complete"
)
function.spawn.assert_called_once()

def test_uncertain_spawn_and_unreadable_receipt_never_retry(self):
function = Mock()
Expand Down Expand Up @@ -401,7 +615,16 @@ def spawn(inputs, **kwargs):
"manifestSha256": recovery.file_hash(directory / "manifest.json"),
"report": {"error": None},
}
return types.SimpleNamespace(object_id="fc-test", get=lambda: result)

def get(*, timeout):
resources = modal_lhm.INFERENCE_RESOURCES
self.assertEqual(
timeout,
resources.get("startup_timeout", resources["timeout"]) + resources["timeout"],
)
return result

return types.SimpleNamespace(object_id="fc-test", get=get)

function.spawn.side_effect = spawn
destination = self.root / "cli-output"
Expand Down
55 changes: 55 additions & 0 deletions scripts/tests/test_lhm_wait_entrypoints.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
"""Local entrypoints must bound waits without changing inference input arguments."""

import sys
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import Mock

import pytest

sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from worker import modal_lhm, modal_multiperson
from worker.stages import lhm_recovery, modal_profile


@pytest.mark.parametrize(
"mode,expected", [("frozen", 1200), ("motion", 1200), ("short_motion", 900)]
)
def test_entrypoint_records_owner_and_effective_deadline(tmp_path, monkeypatch, mode, expected):
for name in (
"source.png",
"mask.png",
"prepared.json",
"source.mp4",
"canonical-state.pt",
"result.json",
"cameras.json",
"frame_000.ply",
"source-poses.pt",
"motion.json",
):
(tmp_path / name).write_bytes(name.encode())
submit = Mock(side_effect=RuntimeError("stop before provider"))
monkeypatch.setattr(lhm_recovery, "submit_once", submit)
monkeypatch.setattr(modal_profile, "effective_modal_profile", lambda: "owner-profile")
module = modal_lhm if mode == "frozen" else modal_multiperson
monkeypatch.setattr(module, "app", SimpleNamespace(app_id="ap-owned"))
monkeypatch.setattr(module, "INFERENCE_RESOURCES", {"timeout": 600, "startup_timeout": 600})
with pytest.raises(RuntimeError, match="stop before provider"):
if mode == "frozen":
module.main(prepared=str(tmp_path), out=str(tmp_path / "output"))
else:
module.animate(
video=str(tmp_path / "source.mp4"),
canonical=str(tmp_path),
cameras=str(tmp_path / "cameras.json"),
track_dir=str(tmp_path),
out=str(tmp_path / "output"),
execution_timeout=300 if mode == "short_motion" else 0,
)
kwargs = submit.call_args.kwargs
assert kwargs["result_timeout_seconds"] == expected
assert kwargs["app_id"] == "ap-owned"
assert kwargs["modal_profile"] == "owner-profile"
if mode != "frozen":
assert kwargs["execution_timeout"] == (300 if mode == "short_motion" else 0)
16 changes: 14 additions & 2 deletions scripts/tests/test_multiperson_recovery.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,9 @@ def test_interrupted_wait_records_identity_and_repeat_cli_never_resubmits(self):
call = bounded.spawn.return_value
call.object_id = "fc-retained-track"

def interrupt():
def interrupt(*, timeout):
resources = modal_multiperson.INFERENCE_RESOURCES
self.assertEqual(timeout, resources.get("startup_timeout", 600) + 600)
receipt = lhm_recovery.load_receipt(self.destination / lhm_recovery.RECEIPT_NAME)
self.assertEqual(receipt["status"], "waiting")
self.assertEqual(receipt["functionCallId"], call.object_id)
Expand All @@ -122,6 +124,7 @@ def interrupt():
timeout=600, cpu=(4, 4), memory=(65536, 65536), retries=0, max_containers=1
)
bounded.spawn.assert_called_once()
call.cancel.assert_called_once_with()
self.assertEqual(
bounded.spawn.call_args.kwargs["flags"],
["--track-id", "0", "--fixed-world-scale", "1.9"],
Expand Down Expand Up @@ -187,7 +190,16 @@ def spawn(inputs, **kwargs):
"manifestSha256": lhm_recovery.file_hash(directory / "manifest.json"),
"report": {"error": None},
}
return types.SimpleNamespace(object_id="fc-partial", get=lambda: result)

def get(*, timeout):
resources = modal_multiperson.INFERENCE_RESOURCES
self.assertEqual(
timeout,
resources.get("startup_timeout", resources["timeout"]) + resources["timeout"],
)
return result

return types.SimpleNamespace(object_id="fc-partial", get=get)

function.spawn.side_effect = spawn
with (
Expand Down
13 changes: 12 additions & 1 deletion worker/modal_lhm.py
Original file line number Diff line number Diff line change
Expand Up @@ -441,6 +441,7 @@ def main(
import tarfile

from worker.stages.lhm_recovery import new_receipt, recover_outputs, submit_once
from worker.stages.modal_profile import effective_modal_profile

if not out:
raise ValueError("LHM output destination is required")
Expand Down Expand Up @@ -482,7 +483,17 @@ def main(
if larger_model and not fixed_inputs:
raise ValueError("Larger checkpoint comparison requires saved fixed inputs")
result = submit_once(
frozen, receipt_path, inputs, animate=bool(canonical), larger_model=larger_model
frozen,
receipt_path,
inputs,
result_timeout_seconds=INFERENCE_RESOURCES.get(
"startup_timeout", INFERENCE_RESOURCES["timeout"]
)
+ INFERENCE_RESOURCES["timeout"],
app_id=app.app_id,
modal_profile=effective_modal_profile(),
animate=bool(canonical),
larger_model=larger_model,
)
manifest = recover_outputs(receipt_path, cache)
# Preserve the normal archive artifact, without GPU compression or a giant return.
Expand Down
Loading
Loading