diff --git a/docs/paid-recovery.md b/docs/paid-recovery.md index f2a2863..88678cf 100644 --- a/docs/paid-recovery.md +++ b/docs/paid-recovery.md @@ -650,3 +650,16 @@ Submission, cancellation and extraction are outside that wait. A timeout or inte requests cancellation of the owned call and records whether the request outcome is known. This receipt tracks the provider call. Tracking outputs still arrive through the returned archive, and `result_received` precedes extraction and checks of the tracking report. + +### Bounded avatar result waits + +Frozen and tracked-avatar entrypoints record the owning app/profile and a local result deadline +before submission. The wait budget is the effective startup timeout plus execution timeout; +shortened tracked-animation execution deadlines shorten the corresponding wait. This does not +increase the provider deadline, retry allowance or spending budget. + +On timeout or interruption the wrapper requests cancellation of only the call it submitted and +retains the recovery receipt. A cancellation request does not certify termination or a complete +checkpoint: inspect provider state and use the existing recovery path. Neither a wait failure +nor a missing checkpoint authorizes resubmission. The result wait still excludes image builds +and final output download; retain an overall operator deadline. diff --git a/scripts/stage_attempts.py b/scripts/stage_attempts.py index a9e1270..7260c56 100644 --- a/scripts/stage_attempts.py +++ b/scripts/stage_attempts.py @@ -138,6 +138,8 @@ def _read_authorization(path): "worker/wander_worker/source_timing.py", ), "worker/modal_lhm.py": ( + "worker/stages/lhm_recovery.py", + "worker/stages/modal_profile.py", "worker/stages/worker_resources.py", "worker/stages/lhm_execution.py", "worker/stages/lhm_person.py", @@ -152,6 +154,7 @@ def _read_authorization(path): "worker/stages/person_exclusions.py", ), "worker/modal_multiperson.py::animate": ( + "worker/stages/modal_profile.py", "worker/stages/worker_resources.py", "worker/stages/lhm_execution.py", "worker/stages/lhm_recovery.py", diff --git a/scripts/tests/test_lhm_recovery.py b/scripts/tests/test_lhm_recovery.py index 10a3305..cb8aa37 100644 --- a/scripts/tests/test_lhm_recovery.py +++ b/scripts/tests/test_lhm_recovery.py @@ -127,6 +127,220 @@ def interrupt(): self.assertEqual(recovered["status"], "complete") function.spawn.assert_called_once() call.get.assert_called_once() + call.cancel.assert_not_called() + + def test_bounded_wait_ownership_is_durable_before_spawn_and_get(self): + function = Mock() + call = Mock(object_id="fc-owned") + result = { + "recoveryId": self.receipt["recoveryId"], + "remotePath": self.receipt["remotePath"], + "manifestSha256": "a" * 64, + } + + def spawn(inputs, **kwargs): + receipt = recovery.load_receipt(self.receipt_path) + self.assertEqual(receipt["status"], "submitting") + self.assertIsNone(receipt["functionCallId"]) + self.assertEqual(receipt["appId"], "ap-owned") + self.assertEqual(receipt["modalProfile"], "test-profile") + self.assertEqual(receipt["resultTimeoutSeconds"], 120) + self.assertEqual(kwargs, {"recovery_id": self.receipt["recoveryId"], "animate": True}) + return call + + def get(*, timeout): + receipt = recovery.load_receipt(self.receipt_path) + self.assertEqual(receipt["functionCallId"], "fc-owned") + self.assertEqual(receipt["status"], "waiting") + self.assertEqual(timeout, 120) + return result + + function.spawn.side_effect = spawn + call.get.side_effect = get + self.assertEqual( + recovery.submit_once( + function, + self.receipt_path, + {}, + result_timeout_seconds=120, + app_id="ap-owned", + modal_profile="test-profile", + animate=True, + ), + result, + ) + function.spawn.assert_called_once() + call.get.assert_called_once_with(timeout=120) + call.cancel.assert_not_called() + + def test_invalid_bounded_timeout_never_spawns(self): + function = Mock() + for value in (True, False, 0, -1, 1.0, 1.5, "120", float("nan"), float("inf")): + with ( + self.subTest(value=value), + self.assertRaisesRegex(ValueError, "positive finite integer"), + ): + recovery.submit_once(function, self.receipt_path, {}, result_timeout_seconds=value) + self.assertEqual(recovery.load_receipt(self.receipt_path)["status"], "prepared") + function.spawn.assert_not_called() + function.spawn.return_value.cancel.assert_not_called() + + def test_bounded_failure_cancels_only_owned_call_and_preserves_recovery(self): + self.write_frozen() + self.finish() + for i, error in enumerate( + ( + TimeoutError("secret timeout"), + KeyboardInterrupt("secret interrupt"), + SystemExit("secret exit"), + OSError("secret transfer"), + ) + ): + with self.subTest(error=type(error).__name__): + receipt_path = self.root / "local" / recovery.RECEIPT_NAME + # Independent test case, restoring the original prepared receipt only in this fixture. + recovery.atomic_json(receipt_path, self.receipt) + function = Mock() + call = function.spawn.return_value + call.object_id = f"fc-owned-{i}" + call.get.side_effect = error + with self.assertRaises(type(error)): + recovery.submit_once(function, receipt_path, {}, result_timeout_seconds=60) + receipt = recovery.load_receipt(receipt_path) + self.assertEqual( + receipt["status"], + "wait_timeout" if isinstance(error, TimeoutError) else "wait_interrupted", + ) + self.assertEqual(receipt["cancellation"], "requested") + self.assertEqual(receipt["waitErrorType"], type(error).__name__) + self.assertNotIn("secret", receipt_path.read_text()) + call.cancel.assert_called_once_with() + call.get.assert_called_once_with(timeout=60) + with self.assertRaisesRegex(ValueError, "cannot resubmit"): + recovery.submit_once(function, receipt_path, {}, result_timeout_seconds=60) + function.spawn.assert_called_once() + self.assertTrue((self.output / "person-posed.ply").exists()) + self.assertEqual( + recovery.recover_outputs(receipt_path, self.volume)["status"], "complete" + ) + + def test_cancellation_error_records_class_only_and_keeps_original_exception(self): + function = Mock() + call = function.spawn.return_value + call.object_id = "fc-owned" + original = TimeoutError("private wait details") + call.get.side_effect = original + call.cancel.side_effect = OSError("private cancellation details") + with self.assertRaises(TimeoutError) as caught: + recovery.submit_once(function, self.receipt_path, {}, result_timeout_seconds=60) + self.assertIs(caught.exception, original) + receipt = recovery.load_receipt(self.receipt_path) + self.assertEqual(receipt["status"], "wait_timeout") + self.assertEqual(receipt["cancellation"], "error") + self.assertEqual(receipt["cancellationErrorType"], "OSError") + self.assertNotIn("private wait details", self.receipt_path.read_text()) + self.assertNotIn("private cancellation details", self.receipt_path.read_text()) + call.cancel.assert_called_once() + + def test_failure_before_spawn_never_cancels(self): + function = Mock() + with patch.object(recovery, "atomic_json", side_effect=OSError("storage failure")): + with self.assertRaises(OSError): + recovery.submit_once(function, self.receipt_path, {}, result_timeout_seconds=60) + function.spawn.assert_not_called() + function.spawn.side_effect = OSError("uncertain spawn") + with self.assertRaises(OSError): + recovery.submit_once(function, self.receipt_path, {}, result_timeout_seconds=60) + function.spawn.return_value.cancel.assert_not_called() + self.assertEqual(recovery.load_receipt(self.receipt_path)["status"], "submission_uncertain") + with self.assertRaisesRegex(ValueError, "cannot resubmit"): + recovery.submit_once(function, self.receipt_path, {}, result_timeout_seconds=60) + function.spawn.assert_called_once() + + def test_post_spawn_receipt_failure_cancels_before_wait_even_without_timeout(self): + function = Mock() + call = function.spawn.return_value + call.object_id = "fc-owned" + real_write = recovery.atomic_json + + def write(path, receipt): + if receipt["status"] == "waiting": + raise OSError("receipt write failed") + real_write(path, receipt) + + with patch.object(recovery, "atomic_json", side_effect=write), self.assertRaises(OSError): + recovery.submit_once(function, self.receipt_path, {}) + call.get.assert_not_called() + call.cancel.assert_called_once() + receipt = recovery.load_receipt(self.receipt_path) + self.assertEqual(receipt["status"], "wait_interrupted") + self.assertEqual(receipt["functionCallId"], "fc-owned") + self.assertEqual(receipt["cancellation"], "requested") + with self.assertRaisesRegex(ValueError, "cannot resubmit"): + recovery.submit_once(function, self.receipt_path, {}) + + def test_persistent_receipt_failure_still_cancels_and_blocks_resubmit(self): + function = Mock() + call = function.spawn.return_value + call.object_id = "fc-owned" + real_write = recovery.atomic_json + + def write(path, receipt): + if receipt["status"] != "submitting": + raise OSError("storage unavailable") + real_write(path, receipt) + + with patch.object(recovery, "atomic_json", side_effect=write), self.assertRaises(OSError): + recovery.submit_once(function, self.receipt_path, {}, result_timeout_seconds=60) + call.cancel.assert_called_once() + call.get.assert_not_called() + self.assertEqual(recovery.load_receipt(self.receipt_path)["status"], "submitting") + with self.assertRaisesRegex(ValueError, "cannot resubmit"): + recovery.submit_once(function, self.receipt_path, {}) + + def test_legacy_success_has_unbounded_get_and_no_new_receipt_fields(self): + function = Mock() + call = function.spawn.return_value + call.object_id = "fc-legacy" + call.get.return_value = { + "recoveryId": self.receipt["recoveryId"], + "remotePath": self.receipt["remotePath"], + "manifestSha256": "a" * 64, + } + recovery.submit_once(function, self.receipt_path, {}) + call.get.assert_called_once_with() + call.cancel.assert_not_called() + receipt = recovery.load_receipt(self.receipt_path) + for field in ("resultTimeoutSeconds", "appId", "modalProfile", "cancellation"): + self.assertNotIn(field, receipt) + + def test_result_receipt_failure_keeps_outputs_and_cancels_owned_call(self): + self.write_frozen() + self.finish() + function = Mock() + call = function.spawn.return_value + call.object_id = "fc-owned" + call.get.return_value = { + "recoveryId": self.receipt["recoveryId"], + "remotePath": self.receipt["remotePath"], + "manifestSha256": recovery.file_hash(self.checkpoint / "manifest.json"), + } + real_write = recovery.atomic_json + + def write(path, receipt): + if receipt["status"] == "checkpoint_available": + raise OSError("receipt storage failed") + real_write(path, receipt) + + with patch.object(recovery, "atomic_json", side_effect=write), self.assertRaises(OSError): + recovery.submit_once(function, self.receipt_path, {}, result_timeout_seconds=60) + call.cancel.assert_called_once() + call.get.assert_called_once_with(timeout=60) + self.assertEqual(recovery.load_receipt(self.receipt_path)["status"], "wait_interrupted") + self.assertEqual( + recovery.recover_outputs(self.receipt_path, self.volume)["status"], "complete" + ) + function.spawn.assert_called_once() def test_uncertain_spawn_and_unreadable_receipt_never_retry(self): function = Mock() @@ -401,7 +615,16 @@ def spawn(inputs, **kwargs): "manifestSha256": recovery.file_hash(directory / "manifest.json"), "report": {"error": None}, } - return types.SimpleNamespace(object_id="fc-test", get=lambda: result) + + def get(*, timeout): + resources = modal_lhm.INFERENCE_RESOURCES + self.assertEqual( + timeout, + resources.get("startup_timeout", resources["timeout"]) + resources["timeout"], + ) + return result + + return types.SimpleNamespace(object_id="fc-test", get=get) function.spawn.side_effect = spawn destination = self.root / "cli-output" diff --git a/scripts/tests/test_lhm_wait_entrypoints.py b/scripts/tests/test_lhm_wait_entrypoints.py new file mode 100644 index 0000000..d8710a3 --- /dev/null +++ b/scripts/tests/test_lhm_wait_entrypoints.py @@ -0,0 +1,55 @@ +"""Local entrypoints must bound waits without changing inference input arguments.""" + +import sys +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from worker import modal_lhm, modal_multiperson +from worker.stages import lhm_recovery, modal_profile + + +@pytest.mark.parametrize( + "mode,expected", [("frozen", 1200), ("motion", 1200), ("short_motion", 900)] +) +def test_entrypoint_records_owner_and_effective_deadline(tmp_path, monkeypatch, mode, expected): + for name in ( + "source.png", + "mask.png", + "prepared.json", + "source.mp4", + "canonical-state.pt", + "result.json", + "cameras.json", + "frame_000.ply", + "source-poses.pt", + "motion.json", + ): + (tmp_path / name).write_bytes(name.encode()) + submit = Mock(side_effect=RuntimeError("stop before provider")) + monkeypatch.setattr(lhm_recovery, "submit_once", submit) + monkeypatch.setattr(modal_profile, "effective_modal_profile", lambda: "owner-profile") + module = modal_lhm if mode == "frozen" else modal_multiperson + monkeypatch.setattr(module, "app", SimpleNamespace(app_id="ap-owned")) + monkeypatch.setattr(module, "INFERENCE_RESOURCES", {"timeout": 600, "startup_timeout": 600}) + with pytest.raises(RuntimeError, match="stop before provider"): + if mode == "frozen": + module.main(prepared=str(tmp_path), out=str(tmp_path / "output")) + else: + module.animate( + video=str(tmp_path / "source.mp4"), + canonical=str(tmp_path), + cameras=str(tmp_path / "cameras.json"), + track_dir=str(tmp_path), + out=str(tmp_path / "output"), + execution_timeout=300 if mode == "short_motion" else 0, + ) + kwargs = submit.call_args.kwargs + assert kwargs["result_timeout_seconds"] == expected + assert kwargs["app_id"] == "ap-owned" + assert kwargs["modal_profile"] == "owner-profile" + if mode != "frozen": + assert kwargs["execution_timeout"] == (300 if mode == "short_motion" else 0) diff --git a/scripts/tests/test_multiperson_recovery.py b/scripts/tests/test_multiperson_recovery.py index 255f9ef..b54a465 100644 --- a/scripts/tests/test_multiperson_recovery.py +++ b/scripts/tests/test_multiperson_recovery.py @@ -98,7 +98,9 @@ def test_interrupted_wait_records_identity_and_repeat_cli_never_resubmits(self): call = bounded.spawn.return_value call.object_id = "fc-retained-track" - def interrupt(): + def interrupt(*, timeout): + resources = modal_multiperson.INFERENCE_RESOURCES + self.assertEqual(timeout, resources.get("startup_timeout", 600) + 600) receipt = lhm_recovery.load_receipt(self.destination / lhm_recovery.RECEIPT_NAME) self.assertEqual(receipt["status"], "waiting") self.assertEqual(receipt["functionCallId"], call.object_id) @@ -122,6 +124,7 @@ def interrupt(): timeout=600, cpu=(4, 4), memory=(65536, 65536), retries=0, max_containers=1 ) bounded.spawn.assert_called_once() + call.cancel.assert_called_once_with() self.assertEqual( bounded.spawn.call_args.kwargs["flags"], ["--track-id", "0", "--fixed-world-scale", "1.9"], @@ -187,7 +190,16 @@ def spawn(inputs, **kwargs): "manifestSha256": lhm_recovery.file_hash(directory / "manifest.json"), "report": {"error": None}, } - return types.SimpleNamespace(object_id="fc-partial", get=lambda: result) + + def get(*, timeout): + resources = modal_multiperson.INFERENCE_RESOURCES + self.assertEqual( + timeout, + resources.get("startup_timeout", resources["timeout"]) + resources["timeout"], + ) + return result + + return types.SimpleNamespace(object_id="fc-partial", get=get) function.spawn.side_effect = spawn with ( diff --git a/worker/modal_lhm.py b/worker/modal_lhm.py index db22c39..8b3595d 100644 --- a/worker/modal_lhm.py +++ b/worker/modal_lhm.py @@ -441,6 +441,7 @@ def main( import tarfile from worker.stages.lhm_recovery import new_receipt, recover_outputs, submit_once + from worker.stages.modal_profile import effective_modal_profile if not out: raise ValueError("LHM output destination is required") @@ -482,7 +483,17 @@ def main( if larger_model and not fixed_inputs: raise ValueError("Larger checkpoint comparison requires saved fixed inputs") result = submit_once( - frozen, receipt_path, inputs, animate=bool(canonical), larger_model=larger_model + frozen, + receipt_path, + inputs, + result_timeout_seconds=INFERENCE_RESOURCES.get( + "startup_timeout", INFERENCE_RESOURCES["timeout"] + ) + + INFERENCE_RESOURCES["timeout"], + app_id=app.app_id, + modal_profile=effective_modal_profile(), + animate=bool(canonical), + larger_model=larger_model, ) manifest = recover_outputs(receipt_path, cache) # Preserve the normal archive artifact, without GPU compression or a giant return. diff --git a/worker/modal_multiperson.py b/worker/modal_multiperson.py index 2644461..39910b8 100644 --- a/worker/modal_multiperson.py +++ b/worker/modal_multiperson.py @@ -483,6 +483,7 @@ def animate( from worker.stages.lhm_execution import animation_options from worker.stages.lhm_recovery import atomic_json, new_receipt, recover_outputs, submit_once + from worker.stages.modal_profile import effective_modal_profile scale_flags, options = animation_options(fixed_world_scale, execution_timeout) if options: @@ -528,6 +529,12 @@ def animate( function, receipt_path, inputs, + result_timeout_seconds=INFERENCE_RESOURCES.get( + "startup_timeout", options.get("timeout", INFERENCE_RESOURCES["timeout"]) + ) + + options.get("timeout", INFERENCE_RESOURCES["timeout"]), + app_id=app.app_id, + modal_profile=effective_modal_profile(), flags=flags, execution_timeout=execution_timeout, **({"recover_missing_poses": True} if recover_missing_poses else {}), diff --git a/worker/stages/lhm_recovery.py b/worker/stages/lhm_recovery.py index a138ac8..4dd9b75 100644 --- a/worker/stages/lhm_recovery.py +++ b/worker/stages/lhm_recovery.py @@ -141,17 +141,78 @@ def receipt_lock(receipt_path): yield -def submit_once(function, receipt_path, inputs, **kwargs): +def submit_once( + function, + receipt_path, + inputs, + *, + result_timeout_seconds=None, + app_id=None, + modal_profile=None, + **kwargs, +): """Persist identity before waiting. An uncertain or interrupted receipt never resubmits.""" with receipt_lock(receipt_path): - return _submit_once(function, receipt_path, inputs, **kwargs) + return _submit_once( + function, + receipt_path, + inputs, + result_timeout_seconds=result_timeout_seconds, + app_id=app_id, + modal_profile=modal_profile, + **kwargs, + ) -def _submit_once(function, receipt_path, inputs, **kwargs): +def _record_wait_failure(call, receipt_path, receipt, error, *, cancel, bounded): + receipt["status"] = ( + "wait_timeout" if bounded and isinstance(error, TimeoutError) else "wait_interrupted" + ) + if cancel: + receipt["waitErrorType"] = type(error).__name__ + try: + call.cancel() + receipt["cancellation"] = "requested" + except BaseException as cancellation_error: + receipt["cancellation"] = "error" + receipt["cancellationErrorType"] = type(cancellation_error).__name__ + try: + atomic_json(receipt_path, receipt) + except BaseException: + # Keep the original failure. The earlier submitting/waiting receipt still + # prevents another submission, even when storage cannot record cleanup. + pass + + +def _submit_once( + function, + receipt_path, + inputs, + *, + result_timeout_seconds=None, + app_id=None, + modal_profile=None, + **kwargs, +): + if result_timeout_seconds is not None and ( + isinstance(result_timeout_seconds, bool) + or not isinstance(result_timeout_seconds, int) + or result_timeout_seconds <= 0 + ): + raise ValueError("result_timeout_seconds must be a positive finite integer") receipt = load_receipt(receipt_path) if receipt.get("status") != "prepared" or receipt.get("functionCallId") is not None: raise ValueError("Existing LHM receipt cannot resubmit inference; use recovery") receipt["status"] = "submitting" + if result_timeout_seconds is not None: + receipt.update( + resultTimeoutSeconds=result_timeout_seconds, appId=app_id, modalProfile=modal_profile + ) + else: + if app_id is not None: + receipt["appId"] = app_id + if modal_profile is not None: + receipt["modalProfile"] = modal_profile atomic_json(receipt_path, receipt) try: call = function.spawn(inputs, recovery_id=receipt["recoveryId"], **kwargs) @@ -159,14 +220,28 @@ def _submit_once(function, receipt_path, inputs, **kwargs): receipt["status"] = "submission_uncertain" atomic_json(receipt_path, receipt) raise - receipt["functionCallId"] = call.object_id - receipt["status"] = "waiting" - atomic_json(receipt_path, receipt) try: - result = call.get() - except BaseException: - receipt["status"] = "wait_interrupted" + receipt["functionCallId"] = call.object_id + receipt["status"] = "waiting" atomic_json(receipt_path, receipt) + except BaseException as error: + _record_wait_failure(call, receipt_path, receipt, error, cancel=True, bounded=False) + raise + try: + result = ( + call.get(timeout=result_timeout_seconds) + if result_timeout_seconds is not None + else call.get() + ) + except BaseException as error: + _record_wait_failure( + call, + receipt_path, + receipt, + error, + cancel=result_timeout_seconds is not None, + bounded=result_timeout_seconds is not None, + ) raise if ( not isinstance(result, dict) @@ -178,7 +253,11 @@ def _submit_once(function, receipt_path, inputs, **kwargs): receipt["totalWorkerSeconds"] = result.get("totalWorkerSeconds") receipt["manifestSha256"] = result["manifestSha256"] receipt["status"] = "checkpoint_available" - atomic_json(receipt_path, receipt) + try: + atomic_json(receipt_path, receipt) + except BaseException as error: + _record_wait_failure(call, receipt_path, receipt, error, cancel=True, bounded=False) + raise return result