From e0732f7bad47374067c92d36a8ba19514ed8d46c Mon Sep 17 00:00:00 2001 From: Austin Jian Date: Sat, 21 Nov 2026 00:00:41 -0500 Subject: [PATCH 1/3] fix(worker): bound avatar result waits and retain call ownership --- docs/paid-recovery.md | 13 ++ scripts/stage_attempts.py | 3 + scripts/tests/test_lhm_recovery.py | 225 ++++++++++++++++++++- scripts/tests/test_lhm_wait_entrypoints.py | 55 +++++ scripts/tests/test_multiperson_recovery.py | 16 +- worker/modal_lhm.py | 13 +- worker/modal_multiperson.py | 7 + worker/stages/lhm_recovery.py | 99 ++++++++- 8 files changed, 417 insertions(+), 14 deletions(-) create mode 100644 scripts/tests/test_lhm_wait_entrypoints.py diff --git a/docs/paid-recovery.md b/docs/paid-recovery.md index f2a2863..88678cf 100644 --- a/docs/paid-recovery.md +++ b/docs/paid-recovery.md @@ -650,3 +650,16 @@ Submission, cancellation and extraction are outside that wait. A timeout or inte requests cancellation of the owned call and records whether the request outcome is known. This receipt tracks the provider call. Tracking outputs still arrive through the returned archive, and `result_received` precedes extraction and checks of the tracking report. + +### Bounded avatar result waits + +Frozen and tracked-avatar entrypoints record the owning app/profile and a local result deadline +before submission. The wait budget is the effective startup timeout plus execution timeout; +shortened tracked-animation execution deadlines shorten the corresponding wait. This does not +increase the provider deadline, retry allowance or spending budget. + +On timeout or interruption the wrapper requests cancellation of only the call it submitted and +retains the recovery receipt. A cancellation request does not certify termination or a complete +checkpoint: inspect provider state and use the existing recovery path. Neither a wait failure +nor a missing checkpoint authorizes resubmission. The result wait still excludes image builds +and final output download; retain an overall operator deadline. diff --git a/scripts/stage_attempts.py b/scripts/stage_attempts.py index 71d72f2..d228a26 100644 --- a/scripts/stage_attempts.py +++ b/scripts/stage_attempts.py @@ -138,6 +138,8 @@ def _read_authorization(path): "worker/wander_worker/source_timing.py", ), "worker/modal_lhm.py": ( + "worker/stages/lhm_recovery.py", + "worker/stages/modal_profile.py", "worker/stages/worker_resources.py", "worker/stages/lhm_execution.py", "worker/stages/lhm_person.py", @@ -150,6 +152,7 @@ def _read_authorization(path): "worker/stages/pose_instances.py", ), "worker/modal_multiperson.py::animate": ( + "worker/stages/modal_profile.py", "worker/stages/worker_resources.py", "worker/stages/lhm_execution.py", "worker/stages/lhm_recovery.py", diff --git a/scripts/tests/test_lhm_recovery.py b/scripts/tests/test_lhm_recovery.py index 10a3305..cb8aa37 100644 --- a/scripts/tests/test_lhm_recovery.py +++ b/scripts/tests/test_lhm_recovery.py @@ -127,6 +127,220 @@ def interrupt(): self.assertEqual(recovered["status"], "complete") function.spawn.assert_called_once() call.get.assert_called_once() + call.cancel.assert_not_called() + + def test_bounded_wait_ownership_is_durable_before_spawn_and_get(self): + function = Mock() + call = Mock(object_id="fc-owned") + result = { + "recoveryId": self.receipt["recoveryId"], + "remotePath": self.receipt["remotePath"], + "manifestSha256": "a" * 64, + } + + def spawn(inputs, **kwargs): + receipt = recovery.load_receipt(self.receipt_path) + self.assertEqual(receipt["status"], "submitting") + self.assertIsNone(receipt["functionCallId"]) + self.assertEqual(receipt["appId"], "ap-owned") + self.assertEqual(receipt["modalProfile"], "test-profile") + self.assertEqual(receipt["resultTimeoutSeconds"], 120) + self.assertEqual(kwargs, {"recovery_id": self.receipt["recoveryId"], "animate": True}) + return call + + def get(*, timeout): + receipt = recovery.load_receipt(self.receipt_path) + self.assertEqual(receipt["functionCallId"], "fc-owned") + self.assertEqual(receipt["status"], "waiting") + self.assertEqual(timeout, 120) + return result + + function.spawn.side_effect = spawn + call.get.side_effect = get + self.assertEqual( + recovery.submit_once( + function, + self.receipt_path, + {}, + result_timeout_seconds=120, + app_id="ap-owned", + modal_profile="test-profile", + animate=True, + ), + result, + ) + function.spawn.assert_called_once() + call.get.assert_called_once_with(timeout=120) + call.cancel.assert_not_called() + + def test_invalid_bounded_timeout_never_spawns(self): + function = Mock() + for value in (True, False, 0, -1, 1.0, 1.5, "120", float("nan"), float("inf")): + with ( + self.subTest(value=value), + self.assertRaisesRegex(ValueError, "positive finite integer"), + ): + recovery.submit_once(function, self.receipt_path, {}, result_timeout_seconds=value) + self.assertEqual(recovery.load_receipt(self.receipt_path)["status"], "prepared") + function.spawn.assert_not_called() + function.spawn.return_value.cancel.assert_not_called() + + def test_bounded_failure_cancels_only_owned_call_and_preserves_recovery(self): + self.write_frozen() + self.finish() + for i, error in enumerate( + ( + TimeoutError("secret timeout"), + KeyboardInterrupt("secret interrupt"), + SystemExit("secret exit"), + OSError("secret transfer"), + ) + ): + with self.subTest(error=type(error).__name__): + receipt_path = self.root / "local" / recovery.RECEIPT_NAME + # Independent test case, restoring the original prepared receipt only in this fixture. + recovery.atomic_json(receipt_path, self.receipt) + function = Mock() + call = function.spawn.return_value + call.object_id = f"fc-owned-{i}" + call.get.side_effect = error + with self.assertRaises(type(error)): + recovery.submit_once(function, receipt_path, {}, result_timeout_seconds=60) + receipt = recovery.load_receipt(receipt_path) + self.assertEqual( + receipt["status"], + "wait_timeout" if isinstance(error, TimeoutError) else "wait_interrupted", + ) + self.assertEqual(receipt["cancellation"], "requested") + self.assertEqual(receipt["waitErrorType"], type(error).__name__) + self.assertNotIn("secret", receipt_path.read_text()) + call.cancel.assert_called_once_with() + call.get.assert_called_once_with(timeout=60) + with self.assertRaisesRegex(ValueError, "cannot resubmit"): + recovery.submit_once(function, receipt_path, {}, result_timeout_seconds=60) + function.spawn.assert_called_once() + self.assertTrue((self.output / "person-posed.ply").exists()) + self.assertEqual( + recovery.recover_outputs(receipt_path, self.volume)["status"], "complete" + ) + + def test_cancellation_error_records_class_only_and_keeps_original_exception(self): + function = Mock() + call = function.spawn.return_value + call.object_id = "fc-owned" + original = TimeoutError("private wait details") + call.get.side_effect = original + call.cancel.side_effect = OSError("private cancellation details") + with self.assertRaises(TimeoutError) as caught: + recovery.submit_once(function, self.receipt_path, {}, result_timeout_seconds=60) + self.assertIs(caught.exception, original) + receipt = recovery.load_receipt(self.receipt_path) + self.assertEqual(receipt["status"], "wait_timeout") + self.assertEqual(receipt["cancellation"], "error") + self.assertEqual(receipt["cancellationErrorType"], "OSError") + self.assertNotIn("private wait details", self.receipt_path.read_text()) + self.assertNotIn("private cancellation details", self.receipt_path.read_text()) + call.cancel.assert_called_once() + + def test_failure_before_spawn_never_cancels(self): + function = Mock() + with patch.object(recovery, "atomic_json", side_effect=OSError("storage failure")): + with self.assertRaises(OSError): + recovery.submit_once(function, self.receipt_path, {}, result_timeout_seconds=60) + function.spawn.assert_not_called() + function.spawn.side_effect = OSError("uncertain spawn") + with self.assertRaises(OSError): + recovery.submit_once(function, self.receipt_path, {}, result_timeout_seconds=60) + function.spawn.return_value.cancel.assert_not_called() + self.assertEqual(recovery.load_receipt(self.receipt_path)["status"], "submission_uncertain") + with self.assertRaisesRegex(ValueError, "cannot resubmit"): + recovery.submit_once(function, self.receipt_path, {}, result_timeout_seconds=60) + function.spawn.assert_called_once() + + def test_post_spawn_receipt_failure_cancels_before_wait_even_without_timeout(self): + function = Mock() + call = function.spawn.return_value + call.object_id = "fc-owned" + real_write = recovery.atomic_json + + def write(path, receipt): + if receipt["status"] == "waiting": + raise OSError("receipt write failed") + real_write(path, receipt) + + with patch.object(recovery, "atomic_json", side_effect=write), self.assertRaises(OSError): + recovery.submit_once(function, self.receipt_path, {}) + call.get.assert_not_called() + call.cancel.assert_called_once() + receipt = recovery.load_receipt(self.receipt_path) + self.assertEqual(receipt["status"], "wait_interrupted") + self.assertEqual(receipt["functionCallId"], "fc-owned") + self.assertEqual(receipt["cancellation"], "requested") + with self.assertRaisesRegex(ValueError, "cannot resubmit"): + recovery.submit_once(function, self.receipt_path, {}) + + def test_persistent_receipt_failure_still_cancels_and_blocks_resubmit(self): + function = Mock() + call = function.spawn.return_value + call.object_id = "fc-owned" + real_write = recovery.atomic_json + + def write(path, receipt): + if receipt["status"] != "submitting": + raise OSError("storage unavailable") + real_write(path, receipt) + + with patch.object(recovery, "atomic_json", side_effect=write), self.assertRaises(OSError): + recovery.submit_once(function, self.receipt_path, {}, result_timeout_seconds=60) + call.cancel.assert_called_once() + call.get.assert_not_called() + self.assertEqual(recovery.load_receipt(self.receipt_path)["status"], "submitting") + with self.assertRaisesRegex(ValueError, "cannot resubmit"): + recovery.submit_once(function, self.receipt_path, {}) + + def test_legacy_success_has_unbounded_get_and_no_new_receipt_fields(self): + function = Mock() + call = function.spawn.return_value + call.object_id = "fc-legacy" + call.get.return_value = { + "recoveryId": self.receipt["recoveryId"], + "remotePath": self.receipt["remotePath"], + "manifestSha256": "a" * 64, + } + recovery.submit_once(function, self.receipt_path, {}) + call.get.assert_called_once_with() + call.cancel.assert_not_called() + receipt = recovery.load_receipt(self.receipt_path) + for field in ("resultTimeoutSeconds", "appId", "modalProfile", "cancellation"): + self.assertNotIn(field, receipt) + + def test_result_receipt_failure_keeps_outputs_and_cancels_owned_call(self): + self.write_frozen() + self.finish() + function = Mock() + call = function.spawn.return_value + call.object_id = "fc-owned" + call.get.return_value = { + "recoveryId": self.receipt["recoveryId"], + "remotePath": self.receipt["remotePath"], + "manifestSha256": recovery.file_hash(self.checkpoint / "manifest.json"), + } + real_write = recovery.atomic_json + + def write(path, receipt): + if receipt["status"] == "checkpoint_available": + raise OSError("receipt storage failed") + real_write(path, receipt) + + with patch.object(recovery, "atomic_json", side_effect=write), self.assertRaises(OSError): + recovery.submit_once(function, self.receipt_path, {}, result_timeout_seconds=60) + call.cancel.assert_called_once() + call.get.assert_called_once_with(timeout=60) + self.assertEqual(recovery.load_receipt(self.receipt_path)["status"], "wait_interrupted") + self.assertEqual( + recovery.recover_outputs(self.receipt_path, self.volume)["status"], "complete" + ) + function.spawn.assert_called_once() def test_uncertain_spawn_and_unreadable_receipt_never_retry(self): function = Mock() @@ -401,7 +615,16 @@ def spawn(inputs, **kwargs): "manifestSha256": recovery.file_hash(directory / "manifest.json"), "report": {"error": None}, } - return types.SimpleNamespace(object_id="fc-test", get=lambda: result) + + def get(*, timeout): + resources = modal_lhm.INFERENCE_RESOURCES + self.assertEqual( + timeout, + resources.get("startup_timeout", resources["timeout"]) + resources["timeout"], + ) + return result + + return types.SimpleNamespace(object_id="fc-test", get=get) function.spawn.side_effect = spawn destination = self.root / "cli-output" diff --git a/scripts/tests/test_lhm_wait_entrypoints.py b/scripts/tests/test_lhm_wait_entrypoints.py new file mode 100644 index 0000000..d8710a3 --- /dev/null +++ b/scripts/tests/test_lhm_wait_entrypoints.py @@ -0,0 +1,55 @@ +"""Local entrypoints must bound waits without changing inference input arguments.""" + +import sys +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from worker import modal_lhm, modal_multiperson +from worker.stages import lhm_recovery, modal_profile + + +@pytest.mark.parametrize( + "mode,expected", [("frozen", 1200), ("motion", 1200), ("short_motion", 900)] +) +def test_entrypoint_records_owner_and_effective_deadline(tmp_path, monkeypatch, mode, expected): + for name in ( + "source.png", + "mask.png", + "prepared.json", + "source.mp4", + "canonical-state.pt", + "result.json", + "cameras.json", + "frame_000.ply", + "source-poses.pt", + "motion.json", + ): + (tmp_path / name).write_bytes(name.encode()) + submit = Mock(side_effect=RuntimeError("stop before provider")) + monkeypatch.setattr(lhm_recovery, "submit_once", submit) + monkeypatch.setattr(modal_profile, "effective_modal_profile", lambda: "owner-profile") + module = modal_lhm if mode == "frozen" else modal_multiperson + monkeypatch.setattr(module, "app", SimpleNamespace(app_id="ap-owned")) + monkeypatch.setattr(module, "INFERENCE_RESOURCES", {"timeout": 600, "startup_timeout": 600}) + with pytest.raises(RuntimeError, match="stop before provider"): + if mode == "frozen": + module.main(prepared=str(tmp_path), out=str(tmp_path / "output")) + else: + module.animate( + video=str(tmp_path / "source.mp4"), + canonical=str(tmp_path), + cameras=str(tmp_path / "cameras.json"), + track_dir=str(tmp_path), + out=str(tmp_path / "output"), + execution_timeout=300 if mode == "short_motion" else 0, + ) + kwargs = submit.call_args.kwargs + assert kwargs["result_timeout_seconds"] == expected + assert kwargs["app_id"] == "ap-owned" + assert kwargs["modal_profile"] == "owner-profile" + if mode != "frozen": + assert kwargs["execution_timeout"] == (300 if mode == "short_motion" else 0) diff --git a/scripts/tests/test_multiperson_recovery.py b/scripts/tests/test_multiperson_recovery.py index 255f9ef..b54a465 100644 --- a/scripts/tests/test_multiperson_recovery.py +++ b/scripts/tests/test_multiperson_recovery.py @@ -98,7 +98,9 @@ def test_interrupted_wait_records_identity_and_repeat_cli_never_resubmits(self): call = bounded.spawn.return_value call.object_id = "fc-retained-track" - def interrupt(): + def interrupt(*, timeout): + resources = modal_multiperson.INFERENCE_RESOURCES + self.assertEqual(timeout, resources.get("startup_timeout", 600) + 600) receipt = lhm_recovery.load_receipt(self.destination / lhm_recovery.RECEIPT_NAME) self.assertEqual(receipt["status"], "waiting") self.assertEqual(receipt["functionCallId"], call.object_id) @@ -122,6 +124,7 @@ def interrupt(): timeout=600, cpu=(4, 4), memory=(65536, 65536), retries=0, max_containers=1 ) bounded.spawn.assert_called_once() + call.cancel.assert_called_once_with() self.assertEqual( bounded.spawn.call_args.kwargs["flags"], ["--track-id", "0", "--fixed-world-scale", "1.9"], @@ -187,7 +190,16 @@ def spawn(inputs, **kwargs): "manifestSha256": lhm_recovery.file_hash(directory / "manifest.json"), "report": {"error": None}, } - return types.SimpleNamespace(object_id="fc-partial", get=lambda: result) + + def get(*, timeout): + resources = modal_multiperson.INFERENCE_RESOURCES + self.assertEqual( + timeout, + resources.get("startup_timeout", resources["timeout"]) + resources["timeout"], + ) + return result + + return types.SimpleNamespace(object_id="fc-partial", get=get) function.spawn.side_effect = spawn with ( diff --git a/worker/modal_lhm.py b/worker/modal_lhm.py index db22c39..8b3595d 100644 --- a/worker/modal_lhm.py +++ b/worker/modal_lhm.py @@ -441,6 +441,7 @@ def main( import tarfile from worker.stages.lhm_recovery import new_receipt, recover_outputs, submit_once + from worker.stages.modal_profile import effective_modal_profile if not out: raise ValueError("LHM output destination is required") @@ -482,7 +483,17 @@ def main( if larger_model and not fixed_inputs: raise ValueError("Larger checkpoint comparison requires saved fixed inputs") result = submit_once( - frozen, receipt_path, inputs, animate=bool(canonical), larger_model=larger_model + frozen, + receipt_path, + inputs, + result_timeout_seconds=INFERENCE_RESOURCES.get( + "startup_timeout", INFERENCE_RESOURCES["timeout"] + ) + + INFERENCE_RESOURCES["timeout"], + app_id=app.app_id, + modal_profile=effective_modal_profile(), + animate=bool(canonical), + larger_model=larger_model, ) manifest = recover_outputs(receipt_path, cache) # Preserve the normal archive artifact, without GPU compression or a giant return. diff --git a/worker/modal_multiperson.py b/worker/modal_multiperson.py index 4851b4d..480f02e 100644 --- a/worker/modal_multiperson.py +++ b/worker/modal_multiperson.py @@ -464,6 +464,7 @@ def animate( from worker.stages.lhm_execution import animation_options from worker.stages.lhm_recovery import atomic_json, new_receipt, recover_outputs, submit_once + from worker.stages.modal_profile import effective_modal_profile scale_flags, options = animation_options(fixed_world_scale, execution_timeout) if options: @@ -509,6 +510,12 @@ def animate( function, receipt_path, inputs, + result_timeout_seconds=INFERENCE_RESOURCES.get( + "startup_timeout", options.get("timeout", INFERENCE_RESOURCES["timeout"]) + ) + + options.get("timeout", INFERENCE_RESOURCES["timeout"]), + app_id=app.app_id, + modal_profile=effective_modal_profile(), flags=flags, execution_timeout=execution_timeout, **({"recover_missing_poses": True} if recover_missing_poses else {}), diff --git a/worker/stages/lhm_recovery.py b/worker/stages/lhm_recovery.py index a138ac8..4dd9b75 100644 --- a/worker/stages/lhm_recovery.py +++ b/worker/stages/lhm_recovery.py @@ -141,17 +141,78 @@ def receipt_lock(receipt_path): yield -def submit_once(function, receipt_path, inputs, **kwargs): +def submit_once( + function, + receipt_path, + inputs, + *, + result_timeout_seconds=None, + app_id=None, + modal_profile=None, + **kwargs, +): """Persist identity before waiting. An uncertain or interrupted receipt never resubmits.""" with receipt_lock(receipt_path): - return _submit_once(function, receipt_path, inputs, **kwargs) + return _submit_once( + function, + receipt_path, + inputs, + result_timeout_seconds=result_timeout_seconds, + app_id=app_id, + modal_profile=modal_profile, + **kwargs, + ) -def _submit_once(function, receipt_path, inputs, **kwargs): +def _record_wait_failure(call, receipt_path, receipt, error, *, cancel, bounded): + receipt["status"] = ( + "wait_timeout" if bounded and isinstance(error, TimeoutError) else "wait_interrupted" + ) + if cancel: + receipt["waitErrorType"] = type(error).__name__ + try: + call.cancel() + receipt["cancellation"] = "requested" + except BaseException as cancellation_error: + receipt["cancellation"] = "error" + receipt["cancellationErrorType"] = type(cancellation_error).__name__ + try: + atomic_json(receipt_path, receipt) + except BaseException: + # Keep the original failure. The earlier submitting/waiting receipt still + # prevents another submission, even when storage cannot record cleanup. + pass + + +def _submit_once( + function, + receipt_path, + inputs, + *, + result_timeout_seconds=None, + app_id=None, + modal_profile=None, + **kwargs, +): + if result_timeout_seconds is not None and ( + isinstance(result_timeout_seconds, bool) + or not isinstance(result_timeout_seconds, int) + or result_timeout_seconds <= 0 + ): + raise ValueError("result_timeout_seconds must be a positive finite integer") receipt = load_receipt(receipt_path) if receipt.get("status") != "prepared" or receipt.get("functionCallId") is not None: raise ValueError("Existing LHM receipt cannot resubmit inference; use recovery") receipt["status"] = "submitting" + if result_timeout_seconds is not None: + receipt.update( + resultTimeoutSeconds=result_timeout_seconds, appId=app_id, modalProfile=modal_profile + ) + else: + if app_id is not None: + receipt["appId"] = app_id + if modal_profile is not None: + receipt["modalProfile"] = modal_profile atomic_json(receipt_path, receipt) try: call = function.spawn(inputs, recovery_id=receipt["recoveryId"], **kwargs) @@ -159,14 +220,28 @@ def _submit_once(function, receipt_path, inputs, **kwargs): receipt["status"] = "submission_uncertain" atomic_json(receipt_path, receipt) raise - receipt["functionCallId"] = call.object_id - receipt["status"] = "waiting" - atomic_json(receipt_path, receipt) try: - result = call.get() - except BaseException: - receipt["status"] = "wait_interrupted" + receipt["functionCallId"] = call.object_id + receipt["status"] = "waiting" atomic_json(receipt_path, receipt) + except BaseException as error: + _record_wait_failure(call, receipt_path, receipt, error, cancel=True, bounded=False) + raise + try: + result = ( + call.get(timeout=result_timeout_seconds) + if result_timeout_seconds is not None + else call.get() + ) + except BaseException as error: + _record_wait_failure( + call, + receipt_path, + receipt, + error, + cancel=result_timeout_seconds is not None, + bounded=result_timeout_seconds is not None, + ) raise if ( not isinstance(result, dict) @@ -178,7 +253,11 @@ def _submit_once(function, receipt_path, inputs, **kwargs): receipt["totalWorkerSeconds"] = result.get("totalWorkerSeconds") receipt["manifestSha256"] = result["manifestSha256"] receipt["status"] = "checkpoint_available" - atomic_json(receipt_path, receipt) + try: + atomic_json(receipt_path, receipt) + except BaseException as error: + _record_wait_failure(call, receipt_path, receipt, error, cancel=True, bounded=False) + raise return result From c219d7680c4fb58b77b40a8890e8d3d8455a9285 Mon Sep 17 00:00:00 2001 From: Austin Jian Date: Sat, 21 Nov 2026 00:10:01 -0500 Subject: [PATCH 2/3] fix(clean): preserve exact tracked source frame grids --- docs/paid-recovery.md | 7 + scripts/run_clip.py | 39 +++-- scripts/tests/test_tracks_to_clean_masks.py | 153 +++++++++++++++++++- scripts/tracks_to_clean_masks.py | 7 +- 4 files changed, 176 insertions(+), 30 deletions(-) diff --git a/docs/paid-recovery.md b/docs/paid-recovery.md index 88678cf..c5bec61 100644 --- a/docs/paid-recovery.md +++ b/docs/paid-recovery.md @@ -513,6 +513,13 @@ Keep the original FPS and source clip. This option changes no current pipeline d not authorize another paid clean, reset the shared inpainting allowance, or replace a failed quality verdict. Inpainting remains subject to the same ledger, approvals and visual review. +Explicit-source clean masks are validated against the decoded frames selected by their recorded +source indices, with matching source SHA, PTS, timebase and packed shape. Their count need not +equal the FFmpeg FPS-filter count: a fractional-FPS tracking grid can retain a final source frame +that the FPS filter omits. Track-mask conversion still requires every declared track/sample mask +and complete sample coverage. Archives without explicit selection metadata must still match the +observed FPS-filter count; incomplete or inconsistent explicit metadata fails before paid cleaning. + ## Experimental isolation of inpainting context The direct runner and clean worker accept `--inpaint-mode isolated`; `legacy` remains the default. diff --git a/scripts/run_clip.py b/scripts/run_clip.py index 990de6c..238e17b 100644 --- a/scripts/run_clip.py +++ b/scripts/run_clip.py @@ -1100,7 +1100,7 @@ def clean_flags(self): return flags + (["--moved-mask"] if a.moved_mask else []) def validate_clean_masks(self, path: Path): - """Require packed masks for this source's observed FFmpeg sampling before any Modal call.""" + """Require masks matching decoded explicit selection or legacy FPS sampling before Modal.""" import numpy as np if not path.is_file(): @@ -1126,24 +1126,6 @@ def validate_clean_masks(self, path: Path): "--clean-masks masks must be a three-dimensional uint8 packed-bit array" ) - expected_shape = ( - clean_sample_count(self.clip, self.a.fps), - expected_height, - expected_packed_width, - ) - if tuple(packed.shape) != expected_shape: - raise ValueError( - "--clean-masks shape " - f"{tuple(packed.shape)} does not match observed source sampling {expected_shape}" - ) - if declared is not None and ( - declared.shape != (3,) - or tuple(declared.tolist()) != (expected_shape[0], expected_height, expected_width) - ): - raise ValueError( - "--clean-masks shape metadata does not match the observed sampled frame count " - "and source dimensions" - ) if selected: from worker.wander_worker.source_timing import select_source_frames @@ -1159,7 +1141,7 @@ def validate_clean_masks(self, path: Path): ) indices, pts = selection["source_indices"], selection["source_pts"] if any( - value.shape != (expected_shape[0],) or not np.issubdtype(value.dtype, np.integer) + value.shape != (packed.shape[0],) or not np.issubdtype(value.dtype, np.integer) for value in (indices, pts) ): raise ValueError( @@ -1175,12 +1157,29 @@ def validate_clean_masks(self, path: Path): _, provenance = select_source_frames( self.clip, self.a.fps, source_indices=indices.tolist() ) + expected_count = len(provenance["frames"]) if ( selection["source_sha256"].item() != provenance["sourceSha256"] or selection["source_time_base"].item() != provenance["sourceTimeBase"] or pts.tolist() != [frame["sourcePts"] for frame in provenance["frames"]] ): raise ValueError("Explicit clean masks do not match the selected source pixels") + else: + expected_count = clean_sample_count(self.clip, self.a.fps) + expected_shape = (expected_count, expected_height, expected_packed_width) + if tuple(packed.shape) != expected_shape: + raise ValueError( + "--clean-masks shape " + f"{tuple(packed.shape)} does not match observed source sampling {expected_shape}" + ) + if declared is not None and ( + declared.shape != (3,) + or tuple(declared.tolist()) != (expected_count, expected_height, expected_width) + ): + raise ValueError( + "--clean-masks shape metadata does not match the observed sampled frame count " + "and source dimensions" + ) def review(self): """The credit gate. 1600 credits are about to be spent on whatever these frames show.""" diff --git a/scripts/tests/test_tracks_to_clean_masks.py b/scripts/tests/test_tracks_to_clean_masks.py index 0e2597a..8146edb 100644 --- a/scripts/tests/test_tracks_to_clean_masks.py +++ b/scripts/tests/test_tracks_to_clean_masks.py @@ -14,6 +14,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) import run_clip import tracks_to_clean_masks +from worker.wander_worker.source_timing import resample_source def track_fixture(root, clip, samples=5, width=11, height=4): @@ -55,9 +56,6 @@ def track_fixture(root, clip, samples=5, width=11, height=4): def tracking(tmp_path, monkeypatch): clip, root, out = tmp_path / "clip.mp4", tmp_path / "tracks", tmp_path / "clean-masks.npz" doc, payload, provenance = track_fixture(root, clip) - monkeypatch.setattr( - tracks_to_clean_masks, "resample_source", lambda *a, **kw: (b"", provenance) - ) monkeypatch.setattr( tracks_to_clean_masks, "select_source_frames", lambda *a, **kw: (b"", provenance) ) @@ -188,7 +186,7 @@ def test_exact_tracker_grid_is_preserved_when_fps_filter_selects_other_frames(tm ], check=True, ) - _, sampled = tracks_to_clean_masks.resample_source(clip, 2) + _, sampled = resample_source(clip, 2) indices = [0, 3] _, exact = tracks_to_clean_masks.select_source_frames(clip, 2, source_indices=indices) assert [frame["sourceIndex"] for frame in sampled["frames"]] != indices @@ -320,3 +318,150 @@ def test_exact_source_masks_rejected_before_paid_multi_image_clean(runner, track with pytest.raises(ValueError, match="not supported by --marble multi"): runner.clean_flags() runner.paid_run.assert_not_called() + + +@pytest.fixture(scope="module") +def fractional_source(tmp_path_factory): + if not shutil.which("ffmpeg") or not shutil.which("ffprobe"): + pytest.skip("FFmpeg required for decoded fractional-FPS fixture") + root = tmp_path_factory.mktemp("fractional-source") + clip = root / "source.mp4" + subprocess.run( + [ + "ffmpeg", + "-v", + "error", + "-f", + "lavfi", + "-i", + "testsrc2=size=16x12:rate=24000/1001", + "-frames:v", + "186", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + str(clip), + ], + check=True, + ) + indices = list(range(0, 186, 2)) + [185] + _, exact = tracks_to_clean_masks.select_source_frames(clip, 12, source_indices=indices) + _, sampled = resample_source(clip, 12) + assert len(exact["frames"]) == 94 + assert len(sampled["frames"]) == 93 + assert run_clip.clean_sample_count(clip, 12) == 93 + assert [frame["sourceIndex"] for frame in sampled["frames"]] == indices[:-1] + return clip, indices, exact + + +@pytest.fixture +def fractional_runner(tmp_path, fractional_source): + from types import SimpleNamespace + from unittest.mock import Mock + + clip, indices, exact = fractional_source + tracks = tmp_path / "tracks" + tracks.mkdir() + doc = { + "sourceSha256": exact["sourceSha256"], + "width": 16, + "height": 12, + "fps": 12, + "samples": 94, + "sourceIndices": indices, + "timestamps": [frame["sourceRelativeTimeSeconds"] for frame in exact["frames"]], + "trackCount": 2, + "tracks": [{"track": i, "samples": list(range(94))} for i in range(2)], + } + (tracks / "tracks.json").write_text(json.dumps(doc)) + payload = {"shape": np.array([12, 16, 1.0])} + for track in range(2): + mask = np.zeros((12, 16), bool) + mask[1:3, 2 + track] = True + for sample in range(94): + payload[f"t{track}_s{sample:03d}"] = np.packbits(mask) + np.savez_compressed(tracks / "masks.npz", **payload) + pipeline = object.__new__(run_clip.Pipeline) + pipeline.ctx, pipeline.clip = tmp_path, clip + pipeline.clean_mp4 = tmp_path / "clean.mp4" + pipeline.multi = True + pipeline.info = {"width": 16, "height": 12} + pipeline.a = SimpleNamespace( + clean_masks=None, moved_mask=False, fps=12, dilate=0, bottom_extra=0, lama_px=960 + ) + pipeline.paid_run = Mock() + return pipeline, tracks, doc, payload + + +def test_fractional_endpoint_grid_reaches_paid_boundary_without_fps_count_check( + fractional_runner, fractional_source +): + pipeline, _, _, _ = fractional_runner + _, indices, exact = fractional_source + with patch.object(run_clip, "clean_sample_count", side_effect=AssertionError("legacy path")): + pipeline.clean() + pipeline.paid_run.assert_called_once() + command = pipeline.paid_run.call_args.args[1] + path = Path(command[command.index("--masks-in") + 1]) + with np.load(path) as archive: + assert archive["shape"].tolist() == [94, 12, 16] + assert archive["source_indices"].tolist() == indices + assert archive["source_pts"].tolist() == [frame["sourcePts"] for frame in exact["frames"]] + assert np.unpackbits(archive["masks"][-1], axis=-1).sum() == 4 + + +@pytest.mark.parametrize("invalid", ["hash", "timestamp", "samples", "endpoint-mask"]) +def test_fractional_endpoint_invalid_tracking_stops_before_paid_run(fractional_runner, invalid): + pipeline, tracks, doc, payload = fractional_runner + if invalid == "hash": + doc["sourceSha256"] = "0" * 64 + elif invalid == "timestamp": + doc["timestamps"][-1] += 0.01 + elif invalid == "samples": + doc["samples"] = 93 + else: + # The other track still covers this sample: every declared mask is required. + payload.pop("t1_s093") + np.savez_compressed(tracks / "masks.npz", **payload) + (tracks / "tracks.json").write_text(json.dumps(doc)) + with pytest.raises(ValueError): + pipeline.clean() + pipeline.paid_run.assert_not_called() + assert not (pipeline.ctx / "track-clean-masks.npz").exists() + + +@pytest.mark.parametrize("invalid", ["pts", "index", "partial", "shape", "legacy-count"]) +def test_fractional_explicit_archive_validation_cannot_fall_back(fractional_runner, invalid): + pipeline, tracks, _, _ = fractional_runner + path = pipeline.ctx / "reviewed.npz" + tracks_to_clean_masks.build_clean_masks( + tracks, clip=pipeline.clip, out=path, width=16, height=12, fps=12, dilate=0, bottom_extra=0 + ) + with np.load(path) as archive: + payload = {key: archive[key] for key in archive.files} + if invalid == "pts": + payload["source_pts"][-1] += 1 + elif invalid == "index": + payload["source_indices"][-1] = 186 + elif invalid == "partial": + payload.pop("source_pts") + elif invalid == "shape": + payload["shape"][0] = 93 + else: + for key in ("source_indices", "source_pts", "source_time_base", "source_sha256"): + payload.pop(key) + np.savez_compressed(path, **payload) + pipeline.a.clean_masks = str(path) + with pytest.raises(ValueError): + pipeline.clean() + pipeline.paid_run.assert_not_called() + + +def test_fractional_legacy_archive_still_accepts_exact_fps_filter_count(fractional_runner): + pipeline, _, _, _ = fractional_runner + path = pipeline.ctx / "legacy.npz" + np.savez_compressed(path, masks=np.zeros((93, 12, 2), np.uint8), shape=np.array([93, 12, 16])) + pipeline.a.clean_masks = str(path) + pipeline.clean() + pipeline.paid_run.assert_called_once() diff --git a/scripts/tracks_to_clean_masks.py b/scripts/tracks_to_clean_masks.py index fb1e1c7..a7b90d0 100644 --- a/scripts/tracks_to_clean_masks.py +++ b/scripts/tracks_to_clean_masks.py @@ -27,7 +27,7 @@ ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) -from worker.wander_worker.source_timing import resample_source, select_source_frames # noqa: E402 +from worker.wander_worker.source_timing import select_source_frames # noqa: E402 def build_clean_masks(tracks, *, clip, out, width, height, fps, dilate=20, bottom_extra=40): @@ -56,12 +56,7 @@ def build_clean_masks(tracks, *, clip, out, width, height, fps, dilate=20, botto or any(type(index) is not int or index < 0 for index in indices) ): raise ValueError("Tracking masks require an explicit source frame grid") - _, sampled = resample_source(clip, fps) - if len(indices) != len(sampled["frames"]): - raise ValueError("Tracking mask frame count does not match observed clean source sampling") _, provenance = select_source_frames(clip, fps, source_indices=indices) - if provenance["sourceSha256"] != sampled["sourceSha256"]: - raise ValueError("Source changed while validating tracking mask sampling") frames = provenance["frames"] n = len(frames) if doc.get("sourceSha256") != provenance["sourceSha256"]: From 21cfc4086186b518c91c5c29e3032bd507876c48 Mon Sep 17 00:00:00 2001 From: Austin Jian Date: Sat, 21 Nov 2026 00:33:33 -0500 Subject: [PATCH 3/3] fix(scale): retain measured median-passing probes after convergence --- docs/known-limits.md | 7 ++ scripts/run_clip.py | 26 ++++- scripts/tests/test_scale_fit_evidence.py | 129 +++++++++++++++++++++++ 3 files changed, 159 insertions(+), 3 deletions(-) diff --git a/docs/known-limits.md b/docs/known-limits.md index cb69f31..3051357 100644 --- a/docs/known-limits.md +++ b/docs/known-limits.md @@ -3,6 +3,13 @@ Approaches already measured to fail. Do not retry one without a new idea, and add a line here when you measure a new dead end. +- Geometric-mean scale convergence can move a previously passing per-frame median outside the + fixed band as visible depth support changes. After normal convergence, the automatic fitter + retains the earlier median-passing probe closest to unit geometric mean only if the final + medians fail, keeping that probe's scale and measured evidence together. Nonconvergence still + fails; explicit scales are unchanged. Median retention does not relax the independent depth + spread gate or establish acceptable geometry. + - The fresh city tracking baseline assigned person poses to street lamps and split the main walker across tracks. Four of five retained tracks failed the source-appearance identity audit; 202 Mask R-CNN detections lacked a pose partner. These counts do not establish how many real diff --git a/scripts/run_clip.py b/scripts/run_clip.py index 238e17b..fafbbbe 100644 --- a/scripts/run_clip.py +++ b/scripts/run_clip.py @@ -3074,13 +3074,22 @@ def scale_fit(self): depth is ~1/scale0, so scale0 <- scale0 * geomean(ratio) converges in two or three probes. """ s, probes = self.a.scale0, [] + selected_probe, selection_reason = "given", "explicit_scale" if s is None: + best_median_probe = None s = self.placement().get("floorMatchedScale") or 1.0 for i in range(self.a.scale_probes): rows, fig, bands = self.diag(s, i) ratios = [r for r, _, _ in rows] gm = math.exp(sum(math.log(r) for r in ratios) / len(ratios)) probes.append((s, gm)) + if all(SCALE_RATIO_BAND[0] <= r <= SCALE_RATIO_BAND[1] for r in ratios) and ( + best_median_probe is None + or abs(math.log(gm)) < abs(math.log(best_median_probe[2])) + ): + # Keep the measured evidence together. Diagnostic filenames round the + # scale, so a later probe can overwrite the same figure path. + best_median_probe = (i, s, gm, rows, bands, Path(fig).read_bytes()) say( f" probe {i}: scale0 {s:.4f} -> ratios {[round(r, 3) for r in ratios]} " f"(geomean {gm:.4f})" @@ -3094,6 +3103,15 @@ def scale_fit(self): f"{[(round(x, 4), round(g, 4)) for x, g in probes]}; pass " f"--scale0 by hand or raise --scale-probes" ) + selected_probe, selection_reason = i, "converged_final" + if best_median_probe is not None and not all( + SCALE_RATIO_BAND[0] <= r <= SCALE_RATIO_BAND[1] for r in ratios + ): + selected_probe, s, gm, rows, bands, figure_bytes = best_median_probe + ratios = [r for r, _, _ in rows] + fig = self.ctx / f"scale-fit-selected-probe-{selected_probe}.png" + fig.write_bytes(figure_bytes) + selection_reason = "earlier_median_pass_closest_log_geomean" else: rows, fig, bands = self.diag(s, "given") ratios = [r for r, _, _ in rows] @@ -3109,9 +3127,9 @@ def scale_fit(self): if spread: median_problem = ( f"depth ratios {[round(r, 3) for r in ratios]} are not all inside " - f"{SCALE_RATIO_BAND[0]}-{SCALE_RATIO_BAND[1]} at any single scale (geomean fit {s:.4f} " - f"gives {gm:.4f}). That is Marble geometry error or SfM drift, not a scale error -- it " - f"means the avatar reads ~{abs(1 - min(ratios)) * 100:.0f}% wrong at one end of the " + f"{SCALE_RATIO_BAND[0]}-{SCALE_RATIO_BAND[1]} at the evaluated scale {s:.4f} " + f"(geomean {gm:.4f}). This measurement alone does not rule out another scale. " + f"The depth mismatch is ~{abs(1 - min(ratios)) * 100:.0f}% at one end of the " f"clip. Read {fig}, then either re-trim the clip to " f"the window that fits or pass --allow-scale-spread to ship the geometric-mean fit " f"with that caveat (gym, atrium and selfie did)." @@ -3132,6 +3150,8 @@ def scale_fit(self): p10=p10, p90=p90, probes=probes, + selectedProbe=selected_probe, + selectionReason=selection_reason, gateOk=not spread and not wide, medianOk=not spread, spreadOk=not wide, diff --git a/scripts/tests/test_scale_fit_evidence.py b/scripts/tests/test_scale_fit_evidence.py index 2c9dd99..54eb7c1 100644 --- a/scripts/tests/test_scale_fit_evidence.py +++ b/scripts/tests/test_scale_fit_evidence.py @@ -213,3 +213,132 @@ def test_band_summary_keeps_gate_precision_and_flags_collapsed_quartiles(): def test_malformed_band_lines_are_evidence_loss_not_a_crash(): assert run_clip.distance_bands('depth-ratio-bands {"frame": 0, "bands": [', 1) is None + + +def probe_sequence(pipeline, monkeypatch, rows_by_probe): + """Different measured evidence per call; reuse a filename to exercise snapshot ownership.""" + calls, evidence = [], [] + figure = pipeline.ctx / "rounded-scale-depth-ratio.png" + + def diag(scale, probe): + index = len(calls) + rows = rows_by_probe[index] + bands = [ + {"frame": frame, "bands": [{"band": 0, "p10": lo, "median": ratio, "p90": hi}]} + for frame, (ratio, lo, hi) in enumerate(rows) + ] + pixels = f"measured probe {index}".encode() + figure.write_bytes(pixels) + calls.append((scale, probe)) + evidence.append((rows, bands, pixels)) + return rows, figure, bands + + monkeypatch.setattr(pipeline, "diag", diag) + return calls, evidence, figure + + +@pytest.mark.parametrize("unaccepted", [False, True]) +def test_retains_closest_passing_probe_with_consistent_evidence_and_failed_spread( + pipeline, monkeypatch, unaccepted +): + p = pipeline + p.a.scale_spread_unaccepted = unaccepted + rows = [ + [(1.4, 1.2, 1.6)], + [(0.98, 0.94, 1.04), (0.98, 0.93, 1.05)], + [(1.01, 0.81, 1.21), (1.01, 0.85, 1.16)], + [(0.94, 0.92, 1.06), (1.06, 0.95, 1.08)], + ] + calls, evidence, overwritten = probe_sequence(p, monkeypatch, rows) + assert p.go() == (0 if unaccepted else 1) + assert len(calls) == 4 # No additional diagnostic or retry. + report = json.loads((p.ctx / "scale-fit.json").read_text()) + assert report == p.state.data["stages"]["_scale"] + assert report["selectedProbe"] == 2 + assert report["selectionReason"] == "earlier_median_pass_closest_log_geomean" + assert report["scale0"] == pytest.approx(calls[2][0]) + assert report["scale0"] == pytest.approx(0.8 * 1.4 * 0.98) + assert report["geomean"] == pytest.approx(1.01) + assert report["ratios"] == [1.01, 1.01] + assert report["p10"] == [0.81, 0.85] + assert report["p90"] == [1.21, 1.16] + assert report["distanceBands"] == evidence[2][1] + assert Path(report["figure"]).read_bytes() == evidence[2][2] + assert overwritten.read_bytes() == evidence[3][2] + assert report["medianOk"] is True + assert report["spreadOk"] is False + assert report["gateOk"] is False + assert "exceed the required spread" in report["gateError"] + assert report["figure"] in report["gateError"] + assert len(report["probes"]) == 4 + assert report["probes"][3][0] == pytest.approx(calls[3][0]) + + +def test_median_passing_final_keeps_final_even_if_earlier_spread_was_better(pipeline, monkeypatch): + rows = [[(1.01, 0.97, 1.04)], [(1.001, 0.8, 1.2)]] + calls, evidence, figure = probe_sequence(pipeline, monkeypatch, rows) + assert pipeline.go() == 1 + report = json.loads((pipeline.ctx / "scale-fit.json").read_text()) + assert len(calls) == 2 + assert report["selectedProbe"] == 1 + assert report["selectionReason"] == "converged_final" + assert report["scale0"] == pytest.approx(calls[1][0]) + assert report["ratios"] == [1.001] + assert report["distanceBands"] == evidence[1][1] + assert report["figure"] == str(figure) + assert report["medianOk"] is True and report["spreadOk"] is False + assert not list(pipeline.ctx.glob("scale-fit-selected-probe-*.png")) + + +def test_no_acceptable_probe_keeps_final_failure_and_limits_error_claim(pipeline, monkeypatch): + calls, _, figure = probe_sequence( + pipeline, monkeypatch, [[(1.2, 1.0, 1.3)], [(0.94, 0.92, 1.08), (1.06, 0.92, 1.08)]] + ) + assert pipeline.go() == 1 + report = json.loads((pipeline.ctx / "scale-fit.json").read_text()) + assert len(calls) == 2 + assert report["selectedProbe"] == 1 + assert report["scale0"] == pytest.approx(calls[1][0]) + assert report["figure"] == str(figure) + assert report["medianOk"] is False and report["spreadOk"] is True + assert "at the evaluated scale" in report["gateError"] + assert "at any single scale" not in report["gateError"] + + +def test_acceptable_probe_does_not_bypass_nonconvergence(pipeline, monkeypatch): + pipeline.a.scale_probes = 2 + calls, _, _ = probe_sequence(pipeline, monkeypatch, [[(1.02, 0.97, 1.04)]] * 2) + assert pipeline.go() == 1 + assert len(calls) == 2 + stages = pipeline.state.data["stages"] + assert "did not converge" in stages["scale_fit"]["error"] + assert "_scale" not in stages + assert not (pipeline.ctx / "scale-fit.json").exists() + assert not list(pipeline.ctx.glob("scale-fit-selected-probe-*.png")) + + +@pytest.mark.parametrize("ratio,accepted", [(1.02, True), (1.07, False)]) +def test_explicit_scale_keeps_single_given_measurement(pipeline, monkeypatch, ratio, accepted): + pipeline.a.scale0 = 0.726089 + calls, evidence, figure = probe_sequence(pipeline, monkeypatch, [[(ratio, 0.96, 1.08)]]) + assert pipeline.go() == (0 if accepted else 1) + assert calls == [(0.726089, "given")] + report = json.loads((pipeline.ctx / "scale-fit.json").read_text()) + assert report["scale0"] == 0.726089 + assert report["selectedProbe"] == "given" and report["selectionReason"] == "explicit_scale" + assert report["probes"] == [] + assert report["figure"] == str(figure) + assert report["distanceBands"] == evidence[0][1] + assert report["gateOk"] is accepted + + +def test_retained_probe_passes_gate_only_when_its_spread_also_passes(pipeline, monkeypatch): + calls, _, _ = probe_sequence( + pipeline, monkeypatch, [[(1.02, 0.96, 1.08)], [(0.94, 0.7, 1.3), (1.06, 0.7, 1.3)]] + ) + assert pipeline.go() == 0 + report = json.loads((pipeline.ctx / "scale-fit.json").read_text()) + assert len(calls) == 2 + assert report["selectedProbe"] == 0 + assert report["scale0"] == 0.8 + assert report["gateOk"] is True and report["gateError"] is None