diff --git a/docs/known-limits.md b/docs/known-limits.md index 89f1d22..615a0db 100644 --- a/docs/known-limits.md +++ b/docs/known-limits.md @@ -3,6 +3,14 @@ Approaches already measured to fail. Do not retry one without a new idea, and add a line here when you measure a new dead end. +- The fresh city tracking baseline assigned person poses to street lamps and split the main + walker across tracks. Four of five retained tracks failed the source-appearance identity audit; + 202 Mask R-CNN detections lacked a pose partner. These counts do not establish how many real + pedestrians were missed. The runner formerly marked tracking successful by skipping the failed + subset, even with `--all-people`. All-person runs now stop on any failed identity audit rather + than discarding those tracks. This repairs acceptance, not the tracker: no avatar or world was + generated from this result. Evidence: `.context/evidence/city-recovery-20261120/`. + - The forest scale gate fails on distant background, not on the route. Replaying the run's own scale diagnostic (unchanged inputs, scale0 0.7414, whole-frame p10 0.842–0.848) split by each frame's source-depth quartiles: rays within about 4 body-heights mostly stay inside the unchanged diff --git a/scripts/run_clip.py b/scripts/run_clip.py index 9cee145..990de6c 100644 --- a/scripts/run_clip.py +++ b/scripts/run_clip.py @@ -610,9 +610,13 @@ def track_or_skip(self, idx): raise SkipStage(f"only {doc['trackCount']} person track(s) in this clip") track = doc["tracks"][idx] ident = self.ctx / "identity.json" - if ident.exists() and track["track"] in ( - json.loads(ident.read_text()).get("failedTracks") or [] - ): + identity = json.loads(ident.read_text()) if ident.exists() else {} + if self.a.all_people and (identity.get("ok") is False or identity.get("failedTracks")): + raise RuntimeError( + "the retained identity audit failed and --all-people forbids dropping tracks. " + f"Repair tracking before preparing or generating avatars; see {ident}." + ) + if track["track"] in (identity.get("failedTracks") or []): raise SkipStage(f"track {track['track']} failed the identity audit (changed person)") return track @@ -1897,6 +1901,11 @@ def identity_guard(self): if p.returncode == 3: doc = json.loads((self.ctx / "identity.json").read_text()) failed = doc.get("failedTracks") or [] + if self.a.all_people: + raise RuntimeError( + "the identity audit failed and --all-people forbids dropping failed tracks. " + f"Preserve the tracking evidence and repair the tracks; see {self.ctx / 'identity.json'}." + ) if failed and len(failed) < len(doc.get("tracks", [])) and not doc.get("reason"): # some tracks changed person, some did not: build the ones that held (the swap # control passed, so the ok verdicts mean something) and skip the rest diff --git a/scripts/tests/test_tracking_identity_gate.py b/scripts/tests/test_tracking_identity_gate.py new file mode 100644 index 0000000..6241e80 --- /dev/null +++ b/scripts/tests/test_tracking_identity_gate.py @@ -0,0 +1,71 @@ +"""An all-person run must not succeed by discarding identity failures.""" + +import json +import sys +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +sys.path[:0] = [str(ROOT), str(ROOT / "scripts")] +import run_clip + + +@pytest.mark.parametrize( + "all_people,code,failed,reason,refused", + [ + (True, 3, [0, 2, 3, 4], None, True), + (False, 3, [0, 2, 3, 4], None, False), + (True, 0, [], None, False), + (False, 0, [], None, False), + (True, 3, [0, 1, 2, 3, 4], None, True), + (False, 3, [0, 1, 2, 3, 4], None, True), + (True, 3, [], "swap control missed", True), + (False, 3, [], "swap control missed", True), + (True, 4, [], "insufficient evidence", True), + ], +) +def test_identity_failure_cannot_remove_people( + tmp_path, monkeypatch, all_people, code, failed, reason, refused +): + pipeline = object.__new__(run_clip.Pipeline) + pipeline.a = SimpleNamespace(all_people=all_people) + pipeline.ctx = tmp_path + pipeline.clip = tmp_path / "source.mp4" + evidence = json.dumps({"tracks": list(range(5)), "failedTracks": failed, "reason": reason}) + receipt = tmp_path / "identity.json" + receipt.write_text(evidence) + audit = Mock(return_value=SimpleNamespace(returncode=code, stdout="audit result")) + monkeypatch.setattr(run_clip.subprocess, "run", audit) + if refused: + with pytest.raises(RuntimeError, match="identity audit"): + pipeline.identity_guard() + else: + pipeline.identity_guard() + assert receipt.read_text() == evidence + audit.assert_called_once() + assert "--swap-test" in audit.call_args.args[0] + + +@pytest.mark.parametrize("index", [0, 1]) +def test_retained_failed_audit_blocks_all_people_before_avatar_work(tmp_path, index): + pipeline = object.__new__(run_clip.Pipeline) + pipeline.a = SimpleNamespace(all_people=True) + pipeline.ctx = tmp_path + pipeline.track_doc = lambda: {"trackCount": 2, "tracks": [{"track": 0}, {"track": 1}]} + (tmp_path / "identity.json").write_text(json.dumps({"ok": False, "failedTracks": [0]})) + with pytest.raises(RuntimeError, match="retained identity audit failed"): + pipeline.track_or_skip(index) + + +def test_explicit_subset_keeps_existing_skip_behavior(tmp_path): + pipeline = object.__new__(run_clip.Pipeline) + pipeline.a = SimpleNamespace(all_people=False) + pipeline.ctx = tmp_path + pipeline.track_doc = lambda: {"trackCount": 2, "tracks": [{"track": 0}, {"track": 1}]} + (tmp_path / "identity.json").write_text(json.dumps({"ok": False, "failedTracks": [0]})) + with pytest.raises(run_clip.SkipStage): + pipeline.track_or_skip(0) + assert pipeline.track_or_skip(1) == {"track": 1}