diff --git a/docs/known-limits.md b/docs/known-limits.md index b6ddd1d..fe697e7 100644 --- a/docs/known-limits.md +++ b/docs/known-limits.md @@ -16,8 +16,16 @@ Do not retry one without a new idea, and add a line here when you measure a new 202 unmatched instance observations still include real background pedestrians and people pictured on signage. Do not animate every unmatched detection or call the single-track result accepted. Its body projection also remains misaligned with the observed silhouette. - Experimental `--pose-crops` adds calibrated inference for unmatched masks, but has no GPU - quality result yet and remains off by default; see [person motion](person-motion.md#experimental-tracking-crop-recovery). + A third paid trial used experimental `--pose-crops` plus explicit source-reviewed exclusions + for 16 advertisement observations. Of 186 crop calls, 67 returned poses and only five passed + the unchanged mask-support test. All five scored 0.160–0.220, below the existing 0.30 threshold + for starting a track; no additional track survived. The main walker retained all 94 samples, + but background people remained missing in source overlays. The median crop joint-box height + was 0.406 observed body-heights, with a median center offset of 0.214 body-heights (image-space + diagnostics, not physical measurements). Do not treat support counts as surviving tracks, + lower thresholds to declare success, or repeat this crop recipe unchanged. All three ordinary + paid tracking executions are used. The option stays off by default; see + [person motion](person-motion.md#experimental-tracking-crop-recovery). Evidence: `.context/evidence/city-recovery-20261120/`. - The forest scale gate fails on distant background, not on the route. Replaying the run's own diff --git a/docs/person-motion.md b/docs/person-motion.md index 9d8c0d8..7d1bd0a 100644 --- a/docs/person-motion.md +++ b/docs/person-motion.md @@ -118,6 +118,29 @@ change the stage retry budget, or distinguish pedestrians from people pictured o statues. `detection-audit.json` records crop bounds, detections, recoveries and pose origins; track summaries record call counts and motion observations preserve their origin. -The geometry and submission plumbing have CPU tests. Actual model recovery and visual -quality have not yet been established; do not enable this by default or treat recovered -detections as an accepted all-people reconstruction. +The geometry and submission plumbing have CPU tests. In the first city GPU trial, 186 crop +calls produced only five mask-supported poses; none met the unchanged track-start score. +No additional person track survived and source overlays still missed pedestrians. The reviewed +advertisement exclusions worked, but all-person reconstruction failed. Keep crop recovery off +by default; `recoveredCropPoses` counts supported detections, not surviving tracks. + +## Reviewed depictions and statues + +`--person-exclusions review.json` (or the tracking stage's `person_exclusions` parameter) +accepts explicit source-bound observations of a depiction or statue. It does not classify +objects automatically. Unreviewed observations remain eligible for pose recovery and tracking, +including small or stationary pedestrians. An exclusion must be visually reviewed with source +context; an agent-authored review must identify its author as an agent, not a human approval. + +The `wander.person-exclusions/1` JSON includes `sourceSha256`, source `width` and `height`, +`reviewer` (`kind`, `name`), and `entries`. Each entry records `sourceIndex`, a source-pixel +`box` in xyxy order, `label` (`depiction` or `statue`), a specific `reason`, and an `evidence` +reference. Every reviewed box must uniquely match a detector box at IoU strictly above 0.95. +Stale source/dimensions, ambiguous matches and missing detections fail rather than silently +excluding something else. Entries outside the tracking schedule are rejected. + +Excluded instances remain competing evidence during pose assignment, so a pose belonging to a +pictured person is not reassigned to its neighbor. They are recorded in `excludedInstances` +separately from unresolved `unmatchedInstances`; the exact review is retained with its SHA-256 +and reviewer in the track receipt. The review content participates in the paid-attempt identity. +It does not change retry caps or establish all-person recall, identity or scene acceptance. diff --git a/orchestrator/steps.py b/orchestrator/steps.py index d2ac600..fb65ee8 100644 --- a/orchestrator/steps.py +++ b/orchestrator/steps.py @@ -118,6 +118,11 @@ def tunables(**knobs: dict[str, Any]) -> dict[str, Any]: ), ) TRACK_PARAMETERS = tunables( + person_exclusions=knob( + "string", + "", + "path to source-bound reviewed depiction/statue exclusions; no automatic semantic approval", + ), pose_crops=knob( "boolean", False, diff --git a/scripts/run_clip.py b/scripts/run_clip.py index 602df7f..5dae114 100644 --- a/scripts/run_clip.py +++ b/scripts/run_clip.py @@ -1840,6 +1840,10 @@ def pose_guard(self): def tracks(self): """Every person in the clip, linked into identity-stable tracks (one MultiHMR pass).""" + if getattr(self.a, "person_exclusions", None): + from worker.stages.person_exclusions import load_video_exclusions + + load_video_exclusions(self.a.person_exclusions, self.clip, fps=self.a.fps) self.paid_run( "tracks", [ @@ -1855,6 +1859,11 @@ def tracks(self): "--det-thresh", str(self.a.det_thresh), *(["--pose-crops"] if getattr(self.a, "pose_crops", False) else []), + *( + ["--person-exclusions", str(self.a.person_exclusions)] + if getattr(self.a, "person_exclusions", None) + else [] + ), "--out", str(self.ctx / "tracks"), ], @@ -4359,6 +4368,9 @@ def main(): default=0.15, help="MultiHMR detection threshold for the tracking stage", ) + ap.add_argument( + "--person-exclusions", default="", help="source-bound reviewed non-person instances" + ) ap.add_argument( "--pose-crops", action="store_true", @@ -4446,8 +4458,10 @@ def main(): animation_options(execution_timeout=a.execution_timeout) except ValueError as error: ap.error(str(error)) - if a.pose_crops and (not (a.people or a.all_people) or a.orchestrated): - ap.error("--pose-crops requires direct tracked runs; use wander step for stage parameters") + if (a.pose_crops or a.person_exclusions) and (not (a.people or a.all_people) or a.orchestrated): + ap.error( + "pose crops/exclusions require direct tracked runs; use wander step for stage parameters" + ) if a.lower_body_provenance is not None: if not a.lower_body_provenance.strip(): ap.error("--lower-body-provenance must be a nonempty measured statement") diff --git a/scripts/stage_attempts.py b/scripts/stage_attempts.py index b6b3b9e..a9e1270 100644 --- a/scripts/stage_attempts.py +++ b/scripts/stage_attempts.py @@ -149,6 +149,7 @@ def _read_authorization(path): "worker/stages/track_people.py", "worker/stages/pose_instances.py", "worker/stages/pose_crops.py", + "worker/stages/person_exclusions.py", ), "worker/modal_multiperson.py::animate": ( "worker/stages/worker_resources.py", @@ -188,6 +189,7 @@ def _read_authorization(path): } TEXT_FLAGS = {"--experiment", "--mask-backend", "--alignment", "--depth-roi", "--refine-prompt"} INPUT_FLAGS = { + "--person-exclusions", "--image", "--cameras", "--masks-in", diff --git a/scripts/tests/test_person_exclusions.py b/scripts/tests/test_person_exclusions.py new file mode 100644 index 0000000..9f12f16 --- /dev/null +++ b/scripts/tests/test_person_exclusions.py @@ -0,0 +1,322 @@ +"""Reviewed exclusions are exact-source instructions, never automatic semantic labels.""" + +import copy +import hashlib +import json +from pathlib import Path +import sys +from unittest.mock import Mock + +import cv2 +import numpy as np +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from worker.stages.person_exclusions import load_exclusions, load_video_exclusions, match_exclusions + + +@pytest.fixture +def doc(): + return { + "schema": "wander.person-exclusions/1", + "sourceSha256": "ab" * 32, + "width": 640, + "height": 480, + "reviewer": {"kind": "agent", "name": "frame reviewer"}, + "entries": [ + { + "sourceIndex": 3, + "box": [100, 50, 200, 250], + "label": "depiction", + "reason": "Reviewed figure on poster", + "evidence": "frame-003.png", + } + ], + } + + +def save(tmp_path, doc): + path = tmp_path / "exclusions.json" + path.write_text(json.dumps(doc)) + return path + + +@pytest.mark.parametrize("kind,label", [("agent", "depiction"), ("human", "statue")]) +def test_valid_load_preserves_provenance_and_document(tmp_path, doc, kind, label): + doc["reviewer"]["kind"] = kind + doc["entries"][0]["label"] = label + before = copy.deepcopy(doc) + path = save(tmp_path, doc) + original_bytes = path.read_bytes() + loaded = load_exclusions(path, doc["sourceSha256"], 640, 480) + assert loaded == before == doc + assert path.read_bytes() == original_bytes + + +@pytest.mark.parametrize("box", [[100, 50, 200, 250], [101, 51, 201, 251]]) +def test_exact_and_near_match_preserve_unreviewed_and_inputs(doc, box): + boxes = np.array([[300, 0, 400, 400], box], dtype=float) + before_boxes, before_doc = boxes.copy(), copy.deepcopy(doc) + result = match_exclusions(doc, 3, boxes) + assert result == {1: doc["entries"][0]} + assert result[1] is doc["entries"][0] + assert 0 not in result + np.testing.assert_array_equal(boxes, before_boxes) + assert doc == before_doc + assert match_exclusions(doc, 4, boxes) == {} + assert match_exclusions(doc, 4, []) == {} + + +def test_empty_review_preserves_every_instance(tmp_path, doc): + doc["entries"] = [] + loaded = load_exclusions(save(tmp_path, doc), doc["sourceSha256"], 640, 480) + assert match_exclusions(loaded, 3, [[100, 50, 200, 250]]) == {} + + +@pytest.mark.parametrize("boxes", [[], [[300, 20, 400, 220]], [[100, 50, 190, 250]]]) +def test_stale_or_missing_current_box_fails(doc, boxes): + with pytest.raises(ValueError, match="stale review"): + match_exclusions(doc, 3, boxes) + + +def test_strict_iou_threshold(doc): + doc["width"] = doc["height"] = 100 + doc["entries"][0]["box"] = [0, 0, 100, 100] + with pytest.raises(ValueError, match="stale review"): + match_exclusions(doc, 3, [[0, 0, 95, 100]]) + assert match_exclusions(doc, 3, [[0, 0, 95.01, 100]]) + + +def test_multiple_candidates_and_shared_instance_are_ambiguous(doc): + box = doc["entries"][0]["box"] + with pytest.raises(ValueError, match="ambiguous review"): + match_exclusions(doc, 3, [box, box]) + second = copy.deepcopy(doc["entries"][0]) + second["box"] = [101, 50, 201, 250] + doc["entries"].append(second) + with pytest.raises(ValueError, match="ambiguous review"): + match_exclusions(doc, 3, [box]) + + +@pytest.mark.parametrize( + "sha,width,height", [("cd" * 32, 640, 480), ("ab" * 32, 641, 480), ("ab" * 32, 640, 481)] +) +def test_stale_binding(tmp_path, doc, sha, width, height): + with pytest.raises(ValueError, match="stale review"): + load_exclusions(save(tmp_path, doc), sha, width, height) + + +@pytest.mark.parametrize( + "key,value", + [ + ("schema", "unknown"), + ("sourceSha256", "x" * 64), + ("sourceSha256", "ab" * 31), + ("width", True), + ("height", 0), + ("width", 640.0), + ("reviewer", None), + ("reviewer", {"kind": "bot", "name": "x"}), + ("reviewer", {"kind": "agent", "name": " "}), + ("entries", {}), + ("entries", [None]), + ], +) +def test_malformed_document(tmp_path, doc, key, value): + doc[key] = value + with pytest.raises(ValueError): + load_exclusions(save(tmp_path, doc), "ab" * 32, 640, 480) + + +@pytest.mark.parametrize( + "key,value", + [ + ("sourceIndex", True), + ("sourceIndex", -1), + ("sourceIndex", 3.0), + ("sourceIndex", "3"), + ("box", [0, 0, 0, 10]), + ("box", [0, 0, 641, 10]), + ("box", [-1, 0, 10, 10]), + ("box", [0, 0, 10, 481]), + ("box", [0, 0, float("nan"), 10]), + ("box", [0, 0, float("inf"), 10]), + ("box", [0, 0, True, 10]), + ("box", [0, 0, "10", 10]), + ("box", [0, 0, 10]), + ("label", "person"), + ("label", []), + ("reason", " "), + ("evidence", ""), + ("evidence", ["frame.png"]), + ("instance", 0), + ], +) +def test_malformed_entry_and_unknown_keys(tmp_path, doc, key, value): + doc["entries"][0][key] = value + with pytest.raises(ValueError): + load_exclusions(save(tmp_path, doc), "ab" * 32, 640, 480) + + +@pytest.mark.parametrize("key", ["sourceIndex", "box", "label", "reason", "evidence"]) +def test_missing_entry_fields(tmp_path, doc, key): + del doc["entries"][0][key] + with pytest.raises(ValueError): + load_exclusions(save(tmp_path, doc), "ab" * 32, 640, 480) + + +def test_duplicate_frame_box_rejected_but_other_frames_allowed(tmp_path, doc): + doc["entries"].append(copy.deepcopy(doc["entries"][0])) + with pytest.raises(ValueError, match="duplicate"): + load_exclusions(save(tmp_path, doc), "ab" * 32, 640, 480) + doc["entries"][1]["sourceIndex"] = 4 + assert load_exclusions(save(tmp_path, doc), "ab" * 32, 640, 480) == doc + + +@pytest.mark.parametrize("raw", ['{"schema":', "[]", '{"schema":1,"schema":2}']) +def test_invalid_json(tmp_path, raw): + path = tmp_path / "review.json" + path.write_text(raw) + with pytest.raises(ValueError): + load_exclusions(path, "ab" * 32, 640, 480) + + +@pytest.fixture +def video_review(tmp_path, doc, monkeypatch): + video = tmp_path / "source.mp4" + video.write_bytes(b"video fixture bytes; capture mocked") + doc["sourceSha256"] = hashlib.sha256(video.read_bytes()).hexdigest() + path = save(tmp_path, doc) + capture = Mock() + capture.isOpened.return_value = True + capture.get.side_effect = [640.0, 480.0, 10.0] + monkeypatch.setattr(cv2, "VideoCapture", lambda filename: capture) + return path, video, capture + + +def test_video_wrapper_binds_bytes_and_metadata(video_review, doc): + path, video, capture = video_review + assert load_video_exclusions(path, video) == doc + capture.release.assert_called_once() + video.write_bytes(b"changed source") + capture.get.side_effect = [640.0, 480.0, 10.0] + with pytest.raises(ValueError, match="hash mismatch"): + load_video_exclusions(path, video) + assert capture.release.call_count == 2 + + +@pytest.mark.parametrize( + "metadata", + [ + [0, 480, 10], + [640, 0, 10], + [640, 480, 0], + [640.5, 480, 10], + [640, 480, 10.5], + [float("nan"), 480, 10], + [640, 480, float("inf")], + [640, 480, -1], + ], +) +def test_video_invalid_metadata_releases_capture(video_review, metadata): + path, video, capture = video_review + capture.get.side_effect = metadata + with pytest.raises(ValueError, match="positive integer"): + load_video_exclusions(path, video) + capture.release.assert_called_once() + + +def test_video_unreadable_and_out_of_range_review(video_review): + path, video, capture = video_review + capture.isOpened.return_value = False + with pytest.raises(ValueError, match="unreadable"): + load_video_exclusions(path, video) + capture.release.assert_called_once() + capture.isOpened.return_value = True + capture.get.side_effect = [640, 480, 3] + with pytest.raises(ValueError, match="frame count"): + load_video_exclusions(path, video) + assert capture.release.call_count == 2 + + +def test_video_metadata_exception_still_releases(video_review): + path, video, capture = video_review + capture.get.side_effect = RuntimeError("decoder failed") + with pytest.raises(RuntimeError, match="decoder failed"): + load_video_exclusions(path, video) + capture.release.assert_called_once() + + +def test_missing_files_rejected(tmp_path): + with pytest.raises(ValueError, match="unreadable"): + load_video_exclusions(tmp_path / "review.json", tmp_path / "absent.mp4") + with pytest.raises(ValueError, match="unreadable"): + load_exclusions(tmp_path / "review.json", "ab" * 32, 640, 480) + + +@pytest.mark.parametrize( + "frame,accepted", + [(0, True), (1, False), (2, True), (3, False), (5, True), (8, True), (9, False)], +) +def test_review_checked_against_exact_30_to_12_schedule(video_review, doc, frame, accepted): + path, video, capture = video_review + doc["entries"][0]["sourceIndex"] = frame + path.write_text(json.dumps(doc)) + original = path.read_bytes() + capture.get.side_effect = [640, 480, 10, 30.0] + if accepted: + assert load_video_exclusions(path, video, fps=12) == doc + else: + with pytest.raises(ValueError, match="tracking schedule"): + load_video_exclusions(path, video, fps=12) + capture.release.assert_called_once() + assert path.read_bytes() == original + + +@pytest.mark.parametrize( + "source_fps,target_fps", + [ + (0, 12), + (-30, 12), + (float("nan"), 12), + (float("inf"), 12), + (True, 12), + ("30", 12), + (None, 12), + (30, 0), + (30, -12), + (30, float("nan")), + (30, float("inf")), + (30, True), + (30, "12"), + (30, [12]), + ], +) +def test_malformed_fps_refused_and_capture_released(video_review, source_fps, target_fps): + path, video, capture = video_review + capture.get.side_effect = [640, 480, 10, source_fps] + with pytest.raises(ValueError, match="FPS must be finite and positive"): + load_video_exclusions(path, video, fps=target_fps) + capture.release.assert_called_once() + + +def test_sampling_matches_worker_function_without_importing_models(video_review, doc): + import ast + + path, video, capture = video_review + source = (Path(__file__).resolve().parents[2] / "worker/stages/track_people.py").read_text() + function = next( + node + for node in ast.parse(source).body + if isinstance(node, ast.FunctionDef) and node.name == "sample_indices" + ) + namespace = {"np": np, "cv2": cv2} + exec(compile(ast.Module(body=[function], type_ignores=[]), "sample_indices", "exec"), namespace) + for source_fps, count, target_fps in [(30, 10, 12), (24000 / 1001, 186, 12), (12, 5, 30)]: + capture.get.side_effect = [source_fps, count] + indices, *_ = namespace["sample_indices"](video, target_fps) + doc["entries"] = [dict(doc["entries"][0], sourceIndex=int(i)) for i in sorted(set(indices))] + path.write_text(json.dumps(doc)) + capture.get.side_effect = [640, 480, count, source_fps] + assert load_video_exclusions(path, video, fps=target_fps) == doc + assert capture.release.call_count == 6 diff --git a/scripts/tests/test_stage_attempts.py b/scripts/tests/test_stage_attempts.py index 0199a04..08e395d 100644 --- a/scripts/tests/test_stage_attempts.py +++ b/scripts/tests/test_stage_attempts.py @@ -1311,3 +1311,23 @@ def provider( if __name__ == "__main__": unittest.main() + + +def test_person_exclusion_content_changes_attempt_identity(tmp_path): + root = Path(__file__).resolve().parents[2] + review = tmp_path / "review.json" + review.write_text( + '{"entries": [{"sourceIndex": 0, "box": [0, 0, 10, 10], "label": "depiction"}]}' + ) + command = [ + "modal", + "run", + "worker/modal_multiperson.py::main", + "--person-exclusions", + str(review), + ] + first = command_identity(command, root) + review.write_text( + '{"entries": [{"sourceIndex": 0, "box": [0, 0, 20, 20], "label": "depiction"}]}' + ) + assert command_identity(command, root) != first diff --git a/scripts/tests/test_tracking_submission.py b/scripts/tests/test_tracking_submission.py index 75c6470..4c046cf 100644 --- a/scripts/tests/test_tracking_submission.py +++ b/scripts/tests/test_tracking_submission.py @@ -98,7 +98,7 @@ def get(*, timeout): launch.call.get.side_effect = get launch.run(fps=15.0, det_thresh=0.3, gate=0.4, max_gap=4, min_track_samples=6, overlay_every=2) launch.function.spawn.assert_called_once_with( - b"video bytes", b"camera bytes", 15.0, 0.3, 0.4, 4, 6, 2, False + b"video bytes", b"camera bytes", 15.0, 0.3, 0.4, 4, 6, 2, False, None ) launch.call.cancel.assert_not_called() assert launch.receipt() == initial | {"status": "result_received", "functionCallId": "fc-track"} @@ -117,7 +117,7 @@ def test_tracking_default_wait_and_reported_inference_failure(launch, monkeypatc with pytest.raises(RuntimeError, match="see the downloaded track.log"): launch.run(cameras="") launch.function.spawn.assert_called_once_with( - b"video bytes", None, 12, 0.15, 0.22, 8, 8, 0, False + b"video bytes", None, 12, 0.15, 0.22, 8, 8, 0, False, None ) launch.call.get.assert_called_once_with(timeout=7200) launch.call.cancel.assert_not_called() @@ -203,7 +203,7 @@ def test_crop_option_forwarded_without_extra_provider_submissions(launch): launch.call.get.return_value = tracking_output() launch.run(pose_crops=True) launch.function.spawn.assert_called_once() - assert launch.function.spawn.call_args.args[-1] is True + assert launch.function.spawn.call_args.args[-2] is True def test_crops_without_calibration_stop_before_claim_or_submission(launch): @@ -229,3 +229,46 @@ def test_runner_forwards_crop_option_only_when_requested(tmp_path, enabled): pipeline.paid_run.assert_called_once() command = pipeline.paid_run.call_args.args[1] assert ("--pose-crops" in command) is enabled + + +def test_reviewed_exclusions_forward_exact_bytes_after_validation(launch, tmp_path, monkeypatch): + from worker.stages import person_exclusions + + path = tmp_path / "review.json" + path.write_bytes(b"reviewed bytes") + validate = Mock(return_value={}) + monkeypatch.setattr(person_exclusions, "load_video_exclusions", validate) + launch.call.get.return_value = tracking_output() + launch.run(person_exclusions=str(path)) + validate.assert_called_once() + launch.function.spawn.assert_called_once() + assert launch.function.spawn.call_args.args[-1] == b"reviewed bytes" + + +def test_invalid_review_stops_before_submission(launch, monkeypatch): + from worker.stages import person_exclusions + + monkeypatch.setattr( + person_exclusions, "load_video_exclusions", Mock(side_effect=ValueError("stale source")) + ) + with pytest.raises(ValueError, match="stale source"): + launch.run(person_exclusions="stale.json") + launch.function.spawn.assert_not_called() + assert not launch.destination.exists() + + +def test_runner_validates_review_before_paid_claim(tmp_path, monkeypatch): + from worker.stages import person_exclusions + import run_clip + + pipeline = object.__new__(run_clip.Pipeline) + pipeline.ctx = tmp_path + pipeline.clip = tmp_path / "source.mp4" + pipeline.a = SimpleNamespace(fps=12, det_thresh=0.15, person_exclusions="stale.json") + pipeline.paid_run = Mock() + monkeypatch.setattr( + person_exclusions, "load_video_exclusions", Mock(side_effect=ValueError("stale source")) + ) + with pytest.raises(ValueError, match="stale source"): + pipeline.tracks() + pipeline.paid_run.assert_not_called() diff --git a/worker/modal_multiperson.py b/worker/modal_multiperson.py index 2be5cda..2644461 100644 --- a/worker/modal_multiperson.py +++ b/worker/modal_multiperson.py @@ -151,6 +151,7 @@ def track( min_track_samples: int, overlay_every: int, pose_crops: bool = False, + person_exclusions: bytes | None = None, ): import io import os @@ -199,6 +200,9 @@ def track( cmd += ["--cameras", str(root / "cameras.json")] if pose_crops: cmd += ["--pose-crops"] + if person_exclusions is not None: + (root / "person-exclusions.json").write_bytes(person_exclusions) + cmd += ["--person-exclusions", str(root / "person-exclusions.json")] try: with (out / "track.log").open("w") as log: proc = subprocess.Popen( @@ -296,6 +300,7 @@ def main( min_track_samples: int = 8, overlay_every: int = 0, pose_crops: bool = False, + person_exclusions: str = "", ): import io import json @@ -303,6 +308,12 @@ def main( if pose_crops and not cameras: raise ValueError("--pose-crops requires source cameras") + exclusion_bytes = None + if person_exclusions: + from worker.stages.person_exclusions import load_video_exclusions + + load_video_exclusions(person_exclusions, video, fps=fps) + exclusion_bytes = Path(person_exclusions).read_bytes() dest = Path(out) dest.mkdir(parents=True, exist_ok=True) result = _tracking_result( @@ -316,6 +327,7 @@ def main( min_track_samples, overlay_every, pose_crops, + exclusion_bytes, ) with tarfile.open(fileobj=io.BytesIO(result["archive"]), mode="r:gz") as archive: archive.extractall(dest, filter="data") diff --git a/worker/stages/person_exclusions.py b/worker/stages/person_exclusions.py new file mode 100644 index 0000000..da359b3 --- /dev/null +++ b/worker/stages/person_exclusions.py @@ -0,0 +1,201 @@ +"""Validate explicit review provenance and box correspondence, not semantic truth.""" + +import hashlib +import json +import math +from numbers import Integral, Real +from pathlib import Path +import re + + +SCHEMA = "wander.person-exclusions/1" +ENTRY_KEYS = {"sourceIndex", "box", "label", "reason", "evidence"} + + +def _integer(value, name, minimum=0): + if isinstance(value, bool) or not isinstance(value, Integral) or value < minimum: + raise ValueError(f"{name} must be an integer >= {minimum}") + return int(value) + + +def _text(value, name): + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{name} must be a nonempty string") + + +def _hash(value): + if not isinstance(value, str) or re.fullmatch(r"[0-9a-fA-F]{64}", value) is None: + raise ValueError("sourceSha256 must be 64 hexadecimal characters") + return value.lower() + + +def _box(value, width, height): + try: + if len(value) != 4: + raise ValueError + if any(isinstance(v, bool) or not isinstance(v, Real) for v in value): + raise ValueError + box = tuple(float(v) for v in value) + if not all(math.isfinite(v) for v in box): + raise ValueError + x0, y0, x1, y1 = box + if not (0 <= x0 < x1 <= width and 0 <= y0 < y1 <= height): + raise ValueError + return box + except (TypeError, ValueError, OverflowError) as exc: + raise ValueError( + "box must have four finite source pixels inside dimensions with positive extent" + ) from exc + + +def _unique_keys(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON key: {key}") + result[key] = value + return result + + +def load_exclusions(path, source_sha, width, height): + """Load a review bound to this exact source; return its unchanged JSON document.""" + expected_sha = _hash(source_sha) + width = _integer(width, "width", 1) + height = _integer(height, "height", 1) + try: + doc = json.loads(Path(path).read_text(), object_pairs_hook=_unique_keys) + except (OSError, UnicodeError, ValueError) as exc: + raise ValueError("unreadable or malformed person exclusions") from exc + if not isinstance(doc, dict) or doc.get("schema") != SCHEMA: + raise ValueError("invalid person exclusions schema") + if _hash(doc.get("sourceSha256")) != expected_sha: + raise ValueError("stale review: source hash mismatch") + review_width = _integer(doc.get("width"), "review width", 1) + review_height = _integer(doc.get("height"), "review height", 1) + if (review_width, review_height) != (width, height): + raise ValueError("stale review: source dimensions mismatch") + reviewer = doc.get("reviewer") + if not isinstance(reviewer, dict) or reviewer.get("kind") not in ("human", "agent"): + raise ValueError("reviewer kind must be human or agent") + _text(reviewer.get("name"), "reviewer name") + entries = doc.get("entries") + if not isinstance(entries, list): + raise ValueError("entries must be a list") + seen = set() + for entry in entries: + if not isinstance(entry, dict) or set(entry) != ENTRY_KEYS: + raise ValueError("entry must contain exactly sourceIndex, box, label, reason, evidence") + index = _integer(entry["sourceIndex"], "sourceIndex") + if not isinstance(entry["box"], list): + raise ValueError("entry box must be a list") + box = _box(entry["box"], width, height) + if entry["label"] not in ("depiction", "statue"): + raise ValueError("entry label must be depiction or statue") + _text(entry["reason"], "reason") + _text(entry["evidence"], "evidence") + key = (index, box) + if key in seen: + raise ValueError("duplicate reviewed frame and box") + seen.add(key) + return doc + + +def load_video_exclusions(path, video, *, fps=None): + """Bind a review to video bytes, dimensions and reported frame count. + + Container metadata is checked, not every frame decoded. This does not establish + footage readability throughout the clip or validate the reviewer's labels. + When fps is supplied, reviewed frames must belong to the tracker's exact + source-index sampling schedule (including NumPy's round-to-even behavior). + """ + import cv2 + + digest = hashlib.sha256() + try: + with Path(video).open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + except OSError as exc: + raise ValueError("unreadable source video") from exc + cap = cv2.VideoCapture(str(video)) + try: + if not cap.isOpened(): + raise ValueError("unreadable source video") + metadata = [ + cap.get(key) + for key in ( + cv2.CAP_PROP_FRAME_WIDTH, + cv2.CAP_PROP_FRAME_HEIGHT, + cv2.CAP_PROP_FRAME_COUNT, + ) + ] + values = [] + for name, value in zip(("width", "height", "frame count"), metadata): + if not isinstance(value, Real) or isinstance(value, bool): + raise ValueError(f"video {name} must be a positive integer") + if not math.isfinite(value) or value <= 0 or int(value) != value: + raise ValueError(f"video {name} must be a positive integer") + values.append(int(value)) + if fps is not None: + source_fps = cap.get(cv2.CAP_PROP_FPS) + for name, value in (("source FPS", source_fps), ("target FPS", fps)): + if ( + isinstance(value, bool) + or not isinstance(value, Real) + or not math.isfinite(value) + or value <= 0 + ): + raise ValueError(f"{name} must be finite and positive") + finally: + cap.release() + width, height, count = values + doc = load_exclusions(path, digest.hexdigest(), width, height) + if any(entry["sourceIndex"] >= count for entry in doc["entries"]): + raise ValueError("stale review: sourceIndex exceeds video frame count") + if fps is not None: + import numpy as np + + duration = count / source_fps + sample_count = duration * fps + if not math.isfinite(duration) or not math.isfinite(sample_count): + raise ValueError("tracking schedule must be finite") + requested = np.arange(int(np.ceil(sample_count))) / fps + requested = requested[requested < duration] + indices = np.clip(np.rint(requested * source_fps).astype(int), 0, count - 1) + if {entry["sourceIndex"] for entry in doc["entries"]} - set(map(int, indices)): + raise ValueError("stale review: reviewed frames outside the tracking schedule") + return doc + + +def match_exclusions(doc, source_index, instance_boxes): + """Match a validated review at one frame, preserving every unreviewed instance. + + Every reviewed box must have exactly one strict IoU > 0.95 correspondence. + Multiple reviewed boxes may not consume the same instance. Return original + entries by instance index without changing the document or the input boxes. + """ + source_index = _integer(source_index, "source_index") + entries = [entry for entry in doc["entries"] if entry["sourceIndex"] == source_index] + if not entries: + return {} + boxes = [_box(box, doc["width"], doc["height"]) for box in instance_boxes] + result = {} + for entry in entries: + reviewed = entry["box"] + matches = [] + for i, box in enumerate(boxes): + # Normalize coordinates first to avoid overflowing area arithmetic. + a = [v / scale for v, scale in zip(reviewed, (doc["width"], doc["height"]) * 2)] + b = [v / scale for v, scale in zip(box, (doc["width"], doc["height"]) * 2)] + intersection = max(0, min(a[2], b[2]) - max(a[0], b[0])) * max( + 0, min(a[3], b[3]) - max(a[1], b[1]) + ) + union = (a[2] - a[0]) * (a[3] - a[1]) + (b[2] - b[0]) * (b[3] - b[1]) - intersection + if union > 0 and intersection / union > 0.95: + matches.append(i) + if not matches: + raise ValueError(f"stale review: no matching instance at source frame {source_index}") + if len(matches) != 1 or matches[0] in result: + raise ValueError(f"ambiguous review: nonunique instance at source frame {source_index}") + result[matches[0]] = entry + return result diff --git a/worker/stages/track_people.py b/worker/stages/track_people.py index 1257139..27e3b75 100644 --- a/worker/stages/track_people.py +++ b/worker/stages/track_people.py @@ -172,10 +172,18 @@ def main(): action="store_true", help="experimental calibrated crop inference for unmatched person instances", ) + ap.add_argument( + "--person-exclusions", + default="", + help="source-bound reviewed depictions/statues to preserve", + ) a = ap.parse_args() if a.pose_crops and (a.no_maskrcnn or not a.cameras): ap.error("--pose-crops requires instance masks and source cameras") + if a.person_exclusions and a.no_maskrcnn: + ap.error("--person-exclusions requires independent instance masks") + out = Path(a.out) out.mkdir(parents=True, exist_ok=True) os.chdir(a.repo) @@ -194,7 +202,25 @@ def main(): Accelerator() source_sha = hashlib.sha256(Path(a.video).read_bytes()).hexdigest() + exclusions = None + exclusions_sha = None + if a.person_exclusions: + from person_exclusions import load_exclusions, match_exclusions + + probe = cv2.VideoCapture(a.video) + width, height = ( + int(probe.get(cv2.CAP_PROP_FRAME_WIDTH)), + int(probe.get(cv2.CAP_PROP_FRAME_HEIGHT)), + ) + probe.release() + exclusions = load_exclusions(a.person_exclusions, source_sha, width, height) + exclusions_sha = hashlib.sha256(Path(a.person_exclusions).read_bytes()).hexdigest() + (out / "person-exclusions.json").write_bytes(Path(a.person_exclusions).read_bytes()) indices, times, src_fps, count, duration = sample_indices(a.video, a.fps) + if exclusions and {entry["sourceIndex"] for entry in exclusions["entries"]} - set( + map(int, indices) + ): + raise ValueError("reviewed exclusions contain frames outside the tracking schedule") camera_doc = json.loads(Path(a.cameras).read_text()) if a.cameras else None cameras = camera_doc["cameras"] if camera_doc else None # Cameras are matched by the source frame they describe, not by position. The solve records @@ -241,6 +267,7 @@ def main(): detection_audit = [] crop_inference_calls = 0 recovered_crop_poses = 0 + excluded_instances_total = 0 w = h = None for sample, (index, timestamp) in enumerate(zip(indices, times)): @@ -310,6 +337,10 @@ def main(): ] boxes = [joints_box(joints, w, h) for joints in projected] matches = match_pose_instances(boxes, inst_boxes) if rcnn is not None else {} + excluded = match_exclusions(exclusions, int(index), inst_boxes) if exclusions else {} + # Keep excluded masks as competing evidence; never reassign their poses to a neighbor. + matches = {pose: pair for pose, pair in matches.items() if pair[0] not in excluded} + excluded_instances_total += len(excluded) pose_origins = ["full_frame"] * len(detected) crop_audit = [] if a.pose_crops: @@ -319,7 +350,7 @@ def main(): ) original_used = {pair[0] for pair in matches.values()} for instance_i, instance_box in enumerate(inst_boxes): - if instance_i in original_used: + if instance_i in original_used or instance_i in excluded: continue bounds = crop_bounds(instance_box, w, h) x0, y0, x1, y1 = bounds @@ -402,11 +433,14 @@ def main(): poseOrigins=pose_origins, cropPasses=crop_audit, instanceBoxes=inst_boxes, + excludedInstances=[dict(instance=i, **entry) for i, entry in excluded.items()], matches=[ dict(pose=i, instance=pair[0], iou=pair[1]) for i, pair in matches.items() ], unassignedPoses=rejected, - unmatchedInstances=[i for i in range(len(inst_boxes)) if i not in used_inst], + unmatchedInstances=[ + i for i in range(len(inst_boxes)) if i not in used_inst and i not in excluded + ], maskSupportRequired=rcnn is not None, ) ) @@ -462,7 +496,7 @@ def main(): poseOrigin=pose_origins[det_i], ) ) - unmatched_rcnn_total += len(inst_boxes) - len(used_inst) + unmatched_rcnn_total += len(inst_boxes) - len(used_inst) - len(excluded) # ---- association ---------------------------------------------------- live = [t for t in tracks if t.missed <= a.max_gap] @@ -711,6 +745,15 @@ def main(): maskrcnnDetectionsWithoutPose=unmatched_rcnn_total, unassignedPoseDetections=unassigned_pose_total, maskSupportRequired=not a.no_maskrcnn, + personExclusions=( + dict( + sha256=exclusions_sha, + reviewer=exclusions["reviewer"], + excludedObservations=excluded_instances_total, + ) + if exclusions + else None + ), poseCrops=a.pose_crops, cropInferenceCalls=crop_inference_calls, recoveredCropPoses=recovered_crop_poses,