From 9cb53bb928189a62ec11798a9ed871f64adfdb36 Mon Sep 17 00:00:00 2001 From: Austin Jian Date: Fri, 20 Nov 2026 23:26:36 -0500 Subject: [PATCH] feat(tracking): add opt-in calibrated pose crop recovery --- docs/known-limits.md | 2 + docs/person-motion.md | 19 +++ orchestrator/steps.py | 6 + scripts/run_clip.py | 8 + scripts/stage_attempts.py | 2 + scripts/tests/test_pose_crops.py | 171 ++++++++++++++++++++++ scripts/tests/test_pose_instances.py | 15 +- scripts/tests/test_stage_attempts.py | 12 ++ scripts/tests/test_tracking_submission.py | 38 ++++- worker/modal_multiperson.py | 7 + worker/stages/pose_crops.py | 110 ++++++++++++++ worker/stages/pose_instances.py | 21 +++ worker/stages/track_people.py | 102 ++++++++++++- 13 files changed, 509 insertions(+), 4 deletions(-) create mode 100644 scripts/tests/test_pose_crops.py create mode 100644 worker/stages/pose_crops.py diff --git a/docs/known-limits.md b/docs/known-limits.md index cb69f31..b6ddd1d 100644 --- a/docs/known-limits.md +++ b/docs/known-limits.md @@ -16,6 +16,8 @@ Do not retry one without a new idea, and add a line here when you measure a new 202 unmatched instance observations still include real background pedestrians and people pictured on signage. Do not animate every unmatched detection or call the single-track result accepted. Its body projection also remains misaligned with the observed silhouette. + Experimental `--pose-crops` adds calibrated inference for unmatched masks, but has no GPU + quality result yet and remains off by default; see [person motion](person-motion.md#experimental-tracking-crop-recovery). Evidence: `.context/evidence/city-recovery-20261120/`. - The forest scale gate fails on distant background, not on the route. Replaying the run's own diff --git a/docs/person-motion.md b/docs/person-motion.md index 42c5e08..9d8c0d8 100644 --- a/docs/person-motion.md +++ b/docs/person-motion.md @@ -102,3 +102,22 @@ Their original author Claude and co-author Claude Fable 5.1 remain credited in t Codex adapted the integration for lossless defaults, format identity, integrity verification and safe fallback. Live rendered comparisons and actual network timing remain separate acceptance checks; unit tests do not establish visual quality or headset performance. + +## Experimental tracking crop recovery + +Tracked runs can opt into `--pose-crops` (or the `pose_crops` tracking stage parameter). +It is off by default and requires source camera calibration for every sampled frame and +independent person instance masks. Each instance unmatched by the ordinary full-frame pose +pass gets one additional crop inference. The crop translates the camera principal point, +then applies the estimator's resize and padding; predicted joints are mapped back to source +pixels. Existing full-frame matches are preserved. A crop may claim only its target mask +under the existing IoU threshold. + +This adds GPU calls and can increase runtime substantially. It does not automatically retry, +change the stage retry budget, or distinguish pedestrians from people pictured on signs or +statues. `detection-audit.json` records crop bounds, detections, recoveries and pose origins; +track summaries record call counts and motion observations preserve their origin. + +The geometry and submission plumbing have CPU tests. Actual model recovery and visual +quality have not yet been established; do not enable this by default or treat recovered +detections as an accepted all-people reconstruction. diff --git a/orchestrator/steps.py b/orchestrator/steps.py index ba49423..d2ac600 100644 --- a/orchestrator/steps.py +++ b/orchestrator/steps.py @@ -118,6 +118,12 @@ def tunables(**knobs: dict[str, Any]) -> dict[str, Any]: ), ) TRACK_PARAMETERS = tunables( + pose_crops=knob( + "boolean", + False, + "experimental additional calibrated pose pass for unmatched person instances; " + "adds inference cost and does not distinguish depicted from real people", + ), fps=knob( "number", 12, diff --git a/scripts/run_clip.py b/scripts/run_clip.py index 990de6c..602df7f 100644 --- a/scripts/run_clip.py +++ b/scripts/run_clip.py @@ -1854,6 +1854,7 @@ def tracks(self): str(self.a.fps), "--det-thresh", str(self.a.det_thresh), + *(["--pose-crops"] if getattr(self.a, "pose_crops", False) else []), "--out", str(self.ctx / "tracks"), ], @@ -4358,6 +4359,11 @@ def main(): default=0.15, help="MultiHMR detection threshold for the tracking stage", ) + ap.add_argument( + "--pose-crops", + action="store_true", + help="experimental crop pose recovery for unmatched tracked instances", + ) ap.add_argument( "--shot", type=int, @@ -4440,6 +4446,8 @@ def main(): animation_options(execution_timeout=a.execution_timeout) except ValueError as error: ap.error(str(error)) + if a.pose_crops and (not (a.people or a.all_people) or a.orchestrated): + ap.error("--pose-crops requires direct tracked runs; use wander step for stage parameters") if a.lower_body_provenance is not None: if not a.lower_body_provenance.strip(): ap.error("--lower-body-provenance must be a nonempty measured statement") diff --git a/scripts/stage_attempts.py b/scripts/stage_attempts.py index 71d72f2..b6b3b9e 100644 --- a/scripts/stage_attempts.py +++ b/scripts/stage_attempts.py @@ -148,6 +148,7 @@ def _read_authorization(path): "worker/stages/worker_resources.py", "worker/stages/track_people.py", "worker/stages/pose_instances.py", + "worker/stages/pose_crops.py", ), "worker/modal_multiperson.py::animate": ( "worker/stages/worker_resources.py", @@ -160,6 +161,7 @@ def _read_authorization(path): } OUTPUT_FLAGS = {"--out", "--out-dir", "--frames-out", "--frame0", "--masks", "--report"} BOOL_FLAGS = { + "--pose-crops", "--anchor-lock", "--allow-person-gaps", "--allow-missing-person-reference", diff --git a/scripts/tests/test_pose_crops.py b/scripts/tests/test_pose_crops.py new file mode 100644 index 0000000..0015320 --- /dev/null +++ b/scripts/tests/test_pose_crops.py @@ -0,0 +1,171 @@ +"""Projection and inverse-transform checks, independent of any pose model.""" + +import sys +from pathlib import Path + +import numpy as np +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from worker.stages.pose_crops import crop_bounds, crop_intrinsics, source_joints + + +@pytest.mark.parametrize( + "box,expected", + [ + ([400.25, 100.75, 700.5, 500.25], (340, 20, 761, 581)), + ([-20, 40, 180, 300], (0, 0, 220, 352)), + ([1100, 600, 1300, 800], (1060, 560, 1280, 720)), + ], +) +def test_bounds_round_outward_and_clamp(box, expected): + original = np.array(box) + assert crop_bounds(original, 1280, 720) == expected + np.testing.assert_array_equal(original, box) + assert crop_bounds([2.2, 3.3, 4.4, 5.5], 1280, 720, padding=0) == (2, 3, 5, 6) + + +@pytest.mark.parametrize("box", [[400, 100, 700, 500], [-20, 40, 180, 300]]) +def test_calibrated_crop_preserves_3d_rays_and_source_projection(box): + bounds = crop_bounds(box, 1280, 720) + K = np.array([[920.0, 31.0, 603.5], [0.0, 870.0, 341.25], [0.0, 0.0, 1.0]]) + original = K.copy() + points = np.array([[-0.7, -0.3, 2.0], [0.4, 0.5, 3.0], [1.2, -0.2, 4.0]]) + source_h = (K @ points.T).T + source_pixels = source_h[:, :2] / source_h[:, 2:] + crop_K = crop_intrinsics(K, bounds) + crop_h = (crop_K @ points.T).T + crop_pixels = crop_h[:, :2] / crop_h[:, 2:] + np.testing.assert_allclose(crop_pixels + bounds[:2], source_pixels, atol=1e-12) + rays = np.linalg.solve(crop_K, np.column_stack((crop_pixels, np.ones(3))).T).T + np.testing.assert_allclose(rays, points / points[:, 2:], atol=1e-12) + assert crop_K[0, 1] == K[0, 1] + np.testing.assert_array_equal(K, original) + + # Asymmetric center padding and preprocessing padding, with a non-unit resize. + pad_left, pad_top, factor, offset_w, offset_h = 17, 29, 0.73, 43, 11 + preprocessing = np.array( + [ + [factor, 0, factor * offset_w + pad_left], + [0, factor, factor * offset_h + pad_top], + [0, 0, 1], + ] + ) + model_h = (preprocessing @ crop_K @ points.T).T + model_pixels = model_h[:, :2] / model_h[:, 2:] + before = model_pixels.copy() + restored = source_joints(model_pixels, pad_left, pad_top, factor, offset_w, offset_h, bounds) + np.testing.assert_allclose(restored, source_pixels, atol=1e-12) + np.testing.assert_array_equal(model_pixels, before) + + +def test_inverse_does_not_clip_predictions_to_crop(): + expected = np.array([[-5.0, 80.0], [900.0, 1200.0]]) + bounds = (100, 200, 400, 600) + predicted = (expected - bounds[:2] + [9, 21]) * 1.7 + [3, 15] + np.testing.assert_allclose(source_joints(predicted, 3, 15, 1.7, 9, 21, bounds), expected) + + +@pytest.mark.parametrize( + "box", + [ + [], + [1, 2, 3], + [[1, 2, 3, 4]], + [0, 0, 0, 1], + [3, 0, 2, 1], + [0, 0, np.nan, 1], + [0, 0, np.inf, 1], + [1300, 0, 1400, 40], + [-40, 0, 0, 40], + [0, 720, 10, 800], + ], +) +def test_invalid_boxes(box): + with pytest.raises(ValueError): + crop_bounds(box, 1280, 720) + + +@pytest.mark.parametrize( + "name,value", + [ + ("width", 0), + ("height", -1), + ("width", 1.5), + ("width", True), + ("height", np.inf), + ("padding", -0.1), + ("padding", np.nan), + ("padding", [0.2]), + ("padding", "0.2"), + ], +) +def test_invalid_crop_parameters(name, value): + kwargs = dict(width=1280, height=720, padding=0.2) + kwargs[name] = value + with pytest.raises(ValueError): + crop_bounds([10, 10, 100, 100], **kwargs) + + +@pytest.mark.parametrize( + "bounds", [[], [0, 0, 1], [0, 0, 0, 1], [-1, 0, 2, 3], [0.5, 0, 2, 3], [0, 0, np.inf, 3]] +) +def test_invalid_bounds_rejected_by_both_transforms(bounds): + with pytest.raises(ValueError): + crop_intrinsics(np.eye(3), bounds) + with pytest.raises(ValueError): + source_joints([[1, 2]], 0, 0, 1, 0, 0, bounds) + + +@pytest.mark.parametrize( + "matrix", + [ + np.eye(2), + np.zeros((3, 3)), + [[1, 0, 0], [0, -1, 0], [0, 0, 1]], + [[1, 0, 0], [0, 1, 0], [0, 0, 2]], + [[1, np.nan, 0], [0, 1, 0], [0, 0, 1]], + [[1, 0, 0], [1, 1, 0], [0, 0, 1]], + ], +) +def test_invalid_intrinsics(matrix): + with pytest.raises(ValueError): + crop_intrinsics(matrix, (10, 20, 30, 40)) + + +@pytest.mark.parametrize( + "name,value", + [ + ("joints", []), + ("joints", [1, 2]), + ("joints", [[1, 2, 3]]), + ("joints", [[np.nan, 2]]), + ("factor", 0), + ("factor", -1), + ("factor", np.inf), + ("pad_left", -1), + ("pad_top", np.nan), + ("offset_w", -1), + ("offset_h", [1]), + ], +) +def test_invalid_inverse_inputs(name, value): + kwargs = dict( + joints=[[1, 2]], + pad_left=0, + pad_top=0, + factor=1, + offset_w=0, + offset_h=0, + bounds=(10, 20, 30, 40), + ) + kwargs[name] = value + with pytest.raises(ValueError): + source_joints(**kwargs) + + +def test_overflow_fails_instead_of_returning_nonfinite_geometry(): + with pytest.raises(ValueError): + crop_bounds([-1e308, 0, 1e308, 20], 1280, 720) + with pytest.raises(ValueError): + source_joints([[1e308, 2]], 0, 0, 1e-308, 0, 0, (0, 0, 10, 10)) diff --git a/scripts/tests/test_pose_instances.py b/scripts/tests/test_pose_instances.py index 13c253c..ef11aa5 100644 --- a/scripts/tests/test_pose_instances.py +++ b/scripts/tests/test_pose_instances.py @@ -7,7 +7,7 @@ import pytest sys.path.insert(0, str(Path(__file__).resolve().parents[2])) -from worker.stages.pose_instances import match_pose_instances +from worker.stages.pose_instances import match_pose_instances, match_crop_instance def test_lamp_pose_is_not_a_person_instance(): @@ -52,3 +52,16 @@ def test_degenerate_boxes_cannot_supply_support(): def test_nonfinite_boxes_fail_instead_of_becoming_a_match(): with pytest.raises(ValueError, match="finite"): match_pose_instances([[0, 0, np.nan, 20]], [[0, 0, 10, 20]]) + + +def test_crop_cannot_relabel_neighbor_as_missing_person(): + instances = [[0, 0, 20, 80], [10, 0, 30, 80]] + assert match_pose_instances([instances[0]], [instances[1]]) # old target-only false positive + assert match_crop_instance([instances[0]], instances, 1) == {} + assert match_crop_instance([instances[0], instances[1]], instances, 1) == {1: (1, 1.0)} + + +def test_crop_refuses_ambiguous_support_and_duplicate_recovery(): + assert match_crop_instance([[0, 0, 20, 80]], [[0, 0, 20, 80]] * 2, 1) == {} + assert len(match_crop_instance([[0, 0, 20, 80]] * 2, [[0, 0, 20, 80]], 0)) == 1 + assert match_crop_instance([], [[0, 0, 20, 80]], 0) == {} diff --git a/scripts/tests/test_stage_attempts.py b/scripts/tests/test_stage_attempts.py index 409ee0a..0199a04 100644 --- a/scripts/tests/test_stage_attempts.py +++ b/scripts/tests/test_stage_attempts.py @@ -690,6 +690,18 @@ def test_tracking_instance_matching_changes_are_fingerprinted(self): _, after, _ = command_identity(command, self.root) self.assertNotEqual(before[helper], after[helper]) + def test_crop_geometry_and_option_change_tracking_attempt_identity(self): + command = ["modal", "run", "worker/modal_multiperson.py::main"] + params, before, _ = command_identity(command, self.root) + changed, _, _ = command_identity(command + ["--pose-crops"], self.root) + self.assertNotEqual(params, changed) + helper = "worker/stages/pose_crops.py" + self.assertIn(helper, before) + path = self.root / helper + path.write_bytes(path.read_bytes() + b"\n# changed crop calibration\n") + _, after, _ = command_identity(command, self.root) + self.assertNotEqual(before[helper], after[helper]) + def test_single_person_lhm_deadline_guard_changes_are_fingerprinted(self): helper = "worker/stages/lhm_execution.py" command = ["modal", "run", "worker/modal_lhm.py"] diff --git a/scripts/tests/test_tracking_submission.py b/scripts/tests/test_tracking_submission.py index a6af88f..75c6470 100644 --- a/scripts/tests/test_tracking_submission.py +++ b/scripts/tests/test_tracking_submission.py @@ -98,7 +98,7 @@ def get(*, timeout): launch.call.get.side_effect = get launch.run(fps=15.0, det_thresh=0.3, gate=0.4, max_gap=4, min_track_samples=6, overlay_every=2) launch.function.spawn.assert_called_once_with( - b"video bytes", b"camera bytes", 15.0, 0.3, 0.4, 4, 6, 2 + b"video bytes", b"camera bytes", 15.0, 0.3, 0.4, 4, 6, 2, False ) launch.call.cancel.assert_not_called() assert launch.receipt() == initial | {"status": "result_received", "functionCallId": "fc-track"} @@ -116,7 +116,9 @@ def test_tracking_default_wait_and_reported_inference_failure(launch, monkeypatc launch.call.get.return_value = tracking_output("inference failed") with pytest.raises(RuntimeError, match="see the downloaded track.log"): launch.run(cameras="") - launch.function.spawn.assert_called_once_with(b"video bytes", None, 12, 0.15, 0.22, 8, 8, 0) + launch.function.spawn.assert_called_once_with( + b"video bytes", None, 12, 0.15, 0.22, 8, 8, 0, False + ) launch.call.get.assert_called_once_with(timeout=7200) launch.call.cancel.assert_not_called() assert launch.receipt()["status"] == "result_received" @@ -195,3 +197,35 @@ def write(path, receipt): assert receipt["status"] == "wait_interrupted" assert receipt["functionCallId"] == "fc-track" assert receipt["cancellation"] == "requested" + + +def test_crop_option_forwarded_without_extra_provider_submissions(launch): + launch.call.get.return_value = tracking_output() + launch.run(pose_crops=True) + launch.function.spawn.assert_called_once() + assert launch.function.spawn.call_args.args[-1] is True + + +def test_crops_without_calibration_stop_before_claim_or_submission(launch): + with pytest.raises(ValueError, match="requires source cameras"): + launch.run(pose_crops=True, cameras="") + launch.function.spawn.assert_not_called() + assert not launch.destination.exists() + + +@pytest.mark.parametrize("enabled", [False, True]) +def test_runner_forwards_crop_option_only_when_requested(tmp_path, enabled): + sys.path.insert(0, str(ROOT / "scripts")) + import run_clip + + pipeline = object.__new__(run_clip.Pipeline) + pipeline.ctx = tmp_path + pipeline.clip = tmp_path / "source.mp4" + pipeline.a = SimpleNamespace(fps=12, det_thresh=0.15, pose_crops=enabled) + pipeline.paid_run = Mock() + pipeline.track_doc = lambda: {"trackCount": 0, "tracks": []} + pipeline.identity_guard = Mock() + pipeline.tracks() + pipeline.paid_run.assert_called_once() + command = pipeline.paid_run.call_args.args[1] + assert ("--pose-crops" in command) is enabled diff --git a/worker/modal_multiperson.py b/worker/modal_multiperson.py index 4851b4d..2be5cda 100644 --- a/worker/modal_multiperson.py +++ b/worker/modal_multiperson.py @@ -150,6 +150,7 @@ def track( max_gap: int, min_track_samples: int, overlay_every: int, + pose_crops: bool = False, ): import io import os @@ -196,6 +197,8 @@ def track( ] if cameras: cmd += ["--cameras", str(root / "cameras.json")] + if pose_crops: + cmd += ["--pose-crops"] try: with (out / "track.log").open("w") as log: proc = subprocess.Popen( @@ -292,11 +295,14 @@ def main( max_gap: int = 8, min_track_samples: int = 8, overlay_every: int = 0, + pose_crops: bool = False, ): import io import json import tarfile + if pose_crops and not cameras: + raise ValueError("--pose-crops requires source cameras") dest = Path(out) dest.mkdir(parents=True, exist_ok=True) result = _tracking_result( @@ -309,6 +315,7 @@ def main( max_gap, min_track_samples, overlay_every, + pose_crops, ) with tarfile.open(fileobj=io.BytesIO(result["archive"]), mode="r:gz") as archive: archive.extractall(dest, filter="data") diff --git a/worker/stages/pose_crops.py b/worker/stages/pose_crops.py new file mode 100644 index 0000000..2f56bd2 --- /dev/null +++ b/worker/stages/pose_crops.py @@ -0,0 +1,110 @@ +"""CPU geometry for experimental calibrated crops; no inference validation implied.""" + +import numpy as np + + +def _finite(value, name): + try: + raw = np.asarray(value) + if raw.dtype.kind not in "iuf": + raise ValueError + array = raw.astype(np.float64) + if not np.isfinite(array).all(): + raise ValueError + except (TypeError, ValueError, OverflowError) as exc: + raise ValueError(f"{name} must contain finite real numbers") from exc + return array + + +def _scalar(value, name, positive=False, integer=False): + array = _finite(value, name) + if array.shape != (): + raise ValueError(f"{name} must be a scalar") + number = float(array) + if number < 0 or (positive and number == 0) or (integer and not number.is_integer()): + raise ValueError( + f"{name} must be {'positive' if positive else 'nonnegative'}" + f"{' and integral' if integer else ''}" + ) + return number + + +def _bounds(bounds): + values = _finite(bounds, "bounds") + if ( + values.shape != (4,) + or (values < 0).any() + or (values != np.floor(values)).any() + or values[2] <= values[0] + or values[3] <= values[1] + ): + raise ValueError("bounds must be nonnegative integer xyxy with positive extent") + return values + + +def crop_bounds(box, width, height, padding=0.2): + """Return half-open integer xyxy, padding each side by a fraction of box size. + + Round outward, then clamp to the source. The original box must intersect the + image; padding cannot turn an entirely off-image detection into a valid crop. + """ + width = _scalar(width, "width", positive=True, integer=True) + height = _scalar(height, "height", positive=True, integer=True) + padding = _scalar(padding, "padding") + box = _finite(box, "box") + if box.shape != (4,) or box[2] <= box[0] or box[3] <= box[1]: + raise ValueError("box must be xyxy with positive extent") + if box[0] >= width or box[1] >= height or box[2] <= 0 or box[3] <= 0: + raise ValueError("box must intersect the source image") + with np.errstate(over="ignore", invalid="ignore"): + margin = (box[2:] - box[:2]) * padding + expanded = np.concatenate((box[:2] - margin, box[2:] + margin)) + if not np.isfinite(expanded).all(): + raise ValueError("padded box must be finite") + low = np.maximum(np.floor(expanded[:2]), 0) + high = np.minimum(np.ceil(expanded[2:]), [width, height]) + return tuple(int(v) for v in np.concatenate((low, high))) + + +def crop_intrinsics(K, bounds): + """Translate calibrated source intrinsics into crop pixels, preserving skew. + + K must be a normalized pinhole intrinsic matrix with positive focal lengths. + Its principal point may lie outside the crop (or the source image). + """ + matrix = _finite(K, "K") + bounds = _bounds(bounds) + if ( + matrix.shape != (3, 3) + or matrix[0, 0] <= 0 + or matrix[1, 1] <= 0 + or not np.array_equal(matrix[2], [0, 0, 1]) + or matrix[1, 0] != 0 + ): + raise ValueError("K must be normalized upper-triangular pinhole intrinsics") + translation = np.array([[1, 0, -bounds[0]], [0, 1, -bounds[1]], [0, 0, 1]]) + with np.errstate(over="ignore", invalid="ignore"): + result = translation @ matrix + if not np.isfinite(result).all(): + raise ValueError("crop intrinsics must be finite") + return result + + +def source_joints(joints, pad_left, pad_top, factor, offset_w, offset_h, bounds): + """Map an N x 2 estimator pixel array back to original source pixels. + + Undo preprocessing padding, resize, and center padding in that order, then + add the crop origin. Predictions outside the crop remain unclipped. + """ + joints = _finite(joints, "joints") + if joints.ndim != 2 or joints.shape[1] != 2 or not len(joints): + raise ValueError("joints must be a nonempty N x 2 array") + pad = [_scalar(pad_left, "pad_left"), _scalar(pad_top, "pad_top")] + scale = _scalar(factor, "factor", positive=True) + offset = [_scalar(offset_w, "offset_w"), _scalar(offset_h, "offset_h")] + origin = _bounds(bounds)[:2] + with np.errstate(over="ignore", invalid="ignore", divide="ignore"): + result = (joints - pad) / scale - offset + origin + if not np.isfinite(result).all(): + raise ValueError("source joints must be finite") + return result diff --git a/worker/stages/pose_instances.py b/worker/stages/pose_instances.py index c595077..c3d45a9 100644 --- a/worker/stages/pose_instances.py +++ b/worker/stages/pose_instances.py @@ -39,3 +39,24 @@ def match_pose_instances(pose_boxes, instance_boxes): for row, col in zip(rows, cols) if col < len(instances) and supported[row, col] } + + +def match_crop_instance(pose_boxes, instance_boxes, target): + """Recover at most one pose only when its unique best support is the target. + + Competing masks include already matched people. Ambiguous equal support is + refused so a neighboring person's crop pose cannot be relabelled as recovery. + """ + eligible = [] + for pose_i, box in enumerate(pose_boxes): + overlaps = [match_pose_instances([box], [instance]) for instance in instance_boxes] + target_pair = overlaps[target].get(0) + if target_pair is None: + continue + if any( + i != target and pair and pair[0][1] >= target_pair[1] for i, pair in enumerate(overlaps) + ): + continue + eligible.append(pose_i) + selected = match_pose_instances([pose_boxes[i] for i in eligible], [instance_boxes[target]]) + return {eligible[i]: (target, pair[1]) for i, pair in selected.items()} diff --git a/worker/stages/track_people.py b/worker/stages/track_people.py index 689be8a..1257139 100644 --- a/worker/stages/track_people.py +++ b/worker/stages/track_people.py @@ -167,7 +167,14 @@ def main(): ap.add_argument("--overlay-every", type=int, default=0, help="0 = only first/middle/last") ap.add_argument("--mask-scale", type=float, default=0.5, help="stored mask resolution") ap.add_argument("--no-maskrcnn", action="store_true") + ap.add_argument( + "--pose-crops", + action="store_true", + help="experimental calibrated crop inference for unmatched person instances", + ) a = ap.parse_args() + if a.pose_crops and (a.no_maskrcnn or not a.cameras): + ap.error("--pose-crops requires instance masks and source cameras") out = Path(a.out) out.mkdir(parents=True, exist_ok=True) @@ -178,7 +185,8 @@ def main(): from accelerate import Accelerator from engine.pose_estimation.pose_estimator import PoseEstimator from scipy.optimize import linear_sum_assignment - from pose_instances import match_pose_instances + from pose_instances import match_pose_instances, match_crop_instance + from pose_crops import crop_bounds, crop_intrinsics, source_joints torch._dynamo.config.disable = True torch.set_num_threads(4) @@ -209,6 +217,9 @@ def main(): flush=True, ) + if a.pose_crops and any(int(i) not in camera_by_source for i in indices): + raise ValueError("--pose-crops requires source calibration for every sampled frame") + rcnn = None if not a.no_maskrcnn: from torchvision.models.detection import ( @@ -228,6 +239,8 @@ def main(): unmatched_rcnn_total = 0 unassigned_pose_total = 0 detection_audit = [] + crop_inference_calls = 0 + recovered_crop_poses = 0 w = h = None for sample, (index, timestamp) in enumerate(zip(indices, times)): @@ -297,6 +310,86 @@ def main(): ] boxes = [joints_box(joints, w, h) for joints in projected] matches = match_pose_instances(boxes, inst_boxes) if rcnn is not None else {} + pose_origins = ["full_frame"] * len(detected) + crop_audit = [] + if a.pose_crops: + if frame_camera is None: + raise RuntimeError( + f"crop inference requires a measured camera at source frame {index}" + ) + original_used = {pair[0] for pair in matches.values()} + for instance_i, instance_box in enumerate(inst_boxes): + if instance_i in original_used: + continue + bounds = crop_bounds(instance_box, w, h) + x0, y0, x1, y1 = bounds + cropped = raw[y0:y1, x0:x1] + padded_crop, crop_ow, crop_oh = estimator.img_center_padding(cropped) + crop_tensor, crop_annotation = estimator._preprocess(padded_crop) + crop_pl, crop_pt, crop_factor, _, _ = crop_annotation + crop_K = crop_intrinsics(source_K, bounds) + crop_affine = np.array( + [ + [crop_factor, 0, crop_factor * crop_ow + crop_pl], + [0, crop_factor, crop_factor * crop_oh + crop_pt], + [0, 0, 1], + ] + ) + network_K = torch.tensor(crop_affine @ crop_K, dtype=torch.float32, device="cuda")[ + None + ] + with torch.inference_mode(), torch.autocast("cuda", dtype=torch.float16): + crop_people = estimator.mhmr_model( + crop_tensor, + is_training=False, + nms_kernel_size=3, + det_thresh=a.det_thresh, + K=network_K, + idx=None, + max_dist=None, + ) + crop_inference_calls += 1 + crop_joints = [ + source_joints( + person["j2d"].float().cpu().numpy(), + crop_pl, + crop_pt, + crop_factor, + crop_ow, + crop_oh, + bounds, + ) + for person in crop_people + ] + crop_boxes = [joints_box(joints, w, h) for joints in crop_joints] + # A crop cannot replace an established full-frame match or claim another mask. + selected = match_crop_instance(crop_boxes, inst_boxes, instance_i) + first = len(detected) + # Drop meshes and detach retained fields before another crop forward. + crop_people = [ + { + k: v.detach().float().cpu() + for k, v in person.items() + if k in ("j2d", "j3d", "scores", "rotvec", "shape", "transl_pelvis") + } + for person in crop_people + ] + detected.extend(crop_people) + projected.extend(crop_joints) + boxes.extend(crop_boxes) + pose_origins.extend([f"instance_crop:{instance_i}"] * len(crop_people)) + for local_i, (_, overlap) in selected.items(): + matches[first + local_i] = (instance_i, overlap) + recovered_crop_poses += 1 + crop_audit.append( + dict( + instance=instance_i, + bounds=list(bounds), + detections=len(crop_people), + recovered=bool(selected), + ) + ) + del crop_tensor, network_K, crop_people used_inst = {pair[0] for pair in matches.values()} rejected = [i for i in range(len(detected)) if rcnn is not None and i not in matches] unassigned_pose_total += len(rejected) @@ -306,6 +399,8 @@ def main(): sourceIndex=int(index), poseBoxes=boxes, poseScores=[float(person["scores"]) for person in detected], + poseOrigins=pose_origins, + cropPasses=crop_audit, instanceBoxes=inst_boxes, matches=[ dict(pose=i, instance=pair[0], iou=pair[1]) for i, pair in matches.items() @@ -364,6 +459,7 @@ def main(): }, source_intrinsics=source_K.tolist(), maskIou=best_iou, + poseOrigin=pose_origins[det_i], ) ) unmatched_rcnn_total += len(inst_boxes) - len(used_inst) @@ -506,6 +602,7 @@ def main(): box=o["box"], maskBox=o["maskBox"], maskIou=o["maskIou"], + poseOrigin=o["poseOrigin"], occludedFraction=o.get("occludedFraction", 0.0), depth=o["depth"], heightFraction=o["heightFrac"], @@ -614,6 +711,9 @@ def main(): maskrcnnDetectionsWithoutPose=unmatched_rcnn_total, unassignedPoseDetections=unassigned_pose_total, maskSupportRequired=not a.no_maskrcnn, + poseCrops=a.pose_crops, + cropInferenceCalls=crop_inference_calls, + recoveredCropPoses=recovered_crop_poses, instanceAssociation=( "disabled; explicit unverified rectangle-mask mode" if a.no_maskrcnn