diff --git a/docs/DEVELOPING.md b/docs/DEVELOPING.md index 7dc56c6..8aae321 100644 --- a/docs/DEVELOPING.md +++ b/docs/DEVELOPING.md @@ -20,6 +20,18 @@ Code identifiers, env vars and the package still use the project's old name, Wan The source video owns the timeline; the XR runtime owns head motion. Scene manifests and media stay outside Git. Viewer installation and builds do not launch inference or download model weights. +## Person tracking + +Multi-person tracking requires independent Mask R-CNN support for each pose by default. +Pose boxes and instance boxes are matched one-to-one, maximizing supported pairs first and +then total overlap, with the existing strict IoU > 0.2 threshold. Unsupported poses cannot +create rectangular person masks or compete for a track. The explicit `--no-maskrcnn` worker +mode remains unverified and uses rectangles. This is not a detector recall guarantee: +unmatched person instances still lack animation poses and must be reviewed before delivery. +`detection-audit.json` retains per-sample pose/instance boxes, assignments and unmatched indices; +`tracks.json` reports unsupported pose and unmatched instance counts separately. Counts are +observations across frames, not unique people. Original source images and quality gates are unchanged. + ## Viewer notes Open **http://127.0.0.1:5399/demo.html**. The root URL redirects here. The local Bun server loads diff --git a/docs/known-limits.md b/docs/known-limits.md index 615a0db..cb69f31 100644 --- a/docs/known-limits.md +++ b/docs/known-limits.md @@ -9,7 +9,14 @@ Do not retry one without a new idea, and add a line here when you measure a new pedestrians were missed. The runner formerly marked tracking successful by skipping the failed subset, even with `--all-people`. All-person runs now stop on any failed identity audit rather than discarding those tracks. This repairs acceptance, not the tracker: no avatar or world was - generated from this result. Evidence: `.context/evidence/city-recovery-20261120/`. + generated from this result. A subsequent one-to-one instance-support correction retained the + main walker as one ID across all 94 samples, with no gaps or duplicate instance assignments; + beginning/middle/end overlays no longer tracked lamps. All 240 retained elevator and 75 + hall-walk associations were unchanged in offline replay. This does not fix all-person recall: + 202 unmatched instance observations still include real background pedestrians and people + pictured on signage. Do not animate every unmatched detection or call the single-track + result accepted. Its body projection also remains misaligned with the observed silhouette. + Evidence: `.context/evidence/city-recovery-20261120/`. - The forest scale gate fails on distant background, not on the route. Replaying the run's own scale diagnostic (unchanged inputs, scale0 0.7414, whole-frame p10 0.842–0.848) split by each diff --git a/scripts/stage_attempts.py b/scripts/stage_attempts.py index 26f4411..71d72f2 100644 --- a/scripts/stage_attempts.py +++ b/scripts/stage_attempts.py @@ -147,6 +147,7 @@ def _read_authorization(path): "worker/modal_multiperson.py::main": ( "worker/stages/worker_resources.py", "worker/stages/track_people.py", + "worker/stages/pose_instances.py", ), "worker/modal_multiperson.py::animate": ( "worker/stages/worker_resources.py", diff --git a/scripts/tests/test_pose_instances.py b/scripts/tests/test_pose_instances.py new file mode 100644 index 0000000..13c253c --- /dev/null +++ b/scripts/tests/test_pose_instances.py @@ -0,0 +1,54 @@ +"""Independent mask support must not create duplicate or unobserved people.""" + +import sys +from pathlib import Path + +import numpy as np +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from worker.stages.pose_instances import match_pose_instances + + +def test_lamp_pose_is_not_a_person_instance(): + assert match_pose_instances([[0, 0, 10, 50], [60, 0, 90, 80]], [[60, 0, 90, 80]]) == { + 1: (0, 1.0) + } + + +def test_duplicate_poses_cannot_claim_same_person(): + pairs = match_pose_instances([[0, 0, 20, 80], [0, 0, 10, 80]], [[0, 0, 20, 80]]) + assert pairs == {0: (0, 1.0)} + + +def test_global_matching_preserves_second_person_over_greedy_choice(): + # First pose supports both masks; second supports only the first mask. + poses = [[0, 0, 18, 10], [0, 0, 6, 10]] + masks = [[0, 0, 10, 10], [10, 0, 20, 10]] + result = match_pose_instances(poses, masks) + assert {i: pair[0] for i, pair in result.items()} == {0: 1, 1: 0} + + +def test_separated_people_remain_separate_and_order_independent(): + boxes = [[0, 0, 10, 30], [40, 0, 50, 30], [80, 0, 90, 30]] + assert match_pose_instances(boxes, boxes) == {i: (i, 1.0) for i in range(3)} + assert match_pose_instances(boxes[::-1], boxes) == {i: (2 - i, 1.0) for i in range(3)} + + +@pytest.mark.parametrize("poses,masks", [([], []), ([], [[0, 0, 1, 1]]), ([[0, 0, 1, 1]], [])]) +def test_absent_support_is_not_fabricated(poses, masks): + assert match_pose_instances(poses, masks) == {} + + +def test_existing_strict_overlap_threshold_is_preserved(): + assert match_pose_instances([[0, 0, 2, 10]], [[0, 0, 10, 10]]) == {} + assert match_pose_instances([[0, 0, 2.01, 10]], [[0, 0, 10, 10]]) + + +def test_degenerate_boxes_cannot_supply_support(): + assert match_pose_instances([[0, 0, 0, 20], [10, 10, 0, 0]], [[0, 0, 10, 20]]) == {} + + +def test_nonfinite_boxes_fail_instead_of_becoming_a_match(): + with pytest.raises(ValueError, match="finite"): + match_pose_instances([[0, 0, np.nan, 20]], [[0, 0, 10, 20]]) diff --git a/scripts/tests/test_stage_attempts.py b/scripts/tests/test_stage_attempts.py index 00676dd..409ee0a 100644 --- a/scripts/tests/test_stage_attempts.py +++ b/scripts/tests/test_stage_attempts.py @@ -680,6 +680,16 @@ def test_motion_submission_and_receipt_changes_are_fingerprinted(self): _, after, _ = command_identity(command, self.root) self.assertNotEqual(before[helper], after[helper]) + def test_tracking_instance_matching_changes_are_fingerprinted(self): + helper = "worker/stages/pose_instances.py" + command = ["modal", "run", "worker/modal_multiperson.py::main"] + _, before, _ = command_identity(command, self.root) + self.assertIn(helper, before) + path = self.root / helper + path.write_bytes(path.read_bytes() + b"\n# changed independent mask support\n") + _, after, _ = command_identity(command, self.root) + self.assertNotEqual(before[helper], after[helper]) + def test_single_person_lhm_deadline_guard_changes_are_fingerprinted(self): helper = "worker/stages/lhm_execution.py" command = ["modal", "run", "worker/modal_lhm.py"] diff --git a/worker/stages/pose_instances.py b/worker/stages/pose_instances.py new file mode 100644 index 0000000..c595077 --- /dev/null +++ b/worker/stages/pose_instances.py @@ -0,0 +1,41 @@ +"""One-to-one support for pose detections from independent person instances.""" + +import numpy as np +from scipy.optimize import linear_sum_assignment + +# Preserve the existing strict support threshold used by track_people. +MIN_INSTANCE_IOU = 0.2 + + +def match_pose_instances(pose_boxes, instance_boxes): + """Return {pose index: (instance index, IoU)} for supported detections. + + Maximize the number of supported pairs, then their total IoU. Each instance can + support only one pose. Dummy columns leave unsupported/duplicate poses unmatched; + no rectangle mask is evidence of a person. Empty inputs produce no matches. + """ + poses = np.asarray(pose_boxes, dtype=float).reshape(-1, 4) + instances = np.asarray(instance_boxes, dtype=float).reshape(-1, 4) + if not np.isfinite(poses).all() or not np.isfinite(instances).all(): + raise ValueError("pose and instance boxes must be finite") + if not len(poses) or not len(instances): + return {} + low = np.maximum(poses[:, None, :2], instances[None, :, :2]) + high = np.minimum(poses[:, None, 2:], instances[None, :, 2:]) + intersection = np.maximum(high - low, 0).prod(axis=2) + pose_area = np.maximum(poses[:, 2:] - poses[:, :2], 0).prod(axis=1) + instance_area = np.maximum(instances[:, 2:] - instances[:, :2], 0).prod(axis=1) + union = pose_area[:, None] + instance_area[None, :] - intersection + overlap = np.divide(intersection, union, out=np.zeros_like(union), where=union > 0) + supported = overlap > MIN_INSTANCE_IOU + # Losing one supported pair cannot be offset by improvements to all other IoUs. + reward = np.zeros((len(poses), len(instances) + len(poses))) + reward[:, : len(instances)] = np.where( + supported, min(len(poses), len(instances)) + 1 + overlap, 0 + ) + rows, cols = linear_sum_assignment(reward, maximize=True) + return { + int(row): (int(col), float(overlap[row, col])) + for row, col in zip(rows, cols) + if col < len(instances) and supported[row, col] + } diff --git a/worker/stages/track_people.py b/worker/stages/track_people.py index be507a5..689be8a 100644 --- a/worker/stages/track_people.py +++ b/worker/stages/track_people.py @@ -3,12 +3,13 @@ MultiHMR is natively multi-person: `estimator.mhmr_model(...)` already returns every detection in the frame, and lhm_animate.py throws all but one away (it keeps `choices[0]`). This script -keeps them all and links them across samples into identity-stable tracks, then writes, per track, +keeps independently mask-supported poses and links them across samples into tracks, then writes, per track, exactly the `--seed-poses` / `--seed-motion` pair lhm_animate.py already knows how to consume. Detection: MultiHMR for the poses (one forward pass gives detection + SMPL-X + 3D joints, so a second pose model is never needed) plus torchvision Mask R-CNN on the same frame for real -per-person instance masks, associated to the MultiHMR detections by joint-box IoU. Mask R-CNN is +per-person instance masks, matched one-to-one to the MultiHMR detections by joint-box IoU. +Unsupported poses are excluded when instance masks are enabled. Mask R-CNN is also the fallback detector: a Mask R-CNN person with no MultiHMR partner is recorded (so the quality score knows somebody was there) but cannot be animated. @@ -177,6 +178,7 @@ def main(): from accelerate import Accelerator from engine.pose_estimation.pose_estimator import PoseEstimator from scipy.optimize import linear_sum_assignment + from pose_instances import match_pose_instances torch._dynamo.config.disable = True torch.set_num_threads(4) @@ -224,6 +226,8 @@ def main(): overlay_samples = {0, len(indices) // 2, len(indices) - 1} mask_store = {} unmatched_rcnn_total = 0 + unassigned_pose_total = 0 + detection_audit = [] w = h = None for sample, (index, timestamp) in enumerate(zip(indices, times)): @@ -287,24 +291,45 @@ def main(): inst_boxes.append([float(v) for v in r["boxes"][i].cpu().numpy()]) del x, r + projected = [ + (person["j2d"].float().cpu().numpy() - [pl, pt]) / factor - [ow, oh] + for person in detected + ] + boxes = [joints_box(joints, w, h) for joints in projected] + matches = match_pose_instances(boxes, inst_boxes) if rcnn is not None else {} + used_inst = {pair[0] for pair in matches.values()} + rejected = [i for i in range(len(detected)) if rcnn is not None and i not in matches] + unassigned_pose_total += len(rejected) + detection_audit.append( + dict( + sample=sample, + sourceIndex=int(index), + poseBoxes=boxes, + poseScores=[float(person["scores"]) for person in detected], + instanceBoxes=inst_boxes, + matches=[ + dict(pose=i, instance=pair[0], iou=pair[1]) for i, pair in matches.items() + ], + unassignedPoses=rejected, + unmatchedInstances=[i for i in range(len(inst_boxes)) if i not in used_inst], + maskSupportRequired=rcnn is not None, + ) + ) observations = [] - used_inst = set() for det_i, person in enumerate(detected): - joints = (person["j2d"].float().cpu().numpy() - [pl, pt]) / factor - [ow, oh] - box = joints_box(joints, w, h) + if rcnn is not None and det_i not in matches: + continue + joints = projected[det_i] + box = boxes[det_i] centre = np.median(joints[:22], axis=0) / [w, h] j3d = person["j3d"].float().cpu().numpy() - # Pick the Mask R-CNN instance that best covers this skeleton. - best_inst, best_iou = -1, 0.0 - for i, ib in enumerate(inst_boxes): - v = iou(box, ib) - if v > best_iou: - best_inst, best_iou = i, v - mask = inst_masks[best_inst] if (best_inst >= 0 and best_iou > 0.2) else None - if mask is not None: - used_inst.add(best_inst) + if rcnn is not None: + best_inst, best_iou = matches[det_i] + mask = inst_masks[best_inst] mbox = inst_boxes[best_inst] else: + # Explicit --no-maskrcnn retains its unverified rectangle-mask mode. + best_iou = 0.0 mbox = box mask = np.zeros((h, w), bool) x0, y0, x1, y1 = [int(max(0, v)) for v in box] @@ -587,18 +612,33 @@ def main(): trackCount=len(kept), discardedShortTracks=len(tracks) - len(kept), maskrcnnDetectionsWithoutPose=unmatched_rcnn_total, + unassignedPoseDetections=unassigned_pose_total, + maskSupportRequired=not a.no_maskrcnn, + instanceAssociation=( + "disabled; explicit unverified rectangle-mask mode" + if a.no_maskrcnn + else "one-to-one maximum supported pairs, then maximum total IoU (>0.2)" + ), tracks=report, method=( - "MultiHMR multi-person detections (one forward pass per sample, all persons kept) " - "associated by Hungarian assignment on constant-velocity centre prediction, box-height " - "ratio, camera-depth ratio and a Mask R-CNN-masked upper/lower-body RGB histogram. " - "Mask R-CNN supplies the per-person instance masks; a Mask R-CNN person with no " - "MultiHMR partner is counted but not tracked." + ( + "MultiHMR poses without independent mask support " + if a.no_maskrcnn + else "MultiHMR poses supported one-to-one by independent person instances " + ) + + "associated by Hungarian assignment on constant-velocity centre prediction, box-height " + "ratio, camera-depth ratio and an upper/lower-body RGB histogram. " + + ( + "Explicit --no-maskrcnn uses unverified rectangle masks." + if a.no_maskrcnn + else "Mask R-CNN supplies person instance masks; instances without a pose are counted but not tracked." + ) ), seconds=time.time() - start, gpu=torch.cuda.get_device_name(), torch=torch.__version__, ) + (out / "detection-audit.json").write_text(json.dumps(detection_audit, indent=1)) (out / "tracks.json").write_text(json.dumps(doc, indent=1)) print( json.dumps(