Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions docs/DEVELOPING.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,18 @@ Code identifiers, env vars and the package still use the project's old name, Wan
The source video owns the timeline; the XR runtime owns head motion. Scene manifests and media
stay outside Git. Viewer installation and builds do not launch inference or download model weights.

## Person tracking

Multi-person tracking requires independent Mask R-CNN support for each pose by default.
Pose boxes and instance boxes are matched one-to-one, maximizing supported pairs first and
then total overlap, with the existing strict IoU > 0.2 threshold. Unsupported poses cannot
create rectangular person masks or compete for a track. The explicit `--no-maskrcnn` worker
mode remains unverified and uses rectangles. This is not a detector recall guarantee:
unmatched person instances still lack animation poses and must be reviewed before delivery.
`detection-audit.json` retains per-sample pose/instance boxes, assignments and unmatched indices;
`tracks.json` reports unsupported pose and unmatched instance counts separately. Counts are
observations across frames, not unique people. Original source images and quality gates are unchanged.

## Viewer notes

Open **http://127.0.0.1:5399/demo.html**. The root URL redirects here. The local Bun server loads
Expand Down
9 changes: 8 additions & 1 deletion docs/known-limits.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,14 @@ Do not retry one without a new idea, and add a line here when you measure a new
pedestrians were missed. The runner formerly marked tracking successful by skipping the failed
subset, even with `--all-people`. All-person runs now stop on any failed identity audit rather
than discarding those tracks. This repairs acceptance, not the tracker: no avatar or world was
generated from this result. Evidence: `.context/evidence/city-recovery-20261120/`.
generated from this result. A subsequent one-to-one instance-support correction retained the
main walker as one ID across all 94 samples, with no gaps or duplicate instance assignments;
beginning/middle/end overlays no longer tracked lamps. All 240 retained elevator and 75
hall-walk associations were unchanged in offline replay. This does not fix all-person recall:
202 unmatched instance observations still include real background pedestrians and people
pictured on signage. Do not animate every unmatched detection or call the single-track
result accepted. Its body projection also remains misaligned with the observed silhouette.
Evidence: `.context/evidence/city-recovery-20261120/`.

- The forest scale gate fails on distant background, not on the route. Replaying the run's own
scale diagnostic (unchanged inputs, scale0 0.7414, whole-frame p10 0.842–0.848) split by each
Expand Down
1 change: 1 addition & 0 deletions scripts/stage_attempts.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,7 @@ def _read_authorization(path):
"worker/modal_multiperson.py::main": (
"worker/stages/worker_resources.py",
"worker/stages/track_people.py",
"worker/stages/pose_instances.py",
),
"worker/modal_multiperson.py::animate": (
"worker/stages/worker_resources.py",
Expand Down
54 changes: 54 additions & 0 deletions scripts/tests/test_pose_instances.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
"""Independent mask support must not create duplicate or unobserved people."""

import sys
from pathlib import Path

import numpy as np
import pytest

sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from worker.stages.pose_instances import match_pose_instances


def test_lamp_pose_is_not_a_person_instance():
assert match_pose_instances([[0, 0, 10, 50], [60, 0, 90, 80]], [[60, 0, 90, 80]]) == {
1: (0, 1.0)
}


def test_duplicate_poses_cannot_claim_same_person():
pairs = match_pose_instances([[0, 0, 20, 80], [0, 0, 10, 80]], [[0, 0, 20, 80]])
assert pairs == {0: (0, 1.0)}


def test_global_matching_preserves_second_person_over_greedy_choice():
# First pose supports both masks; second supports only the first mask.
poses = [[0, 0, 18, 10], [0, 0, 6, 10]]
masks = [[0, 0, 10, 10], [10, 0, 20, 10]]
result = match_pose_instances(poses, masks)
assert {i: pair[0] for i, pair in result.items()} == {0: 1, 1: 0}


def test_separated_people_remain_separate_and_order_independent():
boxes = [[0, 0, 10, 30], [40, 0, 50, 30], [80, 0, 90, 30]]
assert match_pose_instances(boxes, boxes) == {i: (i, 1.0) for i in range(3)}
assert match_pose_instances(boxes[::-1], boxes) == {i: (2 - i, 1.0) for i in range(3)}


@pytest.mark.parametrize("poses,masks", [([], []), ([], [[0, 0, 1, 1]]), ([[0, 0, 1, 1]], [])])
def test_absent_support_is_not_fabricated(poses, masks):
assert match_pose_instances(poses, masks) == {}


def test_existing_strict_overlap_threshold_is_preserved():
assert match_pose_instances([[0, 0, 2, 10]], [[0, 0, 10, 10]]) == {}
assert match_pose_instances([[0, 0, 2.01, 10]], [[0, 0, 10, 10]])


def test_degenerate_boxes_cannot_supply_support():
assert match_pose_instances([[0, 0, 0, 20], [10, 10, 0, 0]], [[0, 0, 10, 20]]) == {}


def test_nonfinite_boxes_fail_instead_of_becoming_a_match():
with pytest.raises(ValueError, match="finite"):
match_pose_instances([[0, 0, np.nan, 20]], [[0, 0, 10, 20]])
10 changes: 10 additions & 0 deletions scripts/tests/test_stage_attempts.py
Original file line number Diff line number Diff line change
Expand Up @@ -680,6 +680,16 @@ def test_motion_submission_and_receipt_changes_are_fingerprinted(self):
_, after, _ = command_identity(command, self.root)
self.assertNotEqual(before[helper], after[helper])

def test_tracking_instance_matching_changes_are_fingerprinted(self):
helper = "worker/stages/pose_instances.py"
command = ["modal", "run", "worker/modal_multiperson.py::main"]
_, before, _ = command_identity(command, self.root)
self.assertIn(helper, before)
path = self.root / helper
path.write_bytes(path.read_bytes() + b"\n# changed independent mask support\n")
_, after, _ = command_identity(command, self.root)
self.assertNotEqual(before[helper], after[helper])

def test_single_person_lhm_deadline_guard_changes_are_fingerprinted(self):
helper = "worker/stages/lhm_execution.py"
command = ["modal", "run", "worker/modal_lhm.py"]
Expand Down
41 changes: 41 additions & 0 deletions worker/stages/pose_instances.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
"""One-to-one support for pose detections from independent person instances."""

import numpy as np
from scipy.optimize import linear_sum_assignment

# Preserve the existing strict support threshold used by track_people.
MIN_INSTANCE_IOU = 0.2


def match_pose_instances(pose_boxes, instance_boxes):
"""Return {pose index: (instance index, IoU)} for supported detections.

Maximize the number of supported pairs, then their total IoU. Each instance can
support only one pose. Dummy columns leave unsupported/duplicate poses unmatched;
no rectangle mask is evidence of a person. Empty inputs produce no matches.
"""
poses = np.asarray(pose_boxes, dtype=float).reshape(-1, 4)
instances = np.asarray(instance_boxes, dtype=float).reshape(-1, 4)
if not np.isfinite(poses).all() or not np.isfinite(instances).all():
raise ValueError("pose and instance boxes must be finite")
if not len(poses) or not len(instances):
return {}
low = np.maximum(poses[:, None, :2], instances[None, :, :2])
high = np.minimum(poses[:, None, 2:], instances[None, :, 2:])
intersection = np.maximum(high - low, 0).prod(axis=2)
pose_area = np.maximum(poses[:, 2:] - poses[:, :2], 0).prod(axis=1)
instance_area = np.maximum(instances[:, 2:] - instances[:, :2], 0).prod(axis=1)
union = pose_area[:, None] + instance_area[None, :] - intersection
overlap = np.divide(intersection, union, out=np.zeros_like(union), where=union > 0)
supported = overlap > MIN_INSTANCE_IOU
# Losing one supported pair cannot be offset by improvements to all other IoUs.
reward = np.zeros((len(poses), len(instances) + len(poses)))
reward[:, : len(instances)] = np.where(
supported, min(len(poses), len(instances)) + 1 + overlap, 0
)
rows, cols = linear_sum_assignment(reward, maximize=True)
return {
int(row): (int(col), float(overlap[row, col]))
for row, col in zip(rows, cols)
if col < len(instances) and supported[row, col]
}
78 changes: 59 additions & 19 deletions worker/stages/track_people.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,13 @@

MultiHMR is natively multi-person: `estimator.mhmr_model(...)` already returns every detection
in the frame, and lhm_animate.py throws all but one away (it keeps `choices[0]`). This script
keeps them all and links them across samples into identity-stable tracks, then writes, per track,
keeps independently mask-supported poses and links them across samples into tracks, then writes, per track,
exactly the `--seed-poses` / `--seed-motion` pair lhm_animate.py already knows how to consume.

Detection: MultiHMR for the poses (one forward pass gives detection + SMPL-X + 3D joints, so a
second pose model is never needed) plus torchvision Mask R-CNN on the same frame for real
per-person instance masks, associated to the MultiHMR detections by joint-box IoU. Mask R-CNN is
per-person instance masks, matched one-to-one to the MultiHMR detections by joint-box IoU.
Unsupported poses are excluded when instance masks are enabled. Mask R-CNN is
also the fallback detector: a Mask R-CNN person with no MultiHMR partner is recorded (so the
quality score knows somebody was there) but cannot be animated.

Expand Down Expand Up @@ -177,6 +178,7 @@ def main():
from accelerate import Accelerator
from engine.pose_estimation.pose_estimator import PoseEstimator
from scipy.optimize import linear_sum_assignment
from pose_instances import match_pose_instances

torch._dynamo.config.disable = True
torch.set_num_threads(4)
Expand Down Expand Up @@ -224,6 +226,8 @@ def main():
overlay_samples = {0, len(indices) // 2, len(indices) - 1}
mask_store = {}
unmatched_rcnn_total = 0
unassigned_pose_total = 0
detection_audit = []
w = h = None

for sample, (index, timestamp) in enumerate(zip(indices, times)):
Expand Down Expand Up @@ -287,24 +291,45 @@ def main():
inst_boxes.append([float(v) for v in r["boxes"][i].cpu().numpy()])
del x, r

projected = [
(person["j2d"].float().cpu().numpy() - [pl, pt]) / factor - [ow, oh]
for person in detected
]
boxes = [joints_box(joints, w, h) for joints in projected]
matches = match_pose_instances(boxes, inst_boxes) if rcnn is not None else {}
used_inst = {pair[0] for pair in matches.values()}
rejected = [i for i in range(len(detected)) if rcnn is not None and i not in matches]
unassigned_pose_total += len(rejected)
detection_audit.append(
dict(
sample=sample,
sourceIndex=int(index),
poseBoxes=boxes,
poseScores=[float(person["scores"]) for person in detected],
instanceBoxes=inst_boxes,
matches=[
dict(pose=i, instance=pair[0], iou=pair[1]) for i, pair in matches.items()
],
unassignedPoses=rejected,
unmatchedInstances=[i for i in range(len(inst_boxes)) if i not in used_inst],
maskSupportRequired=rcnn is not None,
)
)
observations = []
used_inst = set()
for det_i, person in enumerate(detected):
joints = (person["j2d"].float().cpu().numpy() - [pl, pt]) / factor - [ow, oh]
box = joints_box(joints, w, h)
if rcnn is not None and det_i not in matches:
continue
joints = projected[det_i]
box = boxes[det_i]
centre = np.median(joints[:22], axis=0) / [w, h]
j3d = person["j3d"].float().cpu().numpy()
# Pick the Mask R-CNN instance that best covers this skeleton.
best_inst, best_iou = -1, 0.0
for i, ib in enumerate(inst_boxes):
v = iou(box, ib)
if v > best_iou:
best_inst, best_iou = i, v
mask = inst_masks[best_inst] if (best_inst >= 0 and best_iou > 0.2) else None
if mask is not None:
used_inst.add(best_inst)
if rcnn is not None:
best_inst, best_iou = matches[det_i]
mask = inst_masks[best_inst]
mbox = inst_boxes[best_inst]
else:
# Explicit --no-maskrcnn retains its unverified rectangle-mask mode.
best_iou = 0.0
mbox = box
mask = np.zeros((h, w), bool)
x0, y0, x1, y1 = [int(max(0, v)) for v in box]
Expand Down Expand Up @@ -587,18 +612,33 @@ def main():
trackCount=len(kept),
discardedShortTracks=len(tracks) - len(kept),
maskrcnnDetectionsWithoutPose=unmatched_rcnn_total,
unassignedPoseDetections=unassigned_pose_total,
maskSupportRequired=not a.no_maskrcnn,
instanceAssociation=(
"disabled; explicit unverified rectangle-mask mode"
if a.no_maskrcnn
else "one-to-one maximum supported pairs, then maximum total IoU (>0.2)"
),
tracks=report,
method=(
"MultiHMR multi-person detections (one forward pass per sample, all persons kept) "
"associated by Hungarian assignment on constant-velocity centre prediction, box-height "
"ratio, camera-depth ratio and a Mask R-CNN-masked upper/lower-body RGB histogram. "
"Mask R-CNN supplies the per-person instance masks; a Mask R-CNN person with no "
"MultiHMR partner is counted but not tracked."
(
"MultiHMR poses without independent mask support "
if a.no_maskrcnn
else "MultiHMR poses supported one-to-one by independent person instances "
)
+ "associated by Hungarian assignment on constant-velocity centre prediction, box-height "
"ratio, camera-depth ratio and an upper/lower-body RGB histogram. "
+ (
"Explicit --no-maskrcnn uses unverified rectangle masks."
if a.no_maskrcnn
else "Mask R-CNN supplies person instance masks; instances without a pose are counted but not tracked."
)
),
seconds=time.time() - start,
gpu=torch.cuda.get_device_name(),
torch=torch.__version__,
)
(out / "detection-audit.json").write_text(json.dumps(detection_audit, indent=1))
(out / "tracks.json").write_text(json.dumps(doc, indent=1))
print(
json.dumps(
Expand Down
Loading