Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/known-limits.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ Do not retry one without a new idea, and add a line here when you measure a new
202 unmatched instance observations still include real background pedestrians and people
pictured on signage. Do not animate every unmatched detection or call the single-track
result accepted. Its body projection also remains misaligned with the observed silhouette.
Experimental `--pose-crops` adds calibrated inference for unmatched masks, but has no GPU
quality result yet and remains off by default; see [person motion](person-motion.md#experimental-tracking-crop-recovery).
Evidence: `.context/evidence/city-recovery-20261120/`.

- The forest scale gate fails on distant background, not on the route. Replaying the run's own
Expand Down
19 changes: 19 additions & 0 deletions docs/person-motion.md
Original file line number Diff line number Diff line change
Expand Up @@ -102,3 +102,22 @@ Their original author Claude and co-author Claude Fable 5.1 remain credited in t
Codex adapted the integration for lossless defaults, format identity, integrity verification and
safe fallback. Live rendered comparisons and actual network timing remain separate acceptance
checks; unit tests do not establish visual quality or headset performance.

## Experimental tracking crop recovery

Tracked runs can opt into `--pose-crops` (or the `pose_crops` tracking stage parameter).
It is off by default and requires source camera calibration for every sampled frame and
independent person instance masks. Each instance unmatched by the ordinary full-frame pose
pass gets one additional crop inference. The crop translates the camera principal point,
then applies the estimator's resize and padding; predicted joints are mapped back to source
pixels. Existing full-frame matches are preserved. A crop may claim only its target mask
under the existing IoU threshold.

This adds GPU calls and can increase runtime substantially. It does not automatically retry,
change the stage retry budget, or distinguish pedestrians from people pictured on signs or
statues. `detection-audit.json` records crop bounds, detections, recoveries and pose origins;
track summaries record call counts and motion observations preserve their origin.

The geometry and submission plumbing have CPU tests. Actual model recovery and visual
quality have not yet been established; do not enable this by default or treat recovered
detections as an accepted all-people reconstruction.
6 changes: 6 additions & 0 deletions orchestrator/steps.py
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,12 @@ def tunables(**knobs: dict[str, Any]) -> dict[str, Any]:
),
)
TRACK_PARAMETERS = tunables(
pose_crops=knob(
"boolean",
False,
"experimental additional calibrated pose pass for unmatched person instances; "
"adds inference cost and does not distinguish depicted from real people",
),
fps=knob(
"number",
12,
Expand Down
8 changes: 8 additions & 0 deletions scripts/run_clip.py
Original file line number Diff line number Diff line change
Expand Up @@ -1854,6 +1854,7 @@ def tracks(self):
str(self.a.fps),
"--det-thresh",
str(self.a.det_thresh),
*(["--pose-crops"] if getattr(self.a, "pose_crops", False) else []),
"--out",
str(self.ctx / "tracks"),
],
Expand Down Expand Up @@ -4358,6 +4359,11 @@ def main():
default=0.15,
help="MultiHMR detection threshold for the tracking stage",
)
ap.add_argument(
"--pose-crops",
action="store_true",
help="experimental crop pose recovery for unmatched tracked instances",
)
ap.add_argument(
"--shot",
type=int,
Expand Down Expand Up @@ -4440,6 +4446,8 @@ def main():
animation_options(execution_timeout=a.execution_timeout)
except ValueError as error:
ap.error(str(error))
if a.pose_crops and (not (a.people or a.all_people) or a.orchestrated):
ap.error("--pose-crops requires direct tracked runs; use wander step for stage parameters")
if a.lower_body_provenance is not None:
if not a.lower_body_provenance.strip():
ap.error("--lower-body-provenance must be a nonempty measured statement")
Expand Down
2 changes: 2 additions & 0 deletions scripts/stage_attempts.py
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,7 @@ def _read_authorization(path):
"worker/stages/worker_resources.py",
"worker/stages/track_people.py",
"worker/stages/pose_instances.py",
"worker/stages/pose_crops.py",
),
"worker/modal_multiperson.py::animate": (
"worker/stages/worker_resources.py",
Expand All @@ -160,6 +161,7 @@ def _read_authorization(path):
}
OUTPUT_FLAGS = {"--out", "--out-dir", "--frames-out", "--frame0", "--masks", "--report"}
BOOL_FLAGS = {
"--pose-crops",
"--anchor-lock",
"--allow-person-gaps",
"--allow-missing-person-reference",
Expand Down
171 changes: 171 additions & 0 deletions scripts/tests/test_pose_crops.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,171 @@
"""Projection and inverse-transform checks, independent of any pose model."""

import sys
from pathlib import Path

import numpy as np
import pytest

sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from worker.stages.pose_crops import crop_bounds, crop_intrinsics, source_joints


@pytest.mark.parametrize(
"box,expected",
[
([400.25, 100.75, 700.5, 500.25], (340, 20, 761, 581)),
([-20, 40, 180, 300], (0, 0, 220, 352)),
([1100, 600, 1300, 800], (1060, 560, 1280, 720)),
],
)
def test_bounds_round_outward_and_clamp(box, expected):
original = np.array(box)
assert crop_bounds(original, 1280, 720) == expected
np.testing.assert_array_equal(original, box)
assert crop_bounds([2.2, 3.3, 4.4, 5.5], 1280, 720, padding=0) == (2, 3, 5, 6)


@pytest.mark.parametrize("box", [[400, 100, 700, 500], [-20, 40, 180, 300]])
def test_calibrated_crop_preserves_3d_rays_and_source_projection(box):
bounds = crop_bounds(box, 1280, 720)
K = np.array([[920.0, 31.0, 603.5], [0.0, 870.0, 341.25], [0.0, 0.0, 1.0]])
original = K.copy()
points = np.array([[-0.7, -0.3, 2.0], [0.4, 0.5, 3.0], [1.2, -0.2, 4.0]])
source_h = (K @ points.T).T
source_pixels = source_h[:, :2] / source_h[:, 2:]
crop_K = crop_intrinsics(K, bounds)
crop_h = (crop_K @ points.T).T
crop_pixels = crop_h[:, :2] / crop_h[:, 2:]
np.testing.assert_allclose(crop_pixels + bounds[:2], source_pixels, atol=1e-12)
rays = np.linalg.solve(crop_K, np.column_stack((crop_pixels, np.ones(3))).T).T
np.testing.assert_allclose(rays, points / points[:, 2:], atol=1e-12)
assert crop_K[0, 1] == K[0, 1]
np.testing.assert_array_equal(K, original)

# Asymmetric center padding and preprocessing padding, with a non-unit resize.
pad_left, pad_top, factor, offset_w, offset_h = 17, 29, 0.73, 43, 11
preprocessing = np.array(
[
[factor, 0, factor * offset_w + pad_left],
[0, factor, factor * offset_h + pad_top],
[0, 0, 1],
]
)
model_h = (preprocessing @ crop_K @ points.T).T
model_pixels = model_h[:, :2] / model_h[:, 2:]
before = model_pixels.copy()
restored = source_joints(model_pixels, pad_left, pad_top, factor, offset_w, offset_h, bounds)
np.testing.assert_allclose(restored, source_pixels, atol=1e-12)
np.testing.assert_array_equal(model_pixels, before)


def test_inverse_does_not_clip_predictions_to_crop():
expected = np.array([[-5.0, 80.0], [900.0, 1200.0]])
bounds = (100, 200, 400, 600)
predicted = (expected - bounds[:2] + [9, 21]) * 1.7 + [3, 15]
np.testing.assert_allclose(source_joints(predicted, 3, 15, 1.7, 9, 21, bounds), expected)


@pytest.mark.parametrize(
"box",
[
[],
[1, 2, 3],
[[1, 2, 3, 4]],
[0, 0, 0, 1],
[3, 0, 2, 1],
[0, 0, np.nan, 1],
[0, 0, np.inf, 1],
[1300, 0, 1400, 40],
[-40, 0, 0, 40],
[0, 720, 10, 800],
],
)
def test_invalid_boxes(box):
with pytest.raises(ValueError):
crop_bounds(box, 1280, 720)


@pytest.mark.parametrize(
"name,value",
[
("width", 0),
("height", -1),
("width", 1.5),
("width", True),
("height", np.inf),
("padding", -0.1),
("padding", np.nan),
("padding", [0.2]),
("padding", "0.2"),
],
)
def test_invalid_crop_parameters(name, value):
kwargs = dict(width=1280, height=720, padding=0.2)
kwargs[name] = value
with pytest.raises(ValueError):
crop_bounds([10, 10, 100, 100], **kwargs)


@pytest.mark.parametrize(
"bounds", [[], [0, 0, 1], [0, 0, 0, 1], [-1, 0, 2, 3], [0.5, 0, 2, 3], [0, 0, np.inf, 3]]
)
def test_invalid_bounds_rejected_by_both_transforms(bounds):
with pytest.raises(ValueError):
crop_intrinsics(np.eye(3), bounds)
with pytest.raises(ValueError):
source_joints([[1, 2]], 0, 0, 1, 0, 0, bounds)


@pytest.mark.parametrize(
"matrix",
[
np.eye(2),
np.zeros((3, 3)),
[[1, 0, 0], [0, -1, 0], [0, 0, 1]],
[[1, 0, 0], [0, 1, 0], [0, 0, 2]],
[[1, np.nan, 0], [0, 1, 0], [0, 0, 1]],
[[1, 0, 0], [1, 1, 0], [0, 0, 1]],
],
)
def test_invalid_intrinsics(matrix):
with pytest.raises(ValueError):
crop_intrinsics(matrix, (10, 20, 30, 40))


@pytest.mark.parametrize(
"name,value",
[
("joints", []),
("joints", [1, 2]),
("joints", [[1, 2, 3]]),
("joints", [[np.nan, 2]]),
("factor", 0),
("factor", -1),
("factor", np.inf),
("pad_left", -1),
("pad_top", np.nan),
("offset_w", -1),
("offset_h", [1]),
],
)
def test_invalid_inverse_inputs(name, value):
kwargs = dict(
joints=[[1, 2]],
pad_left=0,
pad_top=0,
factor=1,
offset_w=0,
offset_h=0,
bounds=(10, 20, 30, 40),
)
kwargs[name] = value
with pytest.raises(ValueError):
source_joints(**kwargs)


def test_overflow_fails_instead_of_returning_nonfinite_geometry():
with pytest.raises(ValueError):
crop_bounds([-1e308, 0, 1e308, 20], 1280, 720)
with pytest.raises(ValueError):
source_joints([[1e308, 2]], 0, 0, 1e-308, 0, 0, (0, 0, 10, 10))
15 changes: 14 additions & 1 deletion scripts/tests/test_pose_instances.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import pytest

sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from worker.stages.pose_instances import match_pose_instances
from worker.stages.pose_instances import match_pose_instances, match_crop_instance


def test_lamp_pose_is_not_a_person_instance():
Expand Down Expand Up @@ -52,3 +52,16 @@ def test_degenerate_boxes_cannot_supply_support():
def test_nonfinite_boxes_fail_instead_of_becoming_a_match():
with pytest.raises(ValueError, match="finite"):
match_pose_instances([[0, 0, np.nan, 20]], [[0, 0, 10, 20]])


def test_crop_cannot_relabel_neighbor_as_missing_person():
instances = [[0, 0, 20, 80], [10, 0, 30, 80]]
assert match_pose_instances([instances[0]], [instances[1]]) # old target-only false positive
assert match_crop_instance([instances[0]], instances, 1) == {}
assert match_crop_instance([instances[0], instances[1]], instances, 1) == {1: (1, 1.0)}


def test_crop_refuses_ambiguous_support_and_duplicate_recovery():
assert match_crop_instance([[0, 0, 20, 80]], [[0, 0, 20, 80]] * 2, 1) == {}
assert len(match_crop_instance([[0, 0, 20, 80]] * 2, [[0, 0, 20, 80]], 0)) == 1
assert match_crop_instance([], [[0, 0, 20, 80]], 0) == {}
12 changes: 12 additions & 0 deletions scripts/tests/test_stage_attempts.py
Original file line number Diff line number Diff line change
Expand Up @@ -690,6 +690,18 @@ def test_tracking_instance_matching_changes_are_fingerprinted(self):
_, after, _ = command_identity(command, self.root)
self.assertNotEqual(before[helper], after[helper])

def test_crop_geometry_and_option_change_tracking_attempt_identity(self):
command = ["modal", "run", "worker/modal_multiperson.py::main"]
params, before, _ = command_identity(command, self.root)
changed, _, _ = command_identity(command + ["--pose-crops"], self.root)
self.assertNotEqual(params, changed)
helper = "worker/stages/pose_crops.py"
self.assertIn(helper, before)
path = self.root / helper
path.write_bytes(path.read_bytes() + b"\n# changed crop calibration\n")
_, after, _ = command_identity(command, self.root)
self.assertNotEqual(before[helper], after[helper])

def test_single_person_lhm_deadline_guard_changes_are_fingerprinted(self):
helper = "worker/stages/lhm_execution.py"
command = ["modal", "run", "worker/modal_lhm.py"]
Expand Down
38 changes: 36 additions & 2 deletions scripts/tests/test_tracking_submission.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ def get(*, timeout):
launch.call.get.side_effect = get
launch.run(fps=15.0, det_thresh=0.3, gate=0.4, max_gap=4, min_track_samples=6, overlay_every=2)
launch.function.spawn.assert_called_once_with(
b"video bytes", b"camera bytes", 15.0, 0.3, 0.4, 4, 6, 2
b"video bytes", b"camera bytes", 15.0, 0.3, 0.4, 4, 6, 2, False
)
launch.call.cancel.assert_not_called()
assert launch.receipt() == initial | {"status": "result_received", "functionCallId": "fc-track"}
Expand All @@ -116,7 +116,9 @@ def test_tracking_default_wait_and_reported_inference_failure(launch, monkeypatc
launch.call.get.return_value = tracking_output("inference failed")
with pytest.raises(RuntimeError, match="see the downloaded track.log"):
launch.run(cameras="")
launch.function.spawn.assert_called_once_with(b"video bytes", None, 12, 0.15, 0.22, 8, 8, 0)
launch.function.spawn.assert_called_once_with(
b"video bytes", None, 12, 0.15, 0.22, 8, 8, 0, False
)
launch.call.get.assert_called_once_with(timeout=7200)
launch.call.cancel.assert_not_called()
assert launch.receipt()["status"] == "result_received"
Expand Down Expand Up @@ -195,3 +197,35 @@ def write(path, receipt):
assert receipt["status"] == "wait_interrupted"
assert receipt["functionCallId"] == "fc-track"
assert receipt["cancellation"] == "requested"


def test_crop_option_forwarded_without_extra_provider_submissions(launch):
launch.call.get.return_value = tracking_output()
launch.run(pose_crops=True)
launch.function.spawn.assert_called_once()
assert launch.function.spawn.call_args.args[-1] is True


def test_crops_without_calibration_stop_before_claim_or_submission(launch):
with pytest.raises(ValueError, match="requires source cameras"):
launch.run(pose_crops=True, cameras="")
launch.function.spawn.assert_not_called()
assert not launch.destination.exists()


@pytest.mark.parametrize("enabled", [False, True])
def test_runner_forwards_crop_option_only_when_requested(tmp_path, enabled):
sys.path.insert(0, str(ROOT / "scripts"))
import run_clip

pipeline = object.__new__(run_clip.Pipeline)
pipeline.ctx = tmp_path
pipeline.clip = tmp_path / "source.mp4"
pipeline.a = SimpleNamespace(fps=12, det_thresh=0.15, pose_crops=enabled)
pipeline.paid_run = Mock()
pipeline.track_doc = lambda: {"trackCount": 0, "tracks": []}
pipeline.identity_guard = Mock()
pipeline.tracks()
pipeline.paid_run.assert_called_once()
command = pipeline.paid_run.call_args.args[1]
assert ("--pose-crops" in command) is enabled
Loading
Loading