From c3ee8f93fe036bd9544e719ae9adb6f5f1ca1bb6 Mon Sep 17 00:00:00 2001 From: Dipak Date: Tue, 11 Aug 2026 20:16:40 +0530 Subject: [PATCH] feat(viewer): colour cameras by in_view coverage and draw seeing rays MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 3D viewer drew every camera the same crimson, so a scene's coverage — which camera holds the object on a given frame, and where two cameras overlap — was invisible in the rendered figure. Colour each camera's centre, label and frustum by whether it sees the object on the selected frame (green = sees, grey = blind), draw a light dotted ray from every seeing camera to the point it sees so overlap reads directly, and add a legend plus an "n/m cameras see the object" count to the title. Coverage is read straight off the manifest's per-camera `in_view` flags via the new `_coverage_at_frame` helper; the viewer does no reprojection of its own. A new `--frame` flag chooses the frame and rejects one that is not in the manifest, naming the available range. On `handoff_ltr` this reproduces the coverage pattern its test documents: LEFT green at frame 0 and grey by 39, RIGHT the reverse, WIDE green throughout, and a LEFT/MID_RIGHT overlap band in between. --- scripts/view_scene_3d.py | 116 ++++++++++++++++++++++++++++++++---- tests/test_view_scene_3d.py | 114 +++++++++++++++++++++++++++++++++++ 2 files changed, 219 insertions(+), 11 deletions(-) create mode 100644 tests/test_view_scene_3d.py diff --git a/scripts/view_scene_3d.py b/scripts/view_scene_3d.py index 476aad3..6711c1e 100644 --- a/scripts/view_scene_3d.py +++ b/scripts/view_scene_3d.py @@ -12,9 +12,15 @@ frustum edges are the back-projected image corners ``R^T @ (K^-1 @ [u, v, 1])``. World up is +z; the ground plane is drawn at z = 0. +Coverage is legible at a glance: for the frame selected with ``--frame`` each +camera is drawn green when it sees the object on that frame and grey when it is +blind, with a light ray from every seeing camera to the point it sees, so +overlap (two or more cameras on one object) shows up directly. The ``in_view`` +flags are read straight off the manifest -- the viewer never reprojects. + Works on any manifest with the standard schema (``cameras[*].{K,R,t,width, -height}`` and ``entities[*].frames[*].points[*].xyz_gt``); defaults to the -bundled MTMC golden fixture. +height}`` and ``entities[*].frames[*].points[*].{xyz_gt,per_cam[*].in_view}``); +defaults to the bundled MTMC golden fixture. ``matplotlib`` is imported lazily inside :func:`main` (it is not a package dependency), mirroring the lazy-import pattern in ``scripts/record_multiview.py``. @@ -22,6 +28,7 @@ uv run --with matplotlib python scripts/view_scene_3d.py uv run --with matplotlib python scripts/view_scene_3d.py --manifest PATH --out PATH + uv run --with matplotlib python scripts/view_scene_3d.py --frame 40 """ from __future__ import annotations @@ -43,6 +50,10 @@ # How far in front of each camera to draw the frustum apex (world units). _FRUSTUM_DEPTH = 1.2 +# Frustum colours for the selected frame: sees the object vs. blind to it. +_SEEING_COLOUR = "seagreen" +_BLIND_COLOUR = "slategrey" + def _camera_centre( rotation: NDArray[np.float64], translation: NDArray[np.float64] @@ -101,8 +112,47 @@ def _trajectories(manifest: dict[str, Any]) -> list[tuple[str, NDArray[np.float6 return out -def render(manifest: dict[str, Any], out_path: Path) -> Path: - """Render the 3D scene view to ``out_path`` (PNG) and return the path.""" +def _frame_numbers(manifest: dict[str, Any]) -> list[int]: + """Every frame number present on any entity, ascending.""" + numbers = { + int(frame["frame"]) + for entity in manifest.get("entities", []) + for frame in entity.get("frames", []) + if "frame" in frame + } + return sorted(numbers) + + +def _coverage_at_frame( + manifest: dict[str, Any], frame_number: int +) -> dict[int, list[NDArray[np.float64]]]: + """Map each camera id to the ground-truth points it sees on ``frame_number``. + + Read straight from the manifest's per-camera ``in_view`` flags -- the sim's + own projection/visibility answer -- so the viewer never reprojects. Cameras + that see nothing are absent from the mapping. + """ + seen: dict[int, list[NDArray[np.float64]]] = {} + for entity in manifest.get("entities", []): + for frame in entity.get("frames", []): + if int(frame.get("frame", -1)) != frame_number: + continue + for point in frame.get("points", {}).values(): + xyz = point.get("xyz_gt") + if xyz is None: + continue + target = np.asarray(xyz, dtype=np.float64) + for observation in point.get("per_cam", []): + if observation.get("in_view"): + seen.setdefault(int(observation["cam"]), []).append(target) + return seen + + +def render(manifest: dict[str, Any], out_path: Path, frame_number: int = 0) -> Path: + """Render the 3D scene view to ``out_path`` (PNG) and return the path. + + Cameras are coloured by whether they see the object on ``frame_number``. + """ import matplotlib matplotlib.use("Agg") # headless-safe: no display needed @@ -112,8 +162,10 @@ def render(manifest: dict[str, Any], out_path: Path) -> Path: ax = fig.add_subplot(111, projection="3d") all_points: list[NDArray[np.float64]] = [] + seen_by_camera = _coverage_at_frame(manifest, frame_number) - # Cameras: centre marker + a wireframe frustum to the four image corners. + # Cameras: centre marker + a wireframe frustum to the four image corners, + # coloured green when the camera sees the object on this frame. for cam in manifest.get("cameras", []): intrinsics = np.asarray(cam["K"], dtype=np.float64) rotation = np.asarray(cam["R"], dtype=np.float64) @@ -127,13 +179,27 @@ def render(manifest: dict[str, Any], out_path: Path) -> Path: float(cam["height"]), _FRUSTUM_DEPTH, ) - ax.scatter(*centre, color="crimson", s=40, depthshade=False) - ax.text(*centre, f" cam {cam['id']}", color="crimson", fontsize=8) + targets = seen_by_camera.get(int(cam["id"]), []) + colour = _SEEING_COLOUR if targets else _BLIND_COLOUR + ax.scatter(*centre, color=colour, s=40, depthshade=False) + ax.text(*centre, f" cam {cam['id']}", color=colour, fontsize=8) for corner in corners: edge = np.vstack([centre, corner]) - ax.plot(edge[:, 0], edge[:, 1], edge[:, 2], color="crimson", linewidth=0.8) + ax.plot(edge[:, 0], edge[:, 1], edge[:, 2], color=colour, linewidth=0.8) loop = np.vstack([corners, corners[0]]) - ax.plot(loop[:, 0], loop[:, 1], loop[:, 2], color="crimson", linewidth=0.8) + ax.plot(loop[:, 0], loop[:, 1], loop[:, 2], color=colour, linewidth=0.8) + # A light ray to each point this camera sees, so overlap is obvious. + for target in targets: + ray = np.vstack([centre, target]) + ax.plot( + ray[:, 0], + ray[:, 1], + ray[:, 2], + color=_SEEING_COLOUR, + linewidth=0.7, + linestyle=":", + alpha=0.55, + ) all_points.append(centre[None, :]) all_points.append(corners) @@ -164,10 +230,19 @@ def render(manifest: dict[str, Any], out_path: Path) -> Path: ) ax.plot_surface(gx, gy, np.zeros_like(gx), alpha=0.12, color="steelblue") + # Legend proxies for the coverage colours (the frustums are plain lines). + seeing_count = len(seen_by_camera) + total_cameras = len(manifest.get("cameras", [])) + ax.plot([], [], color=_SEEING_COLOUR, linewidth=0.8, label="camera: sees object") + ax.plot([], [], color=_BLIND_COLOUR, linewidth=0.8, label="camera: blind") + ax.set_xlabel("x") ax.set_ylabel("y") ax.set_zlabel("z (up)") - ax.set_title("Scene: camera locations and object trajectory") + ax.set_title( + f"Scene: camera locations and object trajectory\n" + f"frame {frame_number}: {seeing_count}/{total_cameras} cameras see the object" + ) ax.legend(loc="upper left", fontsize=8) # Equal aspect over the combined bounds so directions are not skewed. @@ -195,10 +270,29 @@ def main() -> None: default=_DEFAULT_OUT, help="output PNG path (default: docs/assets/scene_3d.png)", ) + parser.add_argument( + "--frame", + type=int, + default=None, + help="frame to colour camera coverage for (default: the manifest's first frame)", + ) args = parser.parse_args() manifest = _load_manifest(args.manifest) - out_path = render(manifest, args.out) + available = _frame_numbers(manifest) + if args.frame is None: + frame_number = available[0] if available else 0 + elif args.frame in available: + frame_number = args.frame + elif available: + parser.error( + f"--frame {args.frame} is not in the manifest; " + f"available frames: {available[0]}..{available[-1]}" + ) + else: + parser.error(f"--frame {args.frame} is not in the manifest; it has no frames") + + out_path = render(manifest, args.out, frame_number) size = out_path.stat().st_size print(f"wrote {out_path} ({size} bytes)") diff --git a/tests/test_view_scene_3d.py b/tests/test_view_scene_3d.py new file mode 100644 index 0000000..2dbf8df --- /dev/null +++ b/tests/test_view_scene_3d.py @@ -0,0 +1,114 @@ +"""Coverage colouring in the 3D scene viewer. + +The viewer must read camera coverage straight off the manifest's per-camera +``in_view`` flags rather than reprojecting, so these tests assert against the +handoff_ltr scene whose coverage pattern is documented in ``test_handoff_ltr``: +LEFT sees the parcel early and loses it, the RIGHT cameras do the reverse, and +WIDE sees it on every frame. +""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +from types import ModuleType + +import pytest + +from multicam_sim import build_handoff_ltr_scene, build_manifest +from multicam_sim.handoff_ltr import CAM_LEFT, CAM_MID_RIGHT, CAM_RIGHT, CAM_WIDE + + +def _viewer() -> ModuleType: + script_path = Path(__file__).resolve().parents[1] / "scripts" / "view_scene_3d.py" + spec = importlib.util.spec_from_file_location("view_scene_3d", script_path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _handoff_manifest() -> dict: + manifest: dict = json.loads(build_manifest(build_handoff_ltr_scene()).to_json()) + return manifest + + +def test_frame_numbers_are_sorted_and_unique() -> None: + frames = _viewer()._frame_numbers(_handoff_manifest()) + + assert frames == sorted(set(frames)) + assert frames[0] == 0 + + +def test_wide_camera_sees_the_parcel_on_every_frame() -> None: + viewer, manifest = _viewer(), _handoff_manifest() + + for frame in viewer._frame_numbers(manifest): + assert CAM_WIDE in viewer._coverage_at_frame(manifest, frame) + + +def test_left_and_right_cameras_swap_coverage_across_the_handoff() -> None: + """LEFT is green early and grey late; the RIGHT cameras are the reverse.""" + viewer, manifest = _viewer(), _handoff_manifest() + frames = viewer._frame_numbers(manifest) + + first = viewer._coverage_at_frame(manifest, frames[0]) + last = viewer._coverage_at_frame(manifest, frames[-1]) + + assert CAM_LEFT in first + assert CAM_LEFT not in last + assert CAM_RIGHT not in first + assert CAM_RIGHT in last + + +def test_coverage_matches_the_manifest_in_view_flags_exactly() -> None: + """The viewer reports coverage from ``in_view`` alone, never reprojection.""" + viewer, manifest = _viewer(), _handoff_manifest() + frame_number = viewer._frame_numbers(manifest)[0] + + frame = next(f for f in manifest["entities"][0]["frames"] if f["frame"] == frame_number) + expected = { + obs["cam"] + for point in frame["points"].values() + for obs in point["per_cam"] + if obs["in_view"] + } + + assert set(viewer._coverage_at_frame(manifest, frame_number)) == expected + + +def test_overlap_frame_has_two_near_cameras_on_the_parcel() -> None: + """The handoff scene has a band where both right-hand cameras see it.""" + viewer, manifest = _viewer(), _handoff_manifest() + + overlaps = [ + frame + for frame in viewer._frame_numbers(manifest) + if {CAM_MID_RIGHT, CAM_RIGHT} <= set(viewer._coverage_at_frame(manifest, frame)) + ] + + assert overlaps, "expected at least one mid-right/right overlap frame" + + +def test_seeing_cameras_get_a_ray_to_the_point_they_see() -> None: + viewer, manifest = _viewer(), _handoff_manifest() + frame_number = viewer._frame_numbers(manifest)[0] + + seen = viewer._coverage_at_frame(manifest, frame_number) + + assert seen, "expected at least one camera to see the parcel on the first frame" + for targets in seen.values(): + assert targets, "a seeing camera must carry the point it sees" + assert all(target.shape == (3,) for target in targets) + + +def test_render_writes_a_png_for_a_chosen_frame(tmp_path: Path) -> None: + pytest.importorskip("matplotlib") + viewer, manifest = _viewer(), _handoff_manifest() + out = tmp_path / "scene.png" + + written = viewer.render(manifest, out, viewer._frame_numbers(manifest)[-1]) + + assert written == out + assert out.stat().st_size > 0