feat(detection): add StreamingEval for incremental per-image evaluation

CHANGELOG.md

@@ -13,6 +13,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
   precision-recall curve straight from ranked match flags, without the
   cumulative TP/FP arrays `precision_recall_curve_into` reads. Same values, same
   emission order; `accumulate()` now runs on it (below).
+- **`StreamingEval`** (Rust `hotcoco::StreamingEval`, Python `hotcoco.StreamingEval`)
+  — incremental evaluation: call `add_image()` as each image's ground truth and
+  detections become available, instead of loading a whole dataset and calling
+  `COCOeval::evaluate()` once at the end. Per-image matching runs immediately in
+  `add_image()` (overlappable with other work, such as postprocessing between
+  training steps); `finalize()` assembles every image seen so far into an
+  ordinary `COCOeval`, ready for `accumulate()` → `summarize()` → `report()`.
+  This moves `loadRes` and `evaluate()`'s matching off the end-of-epoch critical
+  path — together the largest two phases of `compute()` on an RF-DETR-shaped
+  workload. No Open Images support (hierarchy expansion needs the whole GT
+  dataset up front); the category list is fixed at construction, so a category
+  with zero annotations still gets a scored `-1.0` slot instead of vanishing;
+  and the `COCOeval` `finalize()` returns carries categories but no annotations
+  in `coco_gt`/`coco_dt`, so `confusion_matrix()`, `tide_errors()`, `compare()`,
+  and `slice_by()` need a batch-built `COCOeval` instead. A per-cell `EvalImg`
+  compaction format (~20 B/detection, cutting peak memory) is left for a
+  follow-up PR that can use this one's bit-identity tests as its oracle. New
+  Rust integration tests reproduce a tie-heavy fixture and LVIS federated
+  filtering (`neg_category_ids`/`not_exhaustive_category_ids`) bit-for-bit
+  against the batch pipeline, with two proven fault injections (a wrong
+  `(image, category)` sort order, a dropped area-range slot); new Python tests
+  cover the same LVIS filtering and the spent-evaluator error paths.
 
 ### Changed
 
@@ -316,7 +338,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
   back to. Previously the marker existed only on `EvalReport`, which nothing that
   writes a file uses, so comparability died with the process.
 
-
 - **`hotcoco.metrics` and `hotcoco.primitives` — the functional layer.** Metric
   functions you can call on plain arrays, with no evaluator, no dataset, and no COCO
   JSON:

crates/hotcoco-pyo3/src/lib.rs

@@ -126,8 +126,8 @@ fn freq_group_name(group: hotcoco_core::FreqGroup) -> &'static str {
 
 use convert::{
     IdList, NameList, annotation_to_py, category_to_py, confusion_counts_to_py,
-    dataset_stats_to_py, f64_array, image_to_py, map_to_dict, py_to_annotation, py_to_dataset,
-    rle_to_coco_py,
+    dataset_stats_to_py, f64_array, image_to_py, map_to_dict, py_to_annotation, py_to_category,
+    py_to_dataset, py_to_image, rle_to_coco_py,
 };
 
 // ---------------------------------------------------------------------------
@@ -2618,6 +2618,153 @@ Example\n\
     }
 }
 
+// ---------------------------------------------------------------------------
+// StreamingEval
+// ---------------------------------------------------------------------------
+
+#[doc = "Incremental (streaming) COCO evaluation.
+
+Feed images one at a time as their ground truth and detections become
+available — during postprocessing, overlapped with other work — instead of
+loading a whole dataset and evaluating it in one batch call. Matching runs
+immediately in ``add_image()``; ``finalize()`` assembles every image seen so
+far into an ordinary ``COCOeval``, ready for ``accumulate()`` →
+``summarize()`` → ``report()``. What this moves off the critical path is
+``loadRes`` and the per-image matching normally done in ``COCOeval.evaluate()``.
+
+Restrictions:
+
+- No Open Images support — the constructor raises ``ValueError``.
+- The category list is fixed at construction: pass every category the run
+  will ever see, including ones with no annotations in any image, so they
+  still get a scored slot (``-1.0``) instead of silently vanishing.
+- The ``COCOeval`` returned by ``finalize()`` has ``coco_gt``/``coco_dt``
+  populated with categories only, no annotations — enough for
+  ``accumulate()``, ``summarize()``, ``report()``, and ``results()``, which
+  never read annotations off those objects. ``confusion_matrix()``,
+  ``tide()``, ``compare()``, and ``slice_by()`` do read real annotations and
+  will silently see an empty dataset on a streaming-finalized evaluator;
+  build those from a batch ``COCOeval`` instead.
+
+>>> se = StreamingEval(categories, iou_type='bbox')
+>>> for image, gt_anns, dt_anns in batches:
+...     se.add_image(image, gt_anns, dt_anns)
+>>> ev = se.finalize()
+>>> ev.accumulate()
+>>> ev.summarize()
+"]
+#[pyclass(name = "StreamingEval")]
+struct PyStreamingEval {
+    /// `None` after `finalize()` consumes it — `add_image`/`finalize` on a
+    /// spent evaluator raise `RuntimeError` instead of panicking.
+    inner: Option<hotcoco_core::StreamingEval>,
+}
+
+fn dict_list_to<T>(
+    list: &Bound<'_, PyList>,
+    what: &str,
+    convert: impl Fn(&Bound<'_, PyDict>) -> PyResult<T>,
+) -> PyResult<Vec<T>> {
+    list.iter()
+        .map(|item| {
+            let dict = item.cast::<PyDict>().map_err(|_| {
+                pyo3::exceptions::PyTypeError::new_err(format!(
+                    "{what}: list elements must be dicts"
+                ))
+            })?;
+            convert(dict)
+        })
+        .collect()
+}
+
+#[pymethods]
+impl PyStreamingEval {
+    #[new]
+    #[pyo3(signature = (categories, iou_type="bbox", lvis_style=false, params=None))]
+    fn new(
+        categories: &Bound<'_, PyList>,
+        iou_type: &str,
+        lvis_style: bool,
+        params: Option<&PyParams>,
+    ) -> PyResult<Self> {
+        let iou = parse_iou_type(iou_type)?;
+        let categories = dict_list_to(categories, "StreamingEval", py_to_category)?;
+
+        let params = match params {
+            Some(p) => p.inner.clone(),
+            None => {
+                let mut p = hotcoco_core::Params::new(iou);
+                if lvis_style {
+                    p.max_dets = vec![300];
+                }
+                p
+            }
+        };
+        let eval_mode = if lvis_style {
+            hotcoco_core::EvalMode::Lvis
+        } else {
+            hotcoco_core::EvalMode::Coco
+        };
+
+        let inner = hotcoco_core::StreamingEval::new(params, eval_mode, categories)
+            .map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))?;
+        Ok(PyStreamingEval { inner: Some(inner) })
+    }
+
+    #[doc = "Match one image's ground truth against its detections, immediately.
+
+``image`` is a dict with at least ``id`` — plus, in LVIS mode,
+``neg_category_ids``/``not_exhaustive_category_ids`` where they apply.
+``gt_anns``/``dt_anns`` are lists of annotation dicts, COCO-shaped the same
+way ``COCO(dict)`` accepts them; detection dicts need ``score``.
+
+Raises ``RuntimeError`` if called after ``finalize()``."]
+    fn add_image(
+        &mut self,
+        image: &Bound<'_, PyDict>,
+        gt_anns: &Bound<'_, PyList>,
+        dt_anns: &Bound<'_, PyList>,
+    ) -> PyResult<()> {
+        let image = py_to_image(image)?;
+        let gt = dict_list_to(gt_anns, "add_image", py_to_annotation)?;
+        let dt = dict_list_to(dt_anns, "add_image", py_to_annotation)?;
+
+        let inner = self.inner.as_mut().ok_or_else(|| {
+            pyo3::exceptions::PyRuntimeError::new_err(
+                "add_image() called after finalize(); this StreamingEval is spent",
+            )
+        })?;
+        inner.add_image(&image, &gt, &dt);
+        Ok(())
+    }
+
+    #[doc = "Assemble every image seen so far into a ``COCOeval``, ready for
+``accumulate()`` → ``summarize()`` → ``report()``.
+
+Consumes this ``StreamingEval`` — calling ``add_image()`` or ``finalize()``
+again afterwards raises ``RuntimeError``."]
+    fn finalize(&mut self, py: Python<'_>) -> PyResult<PyCOCOeval> {
+        let inner = self.inner.take().ok_or_else(|| {
+            pyo3::exceptions::PyRuntimeError::new_err(
+                "finalize() already called; this StreamingEval is spent",
+            )
+        })?;
+        let ev = inner.finalize();
+        let params = Py::new(
+            py,
+            PyParams {
+                inner: ev.params.clone(),
+            },
+        )?;
+        Ok(PyCOCOeval {
+            inner: ev,
+            params,
+            eval_cache: None,
+            eval_params: None,
+        })
+    }
+}
+
 // ---------------------------------------------------------------------------
 // EvalImg / AccumulatedEval → Python converters
 // ---------------------------------------------------------------------------
@@ -2871,6 +3018,7 @@ fn compare(
 fn hotcoco(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_class::<PyCOCO>()?;
     m.add_class::<PyCOCOeval>()?;
+    m.add_class::<PyStreamingEval>()?;
     m.add_class::<PyParams>()?;
     m.add_class::<PyHierarchy>()?;
     m.add_function(wrap_pyfunction!(init_as_pycocotools, m)?)?;

crates/hotcoco-pyo3/tests/test_streaming_eval.py

@@ -0,0 +1,151 @@
+"""Regression tests for `StreamingEval` (candidate H, PyO3 layer).
+
+`StreamingEval.add_image()`/`finalize()` must reproduce the batch
+`COCO(dict)` + `COCOeval` pipeline exactly, one image at a time, and must
+raise rather than silently misbehave once `finalize()` has consumed it. None
+of these need the gitignored data/ directory.
+"""
+
+import hotcoco
+import pytest
+from hotcoco import COCO, COCOeval, StreamingEval
+
+
+def categories():
+    return [{"id": 1, "name": "person"}, {"id": 2, "name": "dog"}, {"id": 3, "name": "no-gt"}]
+
+
+def images():
+    return [
+        {"id": 1, "width": 100, "height": 100, "file_name": "a.jpg"},
+        {"id": 2, "width": 100, "height": 100, "file_name": "b.jpg"},
+    ]
+
+
+def gt_annotations():
+    return [
+        {"id": 1, "image_id": 1, "category_id": 1, "bbox": [10, 10, 30, 30], "area": 900, "iscrowd": 0},
+        {"id": 2, "image_id": 2, "category_id": 2, "bbox": [50, 50, 20, 20], "area": 400, "iscrowd": 0},
+    ]
+
+
+def dt_annotations():
+    return [
+        {"id": 101, "image_id": 1, "category_id": 1, "bbox": [11, 11, 30, 30], "score": 0.9},
+        # Unmatched — falls outside gt 2's box entirely.
+        {"id": 102, "image_id": 2, "category_id": 2, "bbox": [5, 5, 10, 10], "score": 0.6},
+    ]
+
+
+def group_by_image(anns):
+    by_image = {}
+    for ann in anns:
+        by_image.setdefault(ann["image_id"], []).append(ann)
+    return by_image
+
+
+def all_area_cell(eval_imgs):
+    """The `aRng == [0.0, 1e10]` ("all") cell among an evaluator's `eval_imgs`."""
+    return next(c for c in eval_imgs if c is not None and c["aRng"] == [0.0, 1e10])
+
+
+# ---------------------------------------------------------------------------
+# StreamingEval reproduces the batch pipeline
+# ---------------------------------------------------------------------------
+
+
+class TestStreamingMatchesBatch:
+    def test_get_results_equal_to_batch(self):
+        cats, imgs, gts, dts = categories(), images(), gt_annotations(), dt_annotations()
+
+        gt_ds = {"images": imgs, "annotations": gts, "categories": cats}
+        dt_ds = {"images": imgs, "annotations": dts, "categories": cats}
+        batch = COCOeval(COCO(gt_ds), COCO(dt_ds).load_res(dts), "bbox")
+        batch.evaluate()
+        batch.accumulate()
+        batch.summarize()
+
+        se = StreamingEval(cats, iou_type="bbox")
+        gt_by_image, dt_by_image = group_by_image(gts), group_by_image(dts)
+        for image in imgs:
+            se.add_image(image, gt_by_image.get(image["id"], []), dt_by_image.get(image["id"], []))
+        streamed = se.finalize()
+        streamed.accumulate()
+        streamed.summarize()
+
+        # Category 3 ("no-gt") has zero annotations on either side and must
+        # still show up as a -1.0 slot in both, not vanish from the K axis.
+        assert batch.get_results(per_class=True) == streamed.get_results(per_class=True)
+        assert batch.stats.tolist() == streamed.stats.tolist()
+
+    def test_empty_image_is_a_no_op(self):
+        se = StreamingEval(categories(), iou_type="bbox")
+        se.add_image({"id": 99, "width": 10, "height": 10}, [], [])
+        ev = se.finalize()
+        # No cells were ever populated: the same "accumulate before evaluate"
+        # warning a batch COCOeval gives for a dataset with zero annotations.
+        with pytest.warns(UserWarning, match="accumulate"):
+            ev.accumulate()
+        ev.summarize()
+        assert all(v == -1.0 for v in ev.stats.tolist())
+
+
+class TestStreamingLvisFederatedCategories:
+    def test_neg_category_scores_unmatched_dt_as_fp(self):
+        cats = [{"id": 1, "name": "a"}, {"id": 2, "name": "b"}]
+        image = {"id": 1, "width": 100, "height": 100, "neg_category_ids": [2]}
+        se = StreamingEval(cats, iou_type="bbox", lvis_style=True)
+        se.add_image(
+            image,
+            [],
+            [{"id": 101, "image_id": 1, "category_id": 2, "bbox": [0, 0, 10, 10], "score": 0.9}],
+        )
+        ev = se.finalize()
+        # A confirmed-negative category is not dropped: the cell exists, and
+        # the unmatched detection is not ignored (dtIgnore False) — it scores
+        # as a false positive at every IoU threshold.
+        cell = all_area_cell(ev.eval_imgs)
+        assert cell["dtIds"] == [101]
+        assert cell["dtIgnore"] == [[False]] * len(cell["dtIgnore"])
+
+    def test_not_exhaustive_category_ignores_unmatched_dt(self):
+        cats = [{"id": 1, "name": "a"}]
+        image = {"id": 1, "width": 100, "height": 100, "not_exhaustive_category_ids": [1]}
+        se = StreamingEval(cats, iou_type="bbox", lvis_style=True)
+        se.add_image(
+            image,
+            [{"id": 1, "image_id": 1, "category_id": 1, "bbox": [10, 10, 30, 30], "area": 900, "iscrowd": 0}],
+            # Unmatched, but not_exhaustive: ignored, not a false positive.
+            [{"id": 101, "image_id": 1, "category_id": 1, "bbox": [50, 50, 10, 10], "score": 0.9}],
+        )
+        ev = se.finalize()
+        # dtIgnore is True at every threshold for the unmatched detection —
+        # without not_exhaustive it would be False (a scored false positive).
+        cell = all_area_cell(ev.eval_imgs)
+        assert cell["dtIds"] == [101]
+        assert cell["dtIgnore"] == [[True]] * len(cell["dtIgnore"])
+
+
+# ---------------------------------------------------------------------------
+# A spent StreamingEval raises instead of silently misbehaving
+# ---------------------------------------------------------------------------
+
+
+class TestSpentStreamingEvalRaises:
+    def test_add_image_after_finalize_raises(self):
+        se = StreamingEval(categories(), iou_type="bbox")
+        se.add_image(images()[0], gt_annotations()[:1], dt_annotations()[:1])
+        se.finalize()
+        with pytest.raises(RuntimeError, match="spent"):
+            se.add_image(images()[0], [], [])
+
+    def test_finalize_twice_raises(self):
+        se = StreamingEval(categories(), iou_type="bbox")
+        se.add_image(images()[0], gt_annotations()[:1], dt_annotations()[:1])
+        se.finalize()
+        with pytest.raises(RuntimeError, match="spent"):
+            se.finalize()
+
+
+def test_streaming_eval_is_exported():
+    assert hotcoco.StreamingEval is StreamingEval

crates/hotcoco/src/detection/matching.rs

@@ -590,3 +590,32 @@ pub(super) struct EvalImgContext<'a> {
     /// times per evaluation.
     pub(super) match_floors: &'a [f64],
 }
+
+/// Construct an [`EvalImgContext`] from its parts.
+///
+/// The one place outside `evaluate()` allowed to write the `ious` field:
+/// `tests/architecture.rs` restricts the literal `ious:` struct-literal syntax
+/// to this module, `mod.rs`, and `evaluate.rs`, because that field is normally
+/// the whole-dataset similarity cache and must stay driver-private. Streaming
+/// evaluation builds its own tiny, per-image similarity map — never the
+/// whole-dataset one — but still needs a context to hand to
+/// [`gather_pair`]/[`evaluate_cell`]. Routing through here keeps the one
+/// allowed write site as-is instead of widening that allowlist for a scratch
+/// map the check was never guarding against.
+pub(super) fn build_context<'a>(
+    coco_gt: &'a COCO,
+    coco_dt: &'a COCO,
+    params: &'a Params,
+    ious: &'a HashMap<(u64, u64), IouMatrix>,
+    eval_mode: EvalMode,
+    match_floors: &'a [f64],
+) -> EvalImgContext<'a> {
+    EvalImgContext {
+        coco_gt,
+        coco_dt,
+        params,
+        ious,
+        eval_mode,
+        match_floors,
+    }
+}