feat(metric-benchmark): correlate matched subset scores

labs/metric-benchmark/README.md

@@ -26,7 +26,7 @@ Python tool and library API:
 2. **[Compute mutant scores](docs/compute-mutant-scores.md):** measure degradation across the mutants for each
    predefined task subset.
 3. **[Compute coverage](docs/compute-coverage.md):** evaluate our metric on those same subsets.
-4. **Correlation:** compare the paired coverage and mutant scores across subsets.
+4. **[Correlation](docs/correlate.md):** compare the paired coverage and mutant scores across subsets.
 
 See the [architecture](docs/architecture.md) and [stage descriptions](docs/stages.md)
 for how the experiment works.

labs/metric-benchmark/correlate.py

@@ -0,0 +1,35 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Tie-aware descriptive Spearman correlation over matched subset scores."""
+
+import argparse
+from pathlib import Path
+
+from metric_benchmark.correlation import correlate
+from metric_benchmark.coverage import CoverageScores
+from metric_benchmark.io import cli, read_json, write_json
+from metric_benchmark.scoring import MutantScores
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    for name in ("coverage", "mutant-scores", "output"):
+        parser.add_argument(f"--{name}", type=Path, required=True)
+    parser.add_argument("--missing-policy", choices=("error", "exclude"), default="error")
+
+    def run(args: argparse.Namespace) -> None:
+        write_json(
+            args.output,
+            correlate(
+                CoverageScores.model_validate(read_json(args.coverage)),
+                MutantScores.model_validate(read_json(args.mutant_scores)),
+                missing_policy=args.missing_policy,
+            ),
+        )
+
+    cli(parser, run)
+
+
+if __name__ == "__main__":
+    main()

labs/metric-benchmark/docs/architecture.md

@@ -212,7 +212,7 @@ configuration and coordinates their execution.
 | Execute evals | `run_evals.py` / `metric_benchmark.execution.execute_benchmark`; a future NeMo Helix tool will produce the same results-directory contract |
 | Compute mutant scores | [`compute_mutant_scores.py`](compute-mutant-scores.md) / `metric_benchmark.scoring.compute_mutant_scores` |
 | Compute coverage | [`compute_coverage.py`](compute-coverage.md) / `metric_benchmark.coverage.invoke_coverage` |
-| Correlation | `correlate.py` / `metric_benchmark.correlation.correlate` |
+| Correlation | [`correlate.py`](correlate.md) / `metric_benchmark.correlation.correlate` |
 
 External prep builds the benchmark and its fixed subset manifest. The
 experiment runner consumes those artifacts. Execution tools

labs/metric-benchmark/docs/correlate.md

@@ -0,0 +1,32 @@
+<!-- SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -->
+<!-- SPDX-License-Identifier: Apache-2.0 -->
+
+# correlate.py
+
+Correlate paired coverage and mutant scores across task subsets.
+
+## Usage
+
+```bash
+uv run python labs/metric-benchmark/correlate.py \
+  --coverage coverage-scores.json --mutant-scores mutant-scores.json \
+  --output correlation.json
+```
+
+## Inputs and behavior
+
+Artifacts must match in source identity, selection and subset IDs. Correlation
+uses Spearman ranks with average ties. Constant ranks or fewer than two usable
+pairs give an explicit undefined coefficient. `--missing-policy exclude` is an
+opt-in that records excluded pairs and their reasons. The default rejects
+unavailable pairs. This coefficient is descriptive: overlapping subsets do not
+provide independent samples, and the tool does not report significance tests.
+
+With exactly two usable pairs and no ties on either axis, the coefficient is
+necessarily +1 or -1; inspect the included pair count before interpreting it.
+Analysis artifact digests identify canonical model content, whereas prepared-file
+pins identify exact file bytes (including formatting).
+
+## Python library
+
+`correlate(coverage, mutant_scores, missing_policy="error")`.

labs/metric-benchmark/docs/stages.md

@@ -18,7 +18,7 @@ fixed subset manifest.
 | 1. Run mutants x eval tasks | `run_evals.py` (local Harbor); future NeMo Helix implementation |
 | 2. Compute mutant scores | [`compute_mutant_scores.py`](compute-mutant-scores.md) |
 | 3. Compute coverage | [`compute_coverage.py`](compute-coverage.md) |
-| 4. Correlation | `correlate.py` |
+| 4. Correlation | [`correlate.py`](correlate.md) |
 
 ## Stage contract
 

labs/metric-benchmark/docs/tools.md

@@ -72,3 +72,8 @@ arguments, evidence staging, cached replay, and the Python API.
 
 The [`eval_author_coverage.py` adapter guide](eval-author-coverage.md) covers
 the Eval Author integration.
+
+## Correlate the subset scores
+
+See the [`correlate.py` command guide](correlate.md) for
+arguments, unavailable pairs, interpretation, and the Python API.

labs/metric-benchmark/metric_benchmark/correlation.py

@@ -0,0 +1,93 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Tie-aware descriptive Spearman correlation over matched subset scores."""
+
+import math
+from typing import Literal
+
+from metric_benchmark.coverage import CoverageScores
+from metric_benchmark.models import Digest, Model, Score, Text
+from metric_benchmark.scoring import Context, MutantScores, identity
+
+
+class Pair(Model):
+    eval_subset_id: Text
+    coverage: float
+    mutant_score: float
+
+
+class Correlation(Model):
+    artifact_type: Literal["correlation_results"] = "correlation_results"
+    context: Context
+    method: Literal["spearman_average_ties_v1"] = "spearman_average_ties_v1"
+    missing_policy: Literal["error", "exclude"]
+    coverage_sha256: Digest
+    mutant_scores_sha256: Digest
+    pairs: list[Pair]
+    exclusions: dict[Text, list[Text]]
+    coefficient: Score
+
+
+def ranks(values: list[float]) -> list[float]:
+    order = sorted(range(len(values)), key=lambda i: values[i])
+    result = [0.0] * len(values)
+    start = 0
+    while start < len(order):
+        stop = start + 1
+        while stop < len(order) and values[order[start]] == values[order[stop]]:
+            stop += 1
+        average = (start + 1 + stop) / 2
+        for i in order[start:stop]:
+            result[i] = average
+        start = stop
+    return result
+
+
+def spearman(x: list[float], y: list[float]) -> Score:
+    if len(x) != len(y) or any(not math.isfinite(v) for v in [*x, *y]):
+        raise ValueError("Spearman requires equal-length finite inputs")
+    if len(x) < 2:
+        return Score(value=None, missing_score_reason="fewer than two paired subsets")
+    mean = (len(x) + 1) / 2
+    rx = [r - mean for r in ranks(x)]
+    ry = [r - mean for r in ranks(y)]
+    xx, yy = math.fsum(r * r for r in rx), math.fsum(r * r for r in ry)
+    if xx == 0 or yy == 0:
+        return Score(value=None, missing_score_reason="constant score ranks")
+    value = math.fsum(a * b for a, b in zip(rx, ry, strict=True)) / math.sqrt(xx * yy)
+    return Score(value=max(-1.0, min(1.0, value)))
+
+
+def correlate(
+    coverage: CoverageScores, mutant_scores: MutantScores, *, missing_policy: Literal["error", "exclude"] = "error"
+) -> Correlation:
+    if missing_policy not in ("error", "exclude"):
+        raise ValueError("unknown missing policy")
+    if coverage.context != mutant_scores.context:
+        raise ValueError("score artifacts refer to different experiment inputs or selection")
+    if set(coverage.scores) != set(mutant_scores.scores):
+        raise ValueError("score artifacts must contain exactly the same subset IDs")
+    pairs, exclusions = [], {}
+    for subset_id in sorted(coverage.scores):
+        cov, mutant = coverage.scores[subset_id], mutant_scores.scores[subset_id]
+        if cov.value is None or mutant.value is None:
+            reasons = []
+            if cov.value is None:
+                reasons.append(f"coverage: {cov.missing_score_reason}")
+            if mutant.value is None:
+                reasons.append(f"mutant score: {mutant.missing_score_reason}")
+            exclusions[subset_id] = reasons
+        else:
+            pairs.append(Pair(eval_subset_id=subset_id, coverage=cov.value, mutant_score=mutant.value))
+    if exclusions and missing_policy == "error":
+        raise ValueError("unavailable pairs; explicitly select missing_policy=exclude to omit them")
+    return Correlation(
+        context=coverage.context,
+        missing_policy=missing_policy,
+        coverage_sha256=identity(coverage),
+        mutant_scores_sha256=identity(mutant_scores),
+        pairs=pairs,
+        exclusions=exclusions,
+        coefficient=spearman([p.coverage for p in pairs], [p.mutant_score for p in pairs]),
+    )

tests/test_metric_benchmark_correlation.py

@@ -0,0 +1,93 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import math
+
+import pytest
+from metric_benchmark.correlation import correlate, ranks, spearman
+from metric_benchmark.coverage import invoke_coverage
+from metric_benchmark.models import Score
+from metric_benchmark.scoring import compute_mutant_scores
+from test_metric_benchmark_coverage import coverage_example
+
+
+def test_tie_ranks_and_known_spearman_values():
+    assert ranks([5.0, 1.0, 5.0, 3.0]) == [3.5, 1.0, 3.5, 2.0]
+    assert spearman([1, 2, 3], [2, 4, 8]).value == 1
+    assert spearman([1, 2, 3], [8, 4, 2]).value == -1
+    assert spearman([1, 1, 2], [1, 2, 3]).value == pytest.approx(math.sqrt(3) / 2)
+    assert spearman([1, 1], [1, 2]).value is None
+    assert spearman([], []).value is None
+    with pytest.raises(ValueError):
+        spearman([float("nan")], [1])
+
+
+def test_matched_pairs_missing_policy_and_provenance(tmp_path):
+    args = coverage_example(tmp_path)
+    coverage = invoke_coverage(*args)
+    mutants = compute_mutant_scores(*args[:3])
+    result = correlate(coverage, mutants)
+    assert result.coefficient.value == pytest.approx(math.sqrt(3) / 2)
+    assert len(result.pairs) == 3
+    assert mutants.diagnostics
+    assert [(p.eval_subset_id, p.mutant_score) for p in result.pairs if p.mutant_score < 0] == [("subset-000003", -1.0)]
+    assert result.exclusions == {}
+    coverage.scores["subset-000001"] = Score(value=None, missing_score_reason="system abstained")
+    with pytest.raises(ValueError, match="unavailable"):
+        correlate(coverage, mutants)
+    result = correlate(coverage, mutants, missing_policy="exclude")
+    assert result.coefficient.value == 1
+    assert result.exclusions == {"subset-000001": ["coverage: system abstained"]}
+    coverage.context.case_id = "different"
+    with pytest.raises(ValueError, match="different experiment"):
+        correlate(coverage, mutants)
+
+
+def test_no_silent_intersection_of_mismatched_subsets(tmp_path):
+    args = coverage_example(tmp_path)
+    coverage = invoke_coverage(*args)
+    mutants = compute_mutant_scores(*args[:3])
+    coverage.scores.pop("subset-000001")
+    with pytest.raises(ValueError, match="same subset IDs"):
+        correlate(coverage, mutants, missing_policy="exclude")
+
+
+def test_correlation_cli_library_parity(tmp_path):
+    import subprocess
+    import sys
+
+    from metric_benchmark.coverage import invoke_coverage
+    from metric_benchmark.io import json_bytes, write_json
+    from metric_benchmark.scoring import compute_mutant_scores
+    from test_metric_benchmark_config import LAB
+    from test_metric_benchmark_coverage import coverage_example
+
+    args = coverage_example(tmp_path)
+    coverage = invoke_coverage(*args)
+    mutants = compute_mutant_scores(args[0], args[1], args[2])
+    write_json(tmp_path / "coverage.json", coverage)
+    write_json(tmp_path / "mutants.json", mutants)
+    out = tmp_path / "correlation.json"
+    subprocess.run(
+        [
+            sys.executable,
+            str(LAB / "correlate.py"),
+            "--coverage",
+            str(tmp_path / "coverage.json"),
+            "--mutant-scores",
+            str(tmp_path / "mutants.json"),
+            "--output",
+            str(out),
+        ],
+        check=True,
+    )
+    assert out.read_bytes() == json_bytes(correlate(coverage, mutants))
+
+
+def test_different_scoring_selection_cannot_be_correlated(tmp_path):
+    args = coverage_example(tmp_path)
+    coverage = invoke_coverage(*args)
+    args[2].direction = "lower_is_better"
+    mutants = compute_mutant_scores(*args[:3])
+    with pytest.raises(ValueError, match="different experiment inputs or selection"):
+        correlate(coverage, mutants)