diff --git a/labs/metric-benchmark/README.md b/labs/metric-benchmark/README.md index 8f5b821c..3b33af42 100644 --- a/labs/metric-benchmark/README.md +++ b/labs/metric-benchmark/README.md @@ -26,7 +26,7 @@ Python tool and library API: 2. **[Compute mutant scores](docs/compute-mutant-scores.md):** measure degradation across the mutants for each predefined task subset. 3. **[Compute coverage](docs/compute-coverage.md):** evaluate our metric on those same subsets. -4. **Correlation:** compare the paired coverage and mutant scores across subsets. +4. **[Correlation](docs/correlate.md):** compare the paired coverage and mutant scores across subsets. See the [architecture](docs/architecture.md) and [stage descriptions](docs/stages.md) for how the experiment works. diff --git a/labs/metric-benchmark/correlate.py b/labs/metric-benchmark/correlate.py new file mode 100644 index 00000000..59327121 --- /dev/null +++ b/labs/metric-benchmark/correlate.py @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tie-aware descriptive Spearman correlation over matched subset scores.""" + +import argparse +from pathlib import Path + +from metric_benchmark.correlation import correlate +from metric_benchmark.coverage import CoverageScores +from metric_benchmark.io import cli, read_json, write_json +from metric_benchmark.scoring import MutantScores + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + for name in ("coverage", "mutant-scores", "output"): + parser.add_argument(f"--{name}", type=Path, required=True) + parser.add_argument("--missing-policy", choices=("error", "exclude"), default="error") + + def run(args: argparse.Namespace) -> None: + write_json( + args.output, + correlate( + CoverageScores.model_validate(read_json(args.coverage)), + MutantScores.model_validate(read_json(args.mutant_scores)), + missing_policy=args.missing_policy, + ), + ) + + cli(parser, run) + + +if __name__ == "__main__": + main() diff --git a/labs/metric-benchmark/docs/architecture.md b/labs/metric-benchmark/docs/architecture.md index 54c4fe72..147a2794 100644 --- a/labs/metric-benchmark/docs/architecture.md +++ b/labs/metric-benchmark/docs/architecture.md @@ -212,7 +212,7 @@ configuration and coordinates their execution. | Execute evals | `run_evals.py` / `metric_benchmark.execution.execute_benchmark`; a future NeMo Helix tool will produce the same results-directory contract | | Compute mutant scores | [`compute_mutant_scores.py`](compute-mutant-scores.md) / `metric_benchmark.scoring.compute_mutant_scores` | | Compute coverage | [`compute_coverage.py`](compute-coverage.md) / `metric_benchmark.coverage.invoke_coverage` | -| Correlation | `correlate.py` / `metric_benchmark.correlation.correlate` | +| Correlation | [`correlate.py`](correlate.md) / `metric_benchmark.correlation.correlate` | External prep builds the benchmark and its fixed subset manifest. The experiment runner consumes those artifacts. Execution tools diff --git a/labs/metric-benchmark/docs/correlate.md b/labs/metric-benchmark/docs/correlate.md new file mode 100644 index 00000000..3c57a5b0 --- /dev/null +++ b/labs/metric-benchmark/docs/correlate.md @@ -0,0 +1,32 @@ + + + +# correlate.py + +Correlate paired coverage and mutant scores across task subsets. + +## Usage + +```bash +uv run python labs/metric-benchmark/correlate.py \ + --coverage coverage-scores.json --mutant-scores mutant-scores.json \ + --output correlation.json +``` + +## Inputs and behavior + +Artifacts must match in source identity, selection and subset IDs. Correlation +uses Spearman ranks with average ties. Constant ranks or fewer than two usable +pairs give an explicit undefined coefficient. `--missing-policy exclude` is an +opt-in that records excluded pairs and their reasons. The default rejects +unavailable pairs. This coefficient is descriptive: overlapping subsets do not +provide independent samples, and the tool does not report significance tests. + +With exactly two usable pairs and no ties on either axis, the coefficient is +necessarily +1 or -1; inspect the included pair count before interpreting it. +Analysis artifact digests identify canonical model content, whereas prepared-file +pins identify exact file bytes (including formatting). + +## Python library + +`correlate(coverage, mutant_scores, missing_policy="error")`. diff --git a/labs/metric-benchmark/docs/stages.md b/labs/metric-benchmark/docs/stages.md index 013bc311..fc9ee169 100644 --- a/labs/metric-benchmark/docs/stages.md +++ b/labs/metric-benchmark/docs/stages.md @@ -18,7 +18,7 @@ fixed subset manifest. | 1. Run mutants x eval tasks | `run_evals.py` (local Harbor); future NeMo Helix implementation | | 2. Compute mutant scores | [`compute_mutant_scores.py`](compute-mutant-scores.md) | | 3. Compute coverage | [`compute_coverage.py`](compute-coverage.md) | -| 4. Correlation | `correlate.py` | +| 4. Correlation | [`correlate.py`](correlate.md) | ## Stage contract diff --git a/labs/metric-benchmark/docs/tools.md b/labs/metric-benchmark/docs/tools.md index ce73d9ae..07d95d22 100644 --- a/labs/metric-benchmark/docs/tools.md +++ b/labs/metric-benchmark/docs/tools.md @@ -72,3 +72,8 @@ arguments, evidence staging, cached replay, and the Python API. The [`eval_author_coverage.py` adapter guide](eval-author-coverage.md) covers the Eval Author integration. + +## Correlate the subset scores + +See the [`correlate.py` command guide](correlate.md) for +arguments, unavailable pairs, interpretation, and the Python API. diff --git a/labs/metric-benchmark/metric_benchmark/correlation.py b/labs/metric-benchmark/metric_benchmark/correlation.py new file mode 100644 index 00000000..dca3d7d5 --- /dev/null +++ b/labs/metric-benchmark/metric_benchmark/correlation.py @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tie-aware descriptive Spearman correlation over matched subset scores.""" + +import math +from typing import Literal + +from metric_benchmark.coverage import CoverageScores +from metric_benchmark.models import Digest, Model, Score, Text +from metric_benchmark.scoring import Context, MutantScores, identity + + +class Pair(Model): + eval_subset_id: Text + coverage: float + mutant_score: float + + +class Correlation(Model): + artifact_type: Literal["correlation_results"] = "correlation_results" + context: Context + method: Literal["spearman_average_ties_v1"] = "spearman_average_ties_v1" + missing_policy: Literal["error", "exclude"] + coverage_sha256: Digest + mutant_scores_sha256: Digest + pairs: list[Pair] + exclusions: dict[Text, list[Text]] + coefficient: Score + + +def ranks(values: list[float]) -> list[float]: + order = sorted(range(len(values)), key=lambda i: values[i]) + result = [0.0] * len(values) + start = 0 + while start < len(order): + stop = start + 1 + while stop < len(order) and values[order[start]] == values[order[stop]]: + stop += 1 + average = (start + 1 + stop) / 2 + for i in order[start:stop]: + result[i] = average + start = stop + return result + + +def spearman(x: list[float], y: list[float]) -> Score: + if len(x) != len(y) or any(not math.isfinite(v) for v in [*x, *y]): + raise ValueError("Spearman requires equal-length finite inputs") + if len(x) < 2: + return Score(value=None, missing_score_reason="fewer than two paired subsets") + mean = (len(x) + 1) / 2 + rx = [r - mean for r in ranks(x)] + ry = [r - mean for r in ranks(y)] + xx, yy = math.fsum(r * r for r in rx), math.fsum(r * r for r in ry) + if xx == 0 or yy == 0: + return Score(value=None, missing_score_reason="constant score ranks") + value = math.fsum(a * b for a, b in zip(rx, ry, strict=True)) / math.sqrt(xx * yy) + return Score(value=max(-1.0, min(1.0, value))) + + +def correlate( + coverage: CoverageScores, mutant_scores: MutantScores, *, missing_policy: Literal["error", "exclude"] = "error" +) -> Correlation: + if missing_policy not in ("error", "exclude"): + raise ValueError("unknown missing policy") + if coverage.context != mutant_scores.context: + raise ValueError("score artifacts refer to different experiment inputs or selection") + if set(coverage.scores) != set(mutant_scores.scores): + raise ValueError("score artifacts must contain exactly the same subset IDs") + pairs, exclusions = [], {} + for subset_id in sorted(coverage.scores): + cov, mutant = coverage.scores[subset_id], mutant_scores.scores[subset_id] + if cov.value is None or mutant.value is None: + reasons = [] + if cov.value is None: + reasons.append(f"coverage: {cov.missing_score_reason}") + if mutant.value is None: + reasons.append(f"mutant score: {mutant.missing_score_reason}") + exclusions[subset_id] = reasons + else: + pairs.append(Pair(eval_subset_id=subset_id, coverage=cov.value, mutant_score=mutant.value)) + if exclusions and missing_policy == "error": + raise ValueError("unavailable pairs; explicitly select missing_policy=exclude to omit them") + return Correlation( + context=coverage.context, + missing_policy=missing_policy, + coverage_sha256=identity(coverage), + mutant_scores_sha256=identity(mutant_scores), + pairs=pairs, + exclusions=exclusions, + coefficient=spearman([p.coverage for p in pairs], [p.mutant_score for p in pairs]), + ) diff --git a/tests/test_metric_benchmark_correlation.py b/tests/test_metric_benchmark_correlation.py new file mode 100644 index 00000000..a53b4e49 --- /dev/null +++ b/tests/test_metric_benchmark_correlation.py @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import math + +import pytest +from metric_benchmark.correlation import correlate, ranks, spearman +from metric_benchmark.coverage import invoke_coverage +from metric_benchmark.models import Score +from metric_benchmark.scoring import compute_mutant_scores +from test_metric_benchmark_coverage import coverage_example + + +def test_tie_ranks_and_known_spearman_values(): + assert ranks([5.0, 1.0, 5.0, 3.0]) == [3.5, 1.0, 3.5, 2.0] + assert spearman([1, 2, 3], [2, 4, 8]).value == 1 + assert spearman([1, 2, 3], [8, 4, 2]).value == -1 + assert spearman([1, 1, 2], [1, 2, 3]).value == pytest.approx(math.sqrt(3) / 2) + assert spearman([1, 1], [1, 2]).value is None + assert spearman([], []).value is None + with pytest.raises(ValueError): + spearman([float("nan")], [1]) + + +def test_matched_pairs_missing_policy_and_provenance(tmp_path): + args = coverage_example(tmp_path) + coverage = invoke_coverage(*args) + mutants = compute_mutant_scores(*args[:3]) + result = correlate(coverage, mutants) + assert result.coefficient.value == pytest.approx(math.sqrt(3) / 2) + assert len(result.pairs) == 3 + assert mutants.diagnostics + assert [(p.eval_subset_id, p.mutant_score) for p in result.pairs if p.mutant_score < 0] == [("subset-000003", -1.0)] + assert result.exclusions == {} + coverage.scores["subset-000001"] = Score(value=None, missing_score_reason="system abstained") + with pytest.raises(ValueError, match="unavailable"): + correlate(coverage, mutants) + result = correlate(coverage, mutants, missing_policy="exclude") + assert result.coefficient.value == 1 + assert result.exclusions == {"subset-000001": ["coverage: system abstained"]} + coverage.context.case_id = "different" + with pytest.raises(ValueError, match="different experiment"): + correlate(coverage, mutants) + + +def test_no_silent_intersection_of_mismatched_subsets(tmp_path): + args = coverage_example(tmp_path) + coverage = invoke_coverage(*args) + mutants = compute_mutant_scores(*args[:3]) + coverage.scores.pop("subset-000001") + with pytest.raises(ValueError, match="same subset IDs"): + correlate(coverage, mutants, missing_policy="exclude") + + +def test_correlation_cli_library_parity(tmp_path): + import subprocess + import sys + + from metric_benchmark.coverage import invoke_coverage + from metric_benchmark.io import json_bytes, write_json + from metric_benchmark.scoring import compute_mutant_scores + from test_metric_benchmark_config import LAB + from test_metric_benchmark_coverage import coverage_example + + args = coverage_example(tmp_path) + coverage = invoke_coverage(*args) + mutants = compute_mutant_scores(args[0], args[1], args[2]) + write_json(tmp_path / "coverage.json", coverage) + write_json(tmp_path / "mutants.json", mutants) + out = tmp_path / "correlation.json" + subprocess.run( + [ + sys.executable, + str(LAB / "correlate.py"), + "--coverage", + str(tmp_path / "coverage.json"), + "--mutant-scores", + str(tmp_path / "mutants.json"), + "--output", + str(out), + ], + check=True, + ) + assert out.read_bytes() == json_bytes(correlate(coverage, mutants)) + + +def test_different_scoring_selection_cannot_be_correlated(tmp_path): + args = coverage_example(tmp_path) + coverage = invoke_coverage(*args) + args[2].direction = "lower_is_better" + mutants = compute_mutant_scores(*args[:3]) + with pytest.raises(ValueError, match="different experiment inputs or selection"): + correlate(coverage, mutants)