feat: report benchmark progress

README.md

@@ -1288,12 +1288,15 @@ faircode benchmark --out results/ --n-resamples 2000 --n-permutations 2000
 faircode benchmark COMPAS/audit.yaml                # run a subset explicitly
 faircode benchmark --root ~/audits                  # search ~/audits/*/audit.yaml instead of the current directory
 faircode benchmark --no-plots                       # CSVs only: no figures/ and matplotlib is not needed
+faircode benchmark --quiet                          # suppress per-combination progress on stderr
 ```
 
 `--root PATH` is the directory searched for `*/audit.yaml` (one level down; default: the current
 directory), so the harness can run from outside the repository. It is ignored when you pass manifest
 paths. `--no-plots` skips the `figures/*.png` files; `results_fairness.csv`, `results_performance.csv`
 and `summary.csv` are still written.
+By default, progress is written to stderr after each audit/strategy/model combination completes.
+`--quiet` suppresses those progress lines; the existing final summary and warnings remain on stderr.
 
 Writes `results_fairness.csv`, `results_performance.csv`, `summary.csv`, and one
 `figures/<audit>_strategies.png` per audit (300 dpi, rendered by `faircode/figures.py` straight from

faircode/benchmark.py

@@ -232,14 +232,17 @@ def load_audit(folder, strategy=None, random_state=None) -> AuditData:
     return prepare_audit(manifest, strategy, random_state)
 
 
-def run_audit(manifest, n_resamples=2000, n_permutations=2000, random_state=None):
+def run_audit(
+    manifest, n_resamples=2000, n_permutations=2000, random_state=None, progress_callback=None
+):
     """Run the full strategy x model x metric grid for one manifest.
 
     Returns (fairness_rows, performance_rows). fairness_rows has one dict per
     (strategy, model, protected attribute, metric), plus one per (strategy,
     model, attribute pair) for the intersectional gap when two or more
     protected attributes are declared. performance_rows has one dict per
-    (strategy, model, performance metric).
+    (strategy, model, performance metric). If provided, progress_callback is
+    called with the audit, strategy and model after each combination completes.
     """
     data = prepare_audit(manifest, random_state=random_state)
     rs, X_all, y, protected_masks = data.random_state, data.X_all, data.y, data.masks
@@ -297,20 +300,31 @@ def run_audit(manifest, n_resamples=2000, n_permutations=2000, random_state=None
                         "note": ("empty_intersectional_cell" if cell_empty
                                   else "superadditive" if inter["superadditive"] else None),
                     })
+            if progress_callback is not None:
+                progress_callback(manifest.name, strategy, model_name)
     return fairness_rows, performance_rows
 
 
-def run_benchmark(root=".", audits=None, n_resamples=2000, n_permutations=2000):
+def run_benchmark(
+    root=".", audits=None, n_resamples=2000, n_permutations=2000, progress_callback=None
+):
     """Discover every audit.yaml under root (or use explicit manifest paths)
-    and run them all. Returns (fairness_df, performance_df) across every audit."""
+    and run them all. Returns (fairness_df, performance_df) across every audit.
+    If provided, progress_callback is passed to each audit run."""
     np.random.seed(GLOBAL_SEED)  # defense-in-depth; see module docstring
     paths = [Path(a) for a in audits] if audits else discover_manifests(root)
     all_fairness = []
     all_performance = []
     for path in paths:
         try:
             manifest = load_manifest(path)
-            fairness_rows, performance_rows = run_audit(manifest, n_resamples, n_permutations)
+            if progress_callback is None:
+                fairness_rows, performance_rows = run_audit(manifest, n_resamples, n_permutations)
+            else:
+                fairness_rows, performance_rows = run_audit(
+                    manifest, n_resamples, n_permutations,
+                    progress_callback=progress_callback,
+                )
         except (yaml.YAMLError, KeyError, ValueError, FileNotFoundError) as exc:
             # A malformed manifest (bad YAML syntax, a missing required key)
             # or a degenerate dataset (e.g. filtered down to 0 rows, which

faircode/cli.py

@@ -538,6 +538,8 @@ def build_parser() -> argparse.ArgumentParser:
                    help="permutation-test shuffles per metric (positive integer, default: 2000)")
     b.add_argument("--no-plots", action="store_true",
                    help="skip rendering figures/*.png (no matplotlib needed)")
+    b.add_argument("--quiet", action="store_true",
+                   help="suppress per-combination benchmark progress on stderr")
 
     return parser
 
@@ -911,10 +913,14 @@ def _held_out_uses_stdin(specs):
                 file=sys.stderr,
             )
 
+        def report_progress(audit, strategy, model):
+            print(f"completed {audit} / {strategy} / {model}", file=sys.stderr, flush=True)
+
         try:
             fairness_df, performance_df = run_benchmark(
                 root=args.root, audits=args.manifests or None,
                 n_resamples=args.n_resamples, n_permutations=args.n_permutations,
+                progress_callback=None if args.quiet else report_progress,
             )
         except ValueError as exc:
             print(f"error: {exc}", file=sys.stderr)

tests/test_benchmark_progress.py

@@ -0,0 +1,81 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+
+pytest.importorskip("sklearn", reason="benchmark extra required")
+pytest.importorskip("fairlearn", reason="benchmark extra required")
+pytest.importorskip("yaml", reason="benchmark extra required")
+
+import faircode.benchmark as benchmark
+from faircode.cli import main
+
+
+def test_run_audit_reports_each_completed_strategy_model(monkeypatch):
+    strategies = ("first_strategy", "second_strategy")
+    models = ("first_model", "second_model")
+    manifest = SimpleNamespace(name="example_audit", protected_attributes=[])
+    data = SimpleNamespace(
+        random_state=7,
+        X_all=None,
+        y=None,
+        masks={},
+        idx_train=None,
+        idx_test=None,
+    )
+    monkeypatch.setattr(benchmark, "STRATEGIES", strategies)
+    monkeypatch.setattr(benchmark, "MODEL_FAMILIES", models)
+    monkeypatch.setattr(benchmark, "PERFORMANCE_METRICS", ("accuracy",))
+    monkeypatch.setattr(benchmark, "prepare_audit", lambda *args, **kwargs: data)
+    monkeypatch.setattr(benchmark, "_run_strategy", lambda *args: (None, None, None))
+    monkeypatch.setattr(
+        benchmark,
+        "compute_performance_metrics",
+        lambda *args, **kwargs: {"accuracy": {"value": 1.0}},
+    )
+    completed = []
+
+    fairness_rows, performance_rows = benchmark.run_audit(
+        manifest, progress_callback=lambda *item: completed.append(item)
+    )
+
+    assert fairness_rows == []
+    assert len(performance_rows) == len(strategies) * len(models)
+    assert completed == [
+        (manifest.name, strategy, model)
+        for strategy in strategies
+        for model in models
+    ]
+
+
+@pytest.mark.parametrize("quiet", [False, True])
+def test_cli_benchmark_progress_uses_stderr_and_quiet_suppresses_it(
+    quiet, monkeypatch, tmp_path, capsys
+):
+    manifest = SimpleNamespace(name="example_audit")
+    monkeypatch.setattr(benchmark, "discover_manifests", lambda root: [Path("audit.yaml")])
+    monkeypatch.setattr(benchmark, "load_manifest", lambda path: manifest)
+
+    def run_audit(manifest, n_resamples, n_permutations, progress_callback=None):
+        if progress_callback is not None:
+            progress_callback(manifest.name, "baseline", "logistic_regression")
+        rows = [{"audit": manifest.name}]
+        return rows, rows
+
+    monkeypatch.setattr(benchmark, "run_audit", run_audit)
+    monkeypatch.setattr("faircode.benchmark.collect_provenance", lambda *args, **kwargs: {})
+    monkeypatch.setattr("faircode.benchmark.write_report", lambda *args, **kwargs: None)
+    args = ["benchmark", "--no-plots", "--out", str(tmp_path / "results")]
+    if quiet:
+        args.append("--quiet")
+
+    assert main(args) == 0
+    captured = capsys.readouterr()
+    assert captured.out == ""
+    progress_line = "completed example_audit / baseline / logistic_regression"
+    if quiet:
+        assert progress_line not in captured.err
+    else:
+        assert captured.err.startswith(progress_line + "\n")
+    assert "Ran 1 audit(s), wrote 1 fairness rows and 1 performance rows" in captured.err