diff --git a/README.md b/README.md index 778a5cd..a7a14f5 100644 --- a/README.md +++ b/README.md @@ -1288,12 +1288,15 @@ faircode benchmark --out results/ --n-resamples 2000 --n-permutations 2000 faircode benchmark COMPAS/audit.yaml # run a subset explicitly faircode benchmark --root ~/audits # search ~/audits/*/audit.yaml instead of the current directory faircode benchmark --no-plots # CSVs only: no figures/ and matplotlib is not needed +faircode benchmark --quiet # suppress per-combination progress on stderr ``` `--root PATH` is the directory searched for `*/audit.yaml` (one level down; default: the current directory), so the harness can run from outside the repository. It is ignored when you pass manifest paths. `--no-plots` skips the `figures/*.png` files; `results_fairness.csv`, `results_performance.csv` and `summary.csv` are still written. +By default, progress is written to stderr after each audit/strategy/model combination completes. +`--quiet` suppresses those progress lines; the existing final summary and warnings remain on stderr. Writes `results_fairness.csv`, `results_performance.csv`, `summary.csv`, and one `figures/_strategies.png` per audit (300 dpi, rendered by `faircode/figures.py` straight from diff --git a/faircode/benchmark.py b/faircode/benchmark.py index f167367..0ab5512 100644 --- a/faircode/benchmark.py +++ b/faircode/benchmark.py @@ -232,14 +232,17 @@ def load_audit(folder, strategy=None, random_state=None) -> AuditData: return prepare_audit(manifest, strategy, random_state) -def run_audit(manifest, n_resamples=2000, n_permutations=2000, random_state=None): +def run_audit( + manifest, n_resamples=2000, n_permutations=2000, random_state=None, progress_callback=None +): """Run the full strategy x model x metric grid for one manifest. Returns (fairness_rows, performance_rows). fairness_rows has one dict per (strategy, model, protected attribute, metric), plus one per (strategy, model, attribute pair) for the intersectional gap when two or more protected attributes are declared. performance_rows has one dict per - (strategy, model, performance metric). + (strategy, model, performance metric). If provided, progress_callback is + called with the audit, strategy and model after each combination completes. """ data = prepare_audit(manifest, random_state=random_state) rs, X_all, y, protected_masks = data.random_state, data.X_all, data.y, data.masks @@ -297,12 +300,17 @@ def run_audit(manifest, n_resamples=2000, n_permutations=2000, random_state=None "note": ("empty_intersectional_cell" if cell_empty else "superadditive" if inter["superadditive"] else None), }) + if progress_callback is not None: + progress_callback(manifest.name, strategy, model_name) return fairness_rows, performance_rows -def run_benchmark(root=".", audits=None, n_resamples=2000, n_permutations=2000): +def run_benchmark( + root=".", audits=None, n_resamples=2000, n_permutations=2000, progress_callback=None +): """Discover every audit.yaml under root (or use explicit manifest paths) - and run them all. Returns (fairness_df, performance_df) across every audit.""" + and run them all. Returns (fairness_df, performance_df) across every audit. + If provided, progress_callback is passed to each audit run.""" np.random.seed(GLOBAL_SEED) # defense-in-depth; see module docstring paths = [Path(a) for a in audits] if audits else discover_manifests(root) all_fairness = [] @@ -310,7 +318,13 @@ def run_benchmark(root=".", audits=None, n_resamples=2000, n_permutations=2000): for path in paths: try: manifest = load_manifest(path) - fairness_rows, performance_rows = run_audit(manifest, n_resamples, n_permutations) + if progress_callback is None: + fairness_rows, performance_rows = run_audit(manifest, n_resamples, n_permutations) + else: + fairness_rows, performance_rows = run_audit( + manifest, n_resamples, n_permutations, + progress_callback=progress_callback, + ) except (yaml.YAMLError, KeyError, ValueError, FileNotFoundError) as exc: # A malformed manifest (bad YAML syntax, a missing required key) # or a degenerate dataset (e.g. filtered down to 0 rows, which diff --git a/faircode/cli.py b/faircode/cli.py index cebb356..c894f54 100644 --- a/faircode/cli.py +++ b/faircode/cli.py @@ -538,6 +538,8 @@ def build_parser() -> argparse.ArgumentParser: help="permutation-test shuffles per metric (positive integer, default: 2000)") b.add_argument("--no-plots", action="store_true", help="skip rendering figures/*.png (no matplotlib needed)") + b.add_argument("--quiet", action="store_true", + help="suppress per-combination benchmark progress on stderr") return parser @@ -911,10 +913,14 @@ def _held_out_uses_stdin(specs): file=sys.stderr, ) + def report_progress(audit, strategy, model): + print(f"completed {audit} / {strategy} / {model}", file=sys.stderr, flush=True) + try: fairness_df, performance_df = run_benchmark( root=args.root, audits=args.manifests or None, n_resamples=args.n_resamples, n_permutations=args.n_permutations, + progress_callback=None if args.quiet else report_progress, ) except ValueError as exc: print(f"error: {exc}", file=sys.stderr) diff --git a/tests/test_benchmark_progress.py b/tests/test_benchmark_progress.py new file mode 100644 index 0000000..a069f67 --- /dev/null +++ b/tests/test_benchmark_progress.py @@ -0,0 +1,81 @@ +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +pytest.importorskip("sklearn", reason="benchmark extra required") +pytest.importorskip("fairlearn", reason="benchmark extra required") +pytest.importorskip("yaml", reason="benchmark extra required") + +import faircode.benchmark as benchmark +from faircode.cli import main + + +def test_run_audit_reports_each_completed_strategy_model(monkeypatch): + strategies = ("first_strategy", "second_strategy") + models = ("first_model", "second_model") + manifest = SimpleNamespace(name="example_audit", protected_attributes=[]) + data = SimpleNamespace( + random_state=7, + X_all=None, + y=None, + masks={}, + idx_train=None, + idx_test=None, + ) + monkeypatch.setattr(benchmark, "STRATEGIES", strategies) + monkeypatch.setattr(benchmark, "MODEL_FAMILIES", models) + monkeypatch.setattr(benchmark, "PERFORMANCE_METRICS", ("accuracy",)) + monkeypatch.setattr(benchmark, "prepare_audit", lambda *args, **kwargs: data) + monkeypatch.setattr(benchmark, "_run_strategy", lambda *args: (None, None, None)) + monkeypatch.setattr( + benchmark, + "compute_performance_metrics", + lambda *args, **kwargs: {"accuracy": {"value": 1.0}}, + ) + completed = [] + + fairness_rows, performance_rows = benchmark.run_audit( + manifest, progress_callback=lambda *item: completed.append(item) + ) + + assert fairness_rows == [] + assert len(performance_rows) == len(strategies) * len(models) + assert completed == [ + (manifest.name, strategy, model) + for strategy in strategies + for model in models + ] + + +@pytest.mark.parametrize("quiet", [False, True]) +def test_cli_benchmark_progress_uses_stderr_and_quiet_suppresses_it( + quiet, monkeypatch, tmp_path, capsys +): + manifest = SimpleNamespace(name="example_audit") + monkeypatch.setattr(benchmark, "discover_manifests", lambda root: [Path("audit.yaml")]) + monkeypatch.setattr(benchmark, "load_manifest", lambda path: manifest) + + def run_audit(manifest, n_resamples, n_permutations, progress_callback=None): + if progress_callback is not None: + progress_callback(manifest.name, "baseline", "logistic_regression") + rows = [{"audit": manifest.name}] + return rows, rows + + monkeypatch.setattr(benchmark, "run_audit", run_audit) + monkeypatch.setattr("faircode.benchmark.collect_provenance", lambda *args, **kwargs: {}) + monkeypatch.setattr("faircode.benchmark.write_report", lambda *args, **kwargs: None) + args = ["benchmark", "--no-plots", "--out", str(tmp_path / "results")] + if quiet: + args.append("--quiet") + + assert main(args) == 0 + captured = capsys.readouterr() + assert captured.out == "" + progress_line = "completed example_audit / baseline / logistic_regression" + if quiet: + assert progress_line not in captured.err + else: + assert captured.err.startswith(progress_line + "\n") + assert "Ran 1 audit(s), wrote 1 fairness rows and 1 performance rows" in captured.err