diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 66c0587..7807b05 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: - id: ruff name: ruff (unused imports, undefined names, redefinitions) - entry: ruff check faircode scripts tests + entry: ruff check faircode scripts tests explainers/code language: system pass_filenames: false always_run: true diff --git a/Makefile b/Makefile index e36e5f4..9307591 100644 --- a/Makefile +++ b/Makefile @@ -56,6 +56,7 @@ lint: ## Enforce the em-dash-free rule + check for broken doc links + issue num $(PY) scripts/check_explainer_structure.py $(PY) scripts/check_traction.py $(PY) scripts/check_contributors.py + ruff check faircode scripts tests explainers/code ruff check faircode scripts tests audits: ## Run every audit's unfair.py/fair.py + the CLI smoke tests (mirrors audits.yml's required run-audits job, plus profiler/benchmark-harness's CLI steps) diff --git a/explainers/aggregation-bias.md b/explainers/aggregation-bias.md index 8ab9912..eb4d071 100644 --- a/explainers/aggregation-bias.md +++ b/explainers/aggregation-bias.md @@ -44,7 +44,6 @@ Two cautions. This model has only three features, so its absolute accuracy is lo Run from the repo root (`PYTHONPATH=. python3 explainers/code/aggregation-bias.py`; that file is generated from the block below). ```python -import numpy as np from sklearn.linear_model import LogisticRegression from faircode.benchmark import load_audit # same loader, encoding and split as the benchmark harness diff --git a/explainers/code/aggregation-bias.py b/explainers/code/aggregation-bias.py index 860ff4f..f4c1797 100644 --- a/explainers/code/aggregation-bias.py +++ b/explainers/code/aggregation-bias.py @@ -1,5 +1,4 @@ # Generated from explainers/aggregation-bias.md by scripts/build_explainers.py. Edit the markdown, not this file. -import numpy as np from sklearn.linear_model import LogisticRegression from faircode.benchmark import load_audit # same loader, encoding and split as the benchmark harness diff --git a/explainers/code/delayed-feedback.py b/explainers/code/delayed-feedback.py index 9ffff9a..6299d33 100644 --- a/explainers/code/delayed-feedback.py +++ b/explainers/code/delayed-feedback.py @@ -18,7 +18,6 @@ def monitored_gap(mean_delay_group, mean_delay_others, today=90): delay = np.minimum(delay, 29.9) seen_positive = (y == 1) & (discharge + delay <= today) window_closed = discharge + 30 <= today - known_negative = (y == 0) & window_closed # a naive monitor counts every discharged patient and treats every outcome it has not seen as "no" naive = {g: seen_positive[group == g].mean() for g in (True, False)} # a careful monitor only counts patients whose 30-day window is closed diff --git a/explainers/code/fairness-in-ranking.py b/explainers/code/fairness-in-ranking.py index 92a0b89..50a8dc9 100644 --- a/explainers/code/fairness-in-ranking.py +++ b/explainers/code/fairness-in-ranking.py @@ -38,7 +38,7 @@ def report(score, name): print(f"{name}\n female share of the top {K}: {female[top].mean():.3f} (pool {female.mean():.3f})" f"\n exposure ratio female/male, whole list: {whole[female == 1].mean() / whole[female == 0].mean():.3f}" f"\n exposure ratio female/male, top {K} only: {cut[female == 1].mean() / cut[female == 0].mean():.3f}" - f"\n exposure per unit of hire rate, female/male: " + "\n exposure per unit of hire rate, female/male: " f"{(cut[female == 1].mean() / hired[female == 1].mean()) / (cut[female == 0].mean() / hired[female == 0].mean()):.3f}") report(scores(X_biased), "baseline (Gender is a feature)") diff --git a/explainers/code/feature-importance-by-group.py b/explainers/code/feature-importance-by-group.py index 3cc3819..73ea037 100644 --- a/explainers/code/feature-importance-by-group.py +++ b/explainers/code/feature-importance-by-group.py @@ -1,5 +1,4 @@ # Generated from explainers/feature-importance-by-group.md by scripts/build_explainers.py. Edit the markdown, not this file. -import numpy as np from sklearn.inspection import permutation_importance from faircode.benchmark import load_audit # same loader, encoding and split as the benchmark harness from faircode.models import build_model diff --git a/explainers/code/hiring-pipeline-adverse-impact.py b/explainers/code/hiring-pipeline-adverse-impact.py index c6bdeb1..0fb0f03 100644 --- a/explainers/code/hiring-pipeline-adverse-impact.py +++ b/explainers/code/hiring-pipeline-adverse-impact.py @@ -16,7 +16,6 @@ def ratio_with_interval(passed, reached, n_boot=500): """Women's selection rate divided by everyone else's, among the people who reached the stage, with a bootstrap interval.""" w, m = reached & (women == 1), reached & (women == 0) - rate = lambda mask, rows: passed[rows][mask[rows]].mean() point = passed[w].mean() / passed[m].mean() idx_w, idx_m = np.flatnonzero(w), np.flatnonzero(m) boots = [passed[rng.choice(idx_w, len(idx_w))].mean() / passed[rng.choice(idx_m, len(idx_m))].mean() for _ in range(n_boot)] diff --git a/explainers/code/leave-one-group-out-evaluation.py b/explainers/code/leave-one-group-out-evaluation.py index 7103f0b..6b58ed8 100644 --- a/explainers/code/leave-one-group-out-evaluation.py +++ b/explainers/code/leave-one-group-out-evaluation.py @@ -1,5 +1,4 @@ # Generated from explainers/leave-one-group-out-evaluation.md by scripts/build_explainers.py. Edit the markdown, not this file. -import numpy as np import pandas as pd from faircode.benchmark import load_audit # same loader, encoding and split as the benchmark harness from faircode.models import build_model diff --git a/explainers/code/monitoring-fairness-in-production.py b/explainers/code/monitoring-fairness-in-production.py index 6f28431..8117d42 100644 --- a/explainers/code/monitoring-fairness-in-production.py +++ b/explainers/code/monitoring-fairness-in-production.py @@ -37,7 +37,7 @@ def batches_until_alert(batch, window, drift, drift_at=20, n_batches=60, z=3): return b - drift_at + 1 if b >= drift_at else -1 # -1 = a false alarm before the drift return None -print(f"\nalert rule: the gap over the last W batches is more than 3 standard errors from the launch value") +print("\nalert rule: the gap over the last W batches is more than 3 standard errors from the launch value") print(f"{'batch size':>10} {'window':>7} {'drift':>6} {'median batches to alert':>25} {'never alerted':>14} {'false alarms before the drift':>31}") for batch, window in ((200, 3), (500, 3), (1000, 3), (2000, 3), (1000, 1), (1000, 6)): for drift in (0.03, 0.06): diff --git a/explainers/code/small-group-shrinkage.py b/explainers/code/small-group-shrinkage.py index 2b51af1..11064e4 100644 --- a/explainers/code/small-group-shrinkage.py +++ b/explainers/code/small-group-shrinkage.py @@ -49,7 +49,7 @@ def shrink(successes, counts): small_raw.append(np.sqrt(np.mean((raw[small] - truth[small]) ** 2))); small_shrunk.append(np.sqrt(np.mean((shrunk[small] - truth[small]) ** 2))) estimates.append((raw[worst], shrunk[worst])) -print(f"\nerror (RMSE against the true cell accuracy) from a 600-row audit, averaged over 300 audits") +print("\nerror (RMSE against the true cell accuracy) from a 600-row audit, averaged over 300 audits") print(f" all {len(cells)} cells: raw {np.mean(raw_err):.3f} | shrunk {np.mean(shrunk_err):.3f}") print(f" the smallest {small.sum()} cells: raw {np.mean(small_raw):.3f} | shrunk {np.mean(small_shrunk):.3f}") mean_raw, mean_shrunk = np.mean(estimates, axis=0) diff --git a/explainers/code/survivorship-bias.py b/explainers/code/survivorship-bias.py index ae27715..1cca3f0 100644 --- a/explainers/code/survivorship-bias.py +++ b/explainers/code/survivorship-bias.py @@ -1,5 +1,4 @@ # Generated from explainers/survivorship-bias.md by scripts/build_explainers.py. Edit the markdown, not this file. -import numpy as np from sklearn.linear_model import LogisticRegression from faircode.benchmark import load_audit # same loader, encoding and split as the benchmark harness diff --git a/explainers/code/utilization-as-a-label.py b/explainers/code/utilization-as-a-label.py index 5f9c7ab..9c52476 100644 --- a/explainers/code/utilization-as-a-label.py +++ b/explainers/code/utilization-as-a-label.py @@ -1,5 +1,4 @@ # Generated from explainers/utilization-as-a-label.md by scripts/build_explainers.py. Edit the markdown, not this file. -import numpy as np import pandas as pd from faircode.benchmark import load_audit # same loader, encoding and split as the benchmark harness diff --git a/explainers/delayed-feedback.md b/explainers/delayed-feedback.md index 65e732a..b3aabcd 100644 --- a/explainers/delayed-feedback.md +++ b/explainers/delayed-feedback.md @@ -59,7 +59,6 @@ def monitored_gap(mean_delay_group, mean_delay_others, today=90): delay = np.minimum(delay, 29.9) seen_positive = (y == 1) & (discharge + delay <= today) window_closed = discharge + 30 <= today - known_negative = (y == 0) & window_closed # a naive monitor counts every discharged patient and treats every outcome it has not seen as "no" naive = {g: seen_positive[group == g].mean() for g in (True, False)} # a careful monitor only counts patients whose 30-day window is closed diff --git a/explainers/fairness-in-ranking.md b/explainers/fairness-in-ranking.md index 9c4d144..4aa8059 100644 --- a/explainers/fairness-in-ranking.md +++ b/explainers/fairness-in-ranking.md @@ -87,7 +87,7 @@ def report(score, name): print(f"{name}\n female share of the top {K}: {female[top].mean():.3f} (pool {female.mean():.3f})" f"\n exposure ratio female/male, whole list: {whole[female == 1].mean() / whole[female == 0].mean():.3f}" f"\n exposure ratio female/male, top {K} only: {cut[female == 1].mean() / cut[female == 0].mean():.3f}" - f"\n exposure per unit of hire rate, female/male: " + "\n exposure per unit of hire rate, female/male: " f"{(cut[female == 1].mean() / hired[female == 1].mean()) / (cut[female == 0].mean() / hired[female == 0].mean()):.3f}") report(scores(X_biased), "baseline (Gender is a feature)") diff --git a/explainers/feature-importance-by-group.md b/explainers/feature-importance-by-group.md index b22dca5..b3481bc 100644 --- a/explainers/feature-importance-by-group.md +++ b/explainers/feature-importance-by-group.md @@ -46,7 +46,6 @@ Part of the gap is mechanical. Women are a minority of the positive class in thi Run from the repo root (`PYTHONPATH=. python3 explainers/code/feature-importance-by-group.py`; that file is generated from the block below). ```python -import numpy as np from sklearn.inspection import permutation_importance from faircode.benchmark import load_audit # same loader, encoding and split as the benchmark harness from faircode.models import build_model diff --git a/explainers/hiring-pipeline-adverse-impact.md b/explainers/hiring-pipeline-adverse-impact.md index 13c6649..b206a5b 100644 --- a/explainers/hiring-pipeline-adverse-impact.md +++ b/explainers/hiring-pipeline-adverse-impact.md @@ -64,7 +64,6 @@ alive = np.ones(len(df), dtype=bool) def ratio_with_interval(passed, reached, n_boot=500): """Women's selection rate divided by everyone else's, among the people who reached the stage, with a bootstrap interval.""" w, m = reached & (women == 1), reached & (women == 0) - rate = lambda mask, rows: passed[rows][mask[rows]].mean() point = passed[w].mean() / passed[m].mean() idx_w, idx_m = np.flatnonzero(w), np.flatnonzero(m) boots = [passed[rng.choice(idx_w, len(idx_w))].mean() / passed[rng.choice(idx_m, len(idx_m))].mean() for _ in range(n_boot)] diff --git a/explainers/leave-one-group-out-evaluation.md b/explainers/leave-one-group-out-evaluation.md index 0994fb8..9c1707b 100644 --- a/explainers/leave-one-group-out-evaluation.md +++ b/explainers/leave-one-group-out-evaluation.md @@ -50,7 +50,6 @@ The overall reading: on this data a forest generalizes to a held-out group about Run from the repo root (`PYTHONPATH=. python3 explainers/code/leave-one-group-out-evaluation.py`; that file is generated from the block below). ```python -import numpy as np import pandas as pd from faircode.benchmark import load_audit # same loader, encoding and split as the benchmark harness from faircode.models import build_model diff --git a/explainers/monitoring-fairness-in-production.md b/explainers/monitoring-fairness-in-production.md index 8e58cae..acf92fb 100644 --- a/explainers/monitoring-fairness-in-production.md +++ b/explainers/monitoring-fairness-in-production.md @@ -90,7 +90,7 @@ def batches_until_alert(batch, window, drift, drift_at=20, n_batches=60, z=3): return b - drift_at + 1 if b >= drift_at else -1 # -1 = a false alarm before the drift return None -print(f"\nalert rule: the gap over the last W batches is more than 3 standard errors from the launch value") +print("\nalert rule: the gap over the last W batches is more than 3 standard errors from the launch value") print(f"{'batch size':>10} {'window':>7} {'drift':>6} {'median batches to alert':>25} {'never alerted':>14} {'false alarms before the drift':>31}") for batch, window in ((200, 3), (500, 3), (1000, 3), (2000, 3), (1000, 1), (1000, 6)): for drift in (0.03, 0.06): diff --git a/explainers/small-group-shrinkage.md b/explainers/small-group-shrinkage.md index cc429f9..138ead1 100644 --- a/explainers/small-group-shrinkage.md +++ b/explainers/small-group-shrinkage.md @@ -92,7 +92,7 @@ for _ in range(300): small_raw.append(np.sqrt(np.mean((raw[small] - truth[small]) ** 2))); small_shrunk.append(np.sqrt(np.mean((shrunk[small] - truth[small]) ** 2))) estimates.append((raw[worst], shrunk[worst])) -print(f"\nerror (RMSE against the true cell accuracy) from a 600-row audit, averaged over 300 audits") +print("\nerror (RMSE against the true cell accuracy) from a 600-row audit, averaged over 300 audits") print(f" all {len(cells)} cells: raw {np.mean(raw_err):.3f} | shrunk {np.mean(shrunk_err):.3f}") print(f" the smallest {small.sum()} cells: raw {np.mean(small_raw):.3f} | shrunk {np.mean(small_shrunk):.3f}") mean_raw, mean_shrunk = np.mean(estimates, axis=0) diff --git a/explainers/survivorship-bias.md b/explainers/survivorship-bias.md index 6cdeb33..3b72a47 100644 --- a/explainers/survivorship-bias.md +++ b/explainers/survivorship-bias.md @@ -46,7 +46,6 @@ The size of the distortion depends on how hard the gate is. A gate that keeps on Run from the repo root (`PYTHONPATH=. python3 explainers/code/survivorship-bias.py`; that file is generated from the block below). The audit has the outcome for everyone, so we can compare against it; with real survivorship data you cannot, and the check is to ask what the gate was and model it. ```python -import numpy as np from sklearn.linear_model import LogisticRegression from faircode.benchmark import load_audit # same loader, encoding and split as the benchmark harness diff --git a/explainers/utilization-as-a-label.md b/explainers/utilization-as-a-label.md index 90d0f3c..0bd8c60 100644 --- a/explainers/utilization-as-a-label.md +++ b/explainers/utilization-as-a-label.md @@ -58,7 +58,6 @@ That is a finding. It says that for this label the Obermeyer mechanism is not vi Run from the repo root (`PYTHONPATH=. python3 explainers/code/utilization-as-a-label.py`; that file is generated from the block below). `data.df` holds the raw rows, so columns the harness does not encode can still be used to define strata. ```python -import numpy as np import pandas as pd from faircode.benchmark import load_audit # same loader, encoding and split as the benchmark harness