From db3094586e28c43eb06412cb4b57170050eddc56 Mon Sep 17 00:00:00 2001 From: propcgamer20-png <276521524+propcgamer20-png@users.noreply.github.com> Date: Wed, 21 Oct 2026 22:45:18 +0530 Subject: [PATCH 1/2] Add traction consistency check, test-set marking in diff_results, Python matrix - scripts/check_traction.py fails when traction figures or the completed-item count disagree across README, ROADMAP and METRICS; wired into make lint (#1097) - diff_results.py marks rows measured on different test sets (#1095) - the console-script check in the -m test only uses the script that belongs to the running interpreter (#1096) - __main__.py pragma so coverage stops reporting 0 percent (#986) - requires-python >=3.9, ruff py39, CI runs the fast suite on 3.9, 3.10, 3.13 (#848) - CONTRIBUTING lists prerequisites and what each failing check means (#989) --- .github/workflows/audits.yml | 33 ++++++++ CHANGELOG.md | 9 +- CONTRIBUTING.md | 19 ++++- Makefile | 3 +- README.md | 5 +- faircode/SPEC.md | 6 +- faircode/__main__.py | 4 +- pyproject.toml | 4 +- scripts/check_traction.py | 160 +++++++++++++++++++++++++++++++++++ scripts/diff_results.py | 62 +++++++++++--- tests/test_check_traction.py | 92 ++++++++++++++++++++ tests/test_cli.py | 8 +- tests/test_diff_results.py | 43 +++++++++- 13 files changed, 425 insertions(+), 23 deletions(-) create mode 100644 scripts/check_traction.py create mode 100644 tests/test_check_traction.py diff --git a/.github/workflows/audits.yml b/.github/workflows/audits.yml index 8fcd66c4..c83f316d 100644 --- a/.github/workflows/audits.yml +++ b/.github/workflows/audits.yml @@ -103,6 +103,39 @@ jobs: python "${{ matrix.audit }}/fair.py" echo "::endgroup::" + python-versions: + # The package declares requires-python >=3.9 (#848); this runs the fast suite on the oldest and + # newest supported interpreters so a change that only works on the version `profiler` uses is + # caught. The mcp extra needs 3.10+, so it is installed only there (its tests skip without it). + needs: changes + if: needs.changes.outputs.code_relevant == 'true' + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + python-version: ["3.9", "3.10", "3.13"] + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-python@v7 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + + - uses: actions/setup-node@v7 + with: + node-version: '20' + + - name: Install faircode + pytest + run: | + python -m pip install --upgrade pip + pip install -e ".[excel,parquet,proxy]" pytest + if python -c "import sys; sys.exit(0 if sys.version_info >= (3, 10) else 1)"; then pip install -e ".[mcp]"; fi + + - name: Run the fast suite + run: python -m pytest tests/ -q -m "not slow" + profiler: # Test + smoke-test the faircode dataset profiler (the diagnostic tool). # Not a required status check - safe to skip when nothing test-relevant diff --git a/CHANGELOG.md b/CHANGELOG.md index bcbb65ef..ce934af6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,10 @@ All notable changes to Fair Code are documented here, newest first. ### Added +- `scripts/check_traction.py` (#1097), run by `make lint`: fails, naming the files and values, when the stars, forks, contributors, watching, social reach, countries or audits figures differ between the README, ROADMAP and METRICS, or when the README's "Show N completed items" differs from its `- [x]` lines. It compares the files with each other, so it works offline. +- `scripts/diff_results.py` marks rows measured on different test sets (#1095): each row has a `sample` field (`same`, `different` or `unknown`, with `sample_sizes` when different), different rows are marked `~` in the text output and counted in `--summary`. +- CONTRIBUTING.md lists the prerequisites for `make check` and a "Checks you may meet" table saying what each failing check means and how to fix it (#989). +- CI runs the fast suite on Python 3.9, 3.10 and 3.13 (#848). - Fourteen more explainers, each run against a real audit with the benchmark's current `results/` rows as the baseline where one exists: [Adversarial Debiasing](explainers/adversarial-debiasing.md), [Fairness Regularization](explainers/fairness-regularization.md) and [Fair Representation Learning](explainers/fair-representation-learning.md) (close #1083); [Measurement Bias](explainers/measurement-bias.md), [Aggregation Bias](explainers/aggregation-bias.md) and [Survivorship Bias](explainers/survivorship-bias.md) (close #1084); [Disaggregated Evaluation](explainers/disaggregated-evaluation.md), [Leave-One-Group-Out Evaluation](explainers/leave-one-group-out-evaluation.md) and [Choosing a Fairness Metric](explainers/choosing-a-fairness-metric.md) (close #1085); [Model Cards](explainers/model-cards.md), [Datasheets for Datasets](explainers/datasheets-for-datasets.md) and [Algorithmic Impact Assessments](explainers/algorithmic-impact-assessment.md) (close #1086); [Inferring Missing Protected Attributes](explainers/inferring-protected-attributes.md) and [Fairness Without Demographics](explainers/fairness-without-demographics.md) (close #1087). The explainer count goes from 68 to 82, with generated pages, OG images, sitemap entries, `llms.txt`, `llms-full.txt` and the MCP mirror. - `faircode.benchmark.load_audit(folder, strategy=None, random_state=None)` returns an `AuditData` prepared by the same `prepare_audit()` that `run_audit` uses, so explainers, notebooks and tests reproduce `results/` rows without copying the loader, encoding and split (closes #1078, SPEC section 11). - Browser tests (Chromium) that a 20,000-row file with about 18,000 distinct labels keeps the profile and compare flag lists at 51 items or fewer (closes #1080). @@ -40,7 +44,7 @@ All notable changes to Fair Code are documented here, newest first. ### Changed -- The no-op `frozen-files.yml` workflow is removed; the repo ruleset no longer requires it (closes #1061). +- `requires-python` is now `>=3.9` and - The no-op `frozen-files.yml` workflow is removed; the repo ruleset no longer requires it (closes #1061). - The profile scaling test compares growth when the number of distinct labels quadruples instead of a wall-clock limit, so it no longer fails on a slow CI runner. - Per-group profile flags past 10 of the same kind in one dimension (or intersection) collapse into one summary flag, and the terminal, HTML and web views list at most 50 flags and then "and N more"; the full per-group detail stays in `--json` and the CSV (closes #1043, SPEC section 6). - `compare` summarises more than 10 appeared or disappeared groups in one dimension as one flag, like the profile side (closes #1052, SPEC section 8). @@ -50,7 +54,8 @@ All notable changes to Fair Code are documented here, newest first. ### Fixed -- Profiling a file with a high-cardinality geography column was quadratic in `_build_flags` (a list membership test per group); it now uses a set, in Python and JavaScript. 60,000 rows with 45,000 zip codes profile in about 1.6 s instead of about 10 s (closes #1042). +- `test_python_dash_m_faircode_runs_the_cli` only runs a `faircode` console script that sits next to the running interpreter, so a stray script on `PATH` installed for another Python no longer fails it (#1096). +- `faircode/__main__.py` is marked `# pragma: no cover` with a pointer to the subprocess test, so the coverage figure no longer reports it as 0 percent - Profiling a file with a high-cardinality geography column was quadratic in `_build_flags` (a list membership test per group); it now uses a set, in Python and JavaScript. 60,000 rows with 45,000 zip codes profile in about 1.6 s instead of about 10 s (closes #1042). - The web engine labelled XLSX date cells with their Excel serial number (`43831`); they now read `2020-01-01 00:00:00` like pandas, computed from the serial so the label does not depend on the time zone, and time-only cells read `HH:MM:SS` (closes #1039). - `get_xlsx_sheet_info` now closes the workbook it opens to list sheet names, rather than leaking the file handle. The unclosed handle raised a `ResourceWarning` on garbage collection and, on Windows, kept the `.xlsx` locked against deletion or overwrite ([#1051](https://github.com/yakew7/Fair-Code/issues/1051)). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0d37d779..606d7974 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -74,9 +74,26 @@ make build-explainers # regenerate explainer pages, sitemap, and OG images (dar make fix-explainer-count # correct any stale "N explainers" mention in README/CONTRIBUTORS/METRICS/ROADMAP after adding/removing an explainer make key-fold # regenerate the JS case-folding table (assets/profiler-engine.js KEY_FOLD) after a Unicode upgrade make favicons # regenerate favicon.ico, apple-touch-icon.png, icon-{192,512}.png after editing logo.svg -make lint # em-dash-free check + broken internal doc link check + ruff (unused imports, undefined names) +make lint # em-dash-free check + broken internal doc link check + issue-number check + explainer counts + traction figures + ruff ``` +Prerequisites for the full `make check`: Python 3.9 or newer (3.10+ for the `mcp` extra), Node 18 or newer (`key-fold-check` and the JavaScript parity tests run the browser engines under `node`), and the GitHub CLI `gh` signed in (`scripts/check_issue_refs.py` asks GitHub whether an issue or PR number cited in the docs exists; it is skipped offline, and the lint workflow runs it as informational only). + +### Checks you may meet + +| If this fails | What it means | How to fix it | +|---|---|---| +| `check_em_dash.py` | A tracked file contains an em dash | Replace it with a hyphen, comma or colon | +| `check_broken_links.py` | A relative link in the docs points at a missing file or heading | Fix the path or heading | +| `check_issue_refs.py` | A `#NNN` in ROADMAP, CHANGELOG, CONTRIBUTORS or METRICS is not a real issue or PR | Correct the number | +| `check_explainer_count.py` | An "N explainers" figure, or an explainer listing in the README or `llms.txt`, is stale | `make fix-explainer-count` (a missing listing has to be added by hand) | +| `check_traction.py` | Stars, forks, contributors, watching, social reach, countries or audits differ between README, ROADMAP and METRICS, or the README's "Show N completed items" differs from its `- [x]` lines | Edit the figure in the file(s) the message names so they agree; the README's "External Contributors" is one below the total | +| `key-fold-check` | The JavaScript case-folding table `KEY_FOLD` no longer matches Python | `make key-fold`, then commit `assets/profiler-engine.js` | +| A test that runs a README command | Every `faircode ...` line in the README, CONTRIBUTING and SPEC code blocks is checked against the CLI's argument parser; one of them uses a flag or value that no longer exists | Fix the example or the CLI | +| `check_generated_files_current.py` (CI) | Generated explainer pages, sitemap, OG images or `llms-full.txt` are out of date | `make build-explainers`, then commit the result | + +The tests run in any pytest import mode, including `--import-mode=importlib`: `pyproject.toml` puts the repo root and `tests/` on `pythonpath`. The helpers shared by the JavaScript parity tests (`tests/test_js_parity_*.py`) live in `tests/js_helpers.py`. + Optionally install the git hooks so the checks run automatically: ```bash diff --git a/Makefile b/Makefile index bc28aba7..7484f8fe 100644 --- a/Makefile +++ b/Makefile @@ -45,11 +45,12 @@ favicons: ## Regenerate favicon.ico/PNGs and apple-touch-icon.png from logo.svg fix-explainer-count: ## Correct stale "N explainers" mentions in README/CONTRIBUTORS/METRICS/ROADMAP $(PY) scripts/check_explainer_count.py --fix -lint: ## Enforce the em-dash-free rule + check for broken doc links + issue numbers in docs (needs gh, skipped offline) + explainer counts and listings + ruff (mirrors the lint workflow) +lint: ## Enforce the em-dash-free rule + check for broken doc links + issue numbers in docs (needs gh, skipped offline) + explainer counts and listings + traction figures agreeing across README/ROADMAP/METRICS + ruff (mirrors the lint workflow) $(PY) scripts/check_em_dash.py $(PY) scripts/check_broken_links.py $(PY) scripts/check_issue_refs.py $(PY) scripts/check_explainer_count.py + $(PY) scripts/check_traction.py ruff check faircode scripts tests audits: ## Run every audit's unfair.py/fair.py + the CLI smoke tests (mirrors audits.yml's required run-audits job, plus profiler/benchmark-harness's CLI steps) diff --git a/README.md b/README.md index bded9b22..a36dd41a 100644 --- a/README.md +++ b/README.md @@ -1022,6 +1022,9 @@ orientation" error pointing you at `split`, which round-trips unambiguously. `.xlsx` needs the optional `excel` extra, `.parquet` needs the optional `parquet` extra. +Python 3.9 or newer is required (Pillow 11.1, a core dependency, does not support 3.8), and the `mcp` extra +needs Python 3.10 or newer. CI runs the suite on 3.9, 3.10, 3.12 and 3.13. + ```bash pip install -e . # installs the faircode console script pip install -e ".[excel]" # + .xlsx support (openpyxl) @@ -1255,7 +1258,7 @@ pip install -r requirements-lock.txt **For new audits and reruns:** Update `results/` as usual. Do not regenerate `paper/results-frozen/` or move the `v1.0-paper` tag as part of an ordinary audit. Run `scripts/freeze_paper_results.py` only when deliberately creating a new reference snapshot; it replaces the existing frozen snapshot and mirrors its CSV files into `faircode/_results_frozen/`. -To see how far `results/` has moved from that snapshot, run `python3 scripts/diff_results.py`. It joins the two tables per audit, strategy, model, protected attribute and metric and prints the per-row change, flagging rows that moved by more than `--threshold` (default 0.01) or exist on one side only (`--json`, `--all`, `--kind` and `--fail-on-change` are also available; `--audit NAME` limits it to one or more audits and `--summary` prints one line per audit; it only reads). +To see how far `results/` has moved from that snapshot, run `python3 scripts/diff_results.py`. It joins the two tables per audit, strategy, model, protected attribute and metric and prints the per-row change, flagging rows that moved by more than `--threshold` (default 0.01) or exist on one side only (`--json`, `--all`, `--kind` and `--fail-on-change` are also available; `--audit NAME` limits it to one or more audits and `--summary` prints one line per audit; each row has a `sample` field, `same`, `different` or `unknown`, and rows whose group sizes differ between the two files (so they were scored on different test rows) are marked `~` and counted; it only reads). To deliberately create a new reference snapshot, run: diff --git a/faircode/SPEC.md b/faircode/SPEC.md index 76e94b5a..1bf8a6a0 100644 --- a/faircode/SPEC.md +++ b/faircode/SPEC.md @@ -606,7 +606,11 @@ value empty on one side only), `unchanged`, `only_current` or `only_reference`. side only are always flagged. `--json` prints the structured diff, `--all` lists unchanged rows, and `--fail-on-change` exits 1 when anything is flagged. `--audit NAME` (repeatable) restricts the join to those audits, and `--summary` prints one line per audit and kind instead (rows compared, -rows flagged, largest absolute delta); both work with `--json`. It only reads both directories. +rows flagged, rows on different test sets, largest absolute delta); both work with `--json`. Each row also +has a `sample` field: `same` when both files report the same group sizes (fairness: `n_disadvantaged` and +`n_advantaged`; performance: `n`), `different` when they differ (the two runs were scored on different test +rows, so the delta mixes a model change with a change in who was tested; `sample_sizes` holds both), and +`unknown` when a size is missing or the row is on one side only. It only reads both directories. ### Loading an audit in code diff --git a/faircode/__main__.py b/faircode/__main__.py index e1506145..ae7b14e4 100644 --- a/faircode/__main__.py +++ b/faircode/__main__.py @@ -2,5 +2,7 @@ from .cli import main -if __name__ == "__main__": +# Covered by test_python_dash_m_faircode_runs_the_cli (#966), which runs it in a subprocess that +# coverage does not measure, hence the pragma (#986). +if __name__ == "__main__": # pragma: no cover raise SystemExit(main()) diff --git a/pyproject.toml b/pyproject.toml index ee0493fd..90adac3e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "faircode" version = "2.3.0" description = "Diagnostic profiler that audits a tabular dataset for demographic representation." readme = "README.md" -requires-python = ">=3.8" +requires-python = ">=3.9" license = { text = "MIT" } authors = [{ name = "Yash Kewlani" }] keywords = ["fairness", "bias", "dataset", "profiling", "demographics", "audit"] @@ -54,7 +54,7 @@ markers = [ ] [tool.ruff] -target-version = "py38" +target-version = "py39" [tool.ruff.lint] # Real bugs only (unused imports, undefined names, redefinitions) - Pyflakes' diff --git a/scripts/check_traction.py b/scripts/check_traction.py new file mode 100644 index 00000000..83cb8bdd --- /dev/null +++ b/scripts/check_traction.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Checks that the traction figures agree across README.md, ROADMAP.md and METRICS.md (#1097). + +The same numbers (stars, forks, contributors, watching, social reach, countries, audits) are typed by +hand in the README's Traction table, the ROADMAP's "Current traction" row, METRICS.md's badges and the +newest row of its weekly table. When one is edited without the others they drift. This reads each +figure from each place and fails, naming the files and the values, when two of them disagree. It also +checks that the README's "Show N completed items" count equals the number of `- [x]` lines inside that +section. It compares the files with each other, never with GitHub, so it works offline. The explainer +count has its own checker, scripts/check_explainer_count.py. + + python3 scripts/check_traction.py + +Exit code: 0 = everything agrees, 1 = at least one disagreement or a figure that could not be found. + +The README table calls its contributor figure "External Contributors", which excludes the repository +owner, so it is expected to be exactly one lower than the total used in ROADMAP.md and METRICS.md. +""" +from __future__ import annotations + +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent + +FIGURES = ("stars", "contributors", "forks", "watching", "social", "countries", "audits") + +# README row label -> figure; the contributors row is offset by the owner (see the module docstring). +README_ROWS = { + "GitHub Stars": "stars", + "External Contributors": "contributors", + "Forks": "forks", + "Watching": "watching", + "Combined Social Reach (Instagram + LinkedIn)": "social", + "Countries Reached (Website Visitors)": "countries", + "Code Audits Published": "audits", +} +README_OFFSETS = {"contributors": -1} + +ROADMAP_HEADER = "| Stars | Contributors | Forks | Watching | Social Reach | Countries | Audits |" +ROADMAP_COLUMNS = ("stars", "contributors", "forks", "watching", "social", "countries", "audits") + +METRICS_BADGES = {"Stars": "stars", "Contributors": "contributors", "Forks": "forks", + "Watching": "watching", "Countries": "countries"} +# | Week | Stars | Forks | Watching | Contributors | Social Views | Countries | Issues Closed | Code Audits | +METRICS_ROW_COLUMNS = {1: "stars", 2: "forks", 3: "watching", 4: "contributors", 5: "social", + 6: "countries", 8: "audits"} + +# Observations are (figure, where, value), value being the number as written, e.g. "30K+" or "51". +Observation = tuple[str, str, str] + + +def _clean(cell: str) -> str: + """A table cell without markdown noise or a trailing " total".""" + return re.sub(r"\s+total$", "", cell.replace("*", "").replace("`", "").strip()) + + +def _number(value: str) -> int | None: + return int(value) if value.isdigit() else None + + +def readme_observations(text: str) -> tuple[list[Observation], list[str]]: + found, problems = [], [] + for label, figure in README_ROWS.items(): + m = re.search(r"^\|\s*" + re.escape(label) + r"\s*\|\s*([^|\n]+?)\s*\|", text, re.M) + if not m: + problems.append(f"README.md: no Traction table row for {label!r}") + continue + value = _clean(m.group(1)) + offset = README_OFFSETS.get(figure) + number = _number(value) + if offset and number is not None: + value = str(number - offset) + found.append((figure, f"README.md ({label})", value)) + return found, problems + + +def roadmap_observations(text: str) -> tuple[list[Observation], list[str]]: + lines = text.splitlines() + for i, line in enumerate(lines): + if line.startswith(ROADMAP_HEADER): + for row in lines[i + 1:i + 4]: + if row.startswith("|") and re.search(r"\d", row): + cells = [_clean(c) for c in row.strip().strip("|").split("|")] + return [(fig, f"ROADMAP.md (Current traction, {fig})", cells[idx]) + for idx, fig in enumerate(ROADMAP_COLUMNS) if idx < len(cells)], [] + break + return [], ["ROADMAP.md: no 'Current traction' table found"] + + +def metrics_observations(text: str) -> tuple[list[Observation], list[str]]: + found, problems = [], [] + for name, figure in METRICS_BADGES.items(): + m = re.search(r"img\.shields\.io/badge/" + name + r"-([^-?]+)-", text) + if m: + found.append((figure, f"METRICS.md (badge {name})", _clean(m.group(1).replace("%2B", "+")))) + else: + problems.append(f"METRICS.md: no {name} badge") + rows = [ln for ln in text.splitlines() if re.match(r"\|\s*\d{4}-W\d+", ln)] + if not rows: + problems.append("METRICS.md: no weekly row found") + else: + row = rows[-1] + cells = [_clean(c) for c in row.strip().strip("|").split("|")] + week = cells[0] + for idx, figure in METRICS_ROW_COLUMNS.items(): + if idx < len(cells) and cells[idx] not in ("-", ""): + found.append((figure, f"METRICS.md (latest weekly row, {week})", cells[idx])) + return found, problems + + +def completed_items(text: str) -> tuple[int | None, int | None]: + """(claimed, counted) for the README's "Show N completed items" section; None if not found.""" + m = re.search(r"Show (\d+) completed items", text) + if not m: + return None, None + end = text.find("", m.end()) + section = text[m.end():end if end != -1 else len(text)] + return int(m.group(1)), len(re.findall(r"^- \[x\]", section, re.M)) + + +def check(root: Path = ROOT) -> list[str]: + """Every disagreement or missing figure, as one message each (empty when all agree).""" + texts = {} + for name in ("README.md", "ROADMAP.md", "METRICS.md"): + path = root / name + texts[name] = path.read_text(encoding="utf-8") if path.is_file() else "" + observations: list[Observation] = [] + problems: list[str] = [] + for found, missing in (readme_observations(texts["README.md"]), roadmap_observations(texts["ROADMAP.md"]), + metrics_observations(texts["METRICS.md"])): + observations += found + problems += missing + for figure in FIGURES: + seen = [(where, value) for fig, where, value in observations if fig == figure] + if len({value for _, value in seen}) > 1: + detail = "; ".join(f"{where} says {value}" for where, value in seen) + problems.append(f"{figure} disagree: {detail}") + claimed, counted = completed_items(texts["README.md"]) + if claimed is None: + problems.append("README.md: no 'Show N completed items' summary found") + elif claimed != counted: + problems.append(f"README.md: 'Show {claimed} completed items' but the section has {counted} '- [x]' lines") + return problems + + +def main() -> int: + problems = check() + if problems: + print("Traction figures are out of step:", file=sys.stderr) + for problem in problems: + print(f" - {problem}", file=sys.stderr) + return 1 + print("Traction figures agree across README.md, ROADMAP.md and METRICS.md.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/diff_results.py b/scripts/diff_results.py index 18e8f970..79ea62da 100644 --- a/scripts/diff_results.py +++ b/scripts/diff_results.py @@ -13,6 +13,12 @@ python3 scripts/diff_results.py --audit COMPAS # only these audits (repeatable) python3 scripts/diff_results.py --summary # one line per audit and kind +Each row also carries a `sample` field: `same` when both sides report the same group sizes +(fairness: n_disadvantaged and n_advantaged; performance: n), `different` when they differ (the two +runs were scored on different test rows, so the delta mixes a model change with a change in who was +tested; the sizes are in `sample_sizes`), and `unknown` when a side has no size columns or the row is +on one side only. `different` rows are marked `~` in the text output and counted in `--summary`. + Join keys: fairness = audit, strategy, model, protected_attribute, metric; performance = audit, strategy, model, metric. A row present on one side only is reported as `only_current` or `only_reference` (a new audit, strategy or metric, or one that was dropped) and always counts as @@ -35,28 +41,56 @@ REFERENCE_DIR = REPO_ROOT / "paper" / "results-frozen" DEFAULT_THRESHOLD = 0.01 +SIZE_COLUMNS = { + "fairness": ["n_disadvantaged", "n_advantaged"], + "performance": ["n"], +} + KINDS = { "fairness": ("results_fairness.csv", ["audit", "strategy", "model", "protected_attribute", "metric"]), "performance": ("results_performance.csv", ["audit", "strategy", "model", "metric"]), } +def _sizes(row, size_columns): + """The row's group sizes as a list of ints, or None if a size column is missing or empty.""" + values = [] + for column in size_columns: + if column not in row.index or pd.isna(row[column]): + return None + values.append(int(row[column])) + return values + + def diff_table(current: pd.DataFrame, reference: pd.DataFrame, keys: list[str], - threshold: float = DEFAULT_THRESHOLD) -> list[dict]: - """One dict per row on either side: the key columns plus status, reference, current, delta.""" + threshold: float = DEFAULT_THRESHOLD, size_columns: list[str] | None = None) -> list[dict]: + """One dict per row on either side: the key columns plus status, reference, current, delta, sample. + + `size_columns` names the columns holding the test-set sizes; when both sides have them for a row, + `sample` is `same` or `different` (with `sample_sizes` giving both), otherwise `unknown`. + """ def index(df): - out = {} + out, sizes = {}, {} for _, row in df.iterrows(): - out[tuple(str(row[k]) for k in keys)] = row["value"] - return out + key = tuple(str(row[k]) for k in keys) + out[key] = row["value"] + sizes[key] = _sizes(row, size_columns or []) + return out, sizes - cur, ref = index(current), index(reference) + (cur, cur_sizes), (ref, ref_sizes) = index(current), index(reference) rows = [] for key in sorted(set(cur) | set(ref)): entry = dict(zip(keys, key)) c, r = cur.get(key), ref.get(key) entry["reference"] = None if r is None or pd.isna(r) else float(r) entry["current"] = None if c is None or pd.isna(c) else float(c) + c_size, r_size = cur_sizes.get(key), ref_sizes.get(key) + if key not in cur or key not in ref or c_size is None or r_size is None: + entry["sample"] = "unknown" + else: + entry["sample"] = "same" if c_size == r_size else "different" + if c_size != r_size: + entry["sample_sizes"] = {"current": c_size, "reference": r_size} if key not in ref: entry["status"], entry["delta"] = "only_current", None elif key not in cur: @@ -88,7 +122,7 @@ def diff_dirs(current_dir: Path, reference_dir: Path, kinds=("fairness", "perfor wanted = set(audits) current = current[current["audit"].astype(str).isin(wanted)] reference = reference[reference["audit"].astype(str).isin(wanted)] - out[kind] = diff_table(current, reference, keys, threshold) + out[kind] = diff_table(current, reference, keys, threshold, SIZE_COLUMNS[kind]) return out @@ -111,9 +145,10 @@ def summarize(diff: dict) -> dict: per_audit = {} for r in rows: s = per_audit.setdefault(r["audit"], {"audit": r["audit"], "compared": 0, "flagged": 0, - "max_abs_delta": None}) + "max_abs_delta": None, "different_sample": 0}) s["compared"] += 1 s["flagged"] += _flagged(r) + s["different_sample"] += r.get("sample") == "different" if r["delta"] is not None and (s["max_abs_delta"] is None or abs(r["delta"]) > s["max_abs_delta"]): s["max_abs_delta"] = abs(r["delta"]) out[kind] = [per_audit[a] for a in sorted(per_audit)] @@ -126,7 +161,8 @@ def render_summary(summary: dict) -> str: lines.append(f"{kind}:") for s in audits: largest = "n/a" if s["max_abs_delta"] is None else f"{s['max_abs_delta']:.4f}" - lines.append(f" {s['audit']}: {s['compared']} compared, {s['flagged']} flagged, largest |delta| {largest}") + lines.append(f" {s['audit']}: {s['compared']} compared, {s['flagged']} flagged, " + f"{s['different_sample']} on different test sets, largest |delta| {largest}") lines.append("") return "\n".join(lines).rstrip() + "\n" @@ -140,8 +176,10 @@ def render(diff: dict, show_all: bool = False, threshold: float = DEFAULT_THRESH for kind, rows in diff.items(): keys = KINDS[kind][1] flagged = [r for r in rows if _flagged(r)] + different = sum(r.get("sample") == "different" for r in rows) lines.append(f"{kind}: {len(rows)} rows compared, {len(flagged)} flagged " - f"(moved by more than {threshold:g}, or present on one side only)") + f"(moved by more than {threshold:g}, or present on one side only), " + f"{different} measured on different test sets") for r in rows if show_all else flagged: label = " / ".join(r[k] for k in keys) if r["status"] == "only_current": @@ -155,6 +193,10 @@ def render(diff: dict, show_all: bool = False, threshold: float = DEFAULT_THRESH delta = "n/a" if r["delta"] is None else f"{r['delta']:+.4f}" detail = f"{ref} -> {cur} ({delta})" mark = "!" if _flagged(r) else " " + if r.get("sample") == "different": + sizes = r["sample_sizes"] + detail += f" [different test set: reference {sizes['reference']}, current {sizes['current']}]" + mark = "~" if mark == " " else mark lines.append(f" {mark} {label}: {detail}") lines.append("") return "\n".join(lines).rstrip() + "\n" diff --git a/tests/test_check_traction.py b/tests/test_check_traction.py new file mode 100644 index 00000000..77c9c307 --- /dev/null +++ b/tests/test_check_traction.py @@ -0,0 +1,92 @@ +"""scripts/check_traction.py keeps the traction figures in step across README, ROADMAP and METRICS (#1097).""" +import importlib +from pathlib import Path + +check = importlib.import_module("scripts.check_traction") + +README = """| Metric | Count | +|--------|------:| +| GitHub Stars | 51 | +| External Contributors | 53 | +| Forks | 63 | +| Watching | 8 | +| Combined Social Reach (Instagram + LinkedIn) | 30K+ | +| Countries Reached (Website Visitors) | 20 | +| Code Audits Published | 7 | + +
+Show 2 completed items + +- [x] One +- [x] Two +- [ ] Not done + +
+""" +ROADMAP = """| Stars | Contributors | Forks | Watching | Social Reach | Countries | Audits | Explainers | CI | +|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:| +| 51 | 54 | 63 | 8 | 30K+ | 20 | 7 | 82 | ok | +""" +METRICS = """![Stars](https://img.shields.io/badge/Stars-51-brightgreen?style=flat-square) +![Contributors](https://img.shields.io/badge/Contributors-54-blue?style=flat-square) +![Forks](https://img.shields.io/badge/Forks-63-orange?style=flat-square) +![Watching](https://img.shields.io/badge/Watching-8-yellow?style=flat-square) +![Countries](https://img.shields.io/badge/Countries-20-informational?style=flat-square) + +| Week | Stars | Forks | Watching | Contributors | Social Views | Countries | Issues Closed | Code Audits | +|------|------:|------:|---------:|-------------:|-------------:|----------:|--------------:|------------:| +| 2026-W42 | 50 | 58 | 8 | 53 | 30K+ total | 20 | 123 | 7 total | +| 2026-W43 | 51 | 63 | 8 | 54 | 30K+ total | 20 | 28 | 7 total | +""" + + +def _site(root: Path, readme=README, roadmap=ROADMAP, metrics=METRICS) -> Path: + (root / "README.md").write_text(readme, encoding="utf-8") + (root / "ROADMAP.md").write_text(roadmap, encoding="utf-8") + (root / "METRICS.md").write_text(metrics, encoding="utf-8") + return root + + +def test_matching_figures_pass(tmp_path): + assert check.check(_site(tmp_path)) == [] + + +def test_a_changed_traction_figure_names_every_place_that_disagrees(tmp_path): + problems = check.check(_site(tmp_path, roadmap=ROADMAP.replace("| 63 |", "| 58 |"))) + assert len(problems) == 1 + assert problems[0].startswith("forks disagree") + assert "ROADMAP.md" in problems[0] and "says 58" in problems[0] + assert "README.md (Forks) says 63" in problems[0] and "METRICS.md (badge Forks) says 63" in problems[0] + + +def test_only_the_newest_weekly_row_is_compared(tmp_path): + # W42 says 50 stars; that is history, not a disagreement. + assert check.check(_site(tmp_path)) == [] + stale = METRICS.replace("| 2026-W43 | 51 |", "| 2026-W43 | 52 |") + assert any(p.startswith("stars disagree") and "W43" in p for p in check.check(_site(tmp_path, metrics=stale))) + + +def test_readme_external_contributors_is_expected_one_below_the_total(tmp_path): + assert check.check(_site(tmp_path)) == [] + problems = check.check(_site(tmp_path, readme=README.replace("| 53 |", "| 54 |"))) + assert any(p.startswith("contributors disagree") for p in problems) + + +def test_social_reach_text_is_compared_too(tmp_path): + problems = check.check(_site(tmp_path, readme=README.replace("30K+", "27K+"))) + assert any(p.startswith("social disagree") for p in problems) + + +def test_completed_item_count_must_match_the_checked_lines(tmp_path): + assert check.completed_items(README) == (2, 2) + problems = check.check(_site(tmp_path, readme=README.replace("Show 2", "Show 3"))) + assert problems == ["README.md: 'Show 3 completed items' but the section has 2 '- [x]' lines"] + + +def test_missing_rows_are_reported_not_ignored(tmp_path): + problems = check.check(_site(tmp_path, roadmap="no table here\n")) + assert "ROADMAP.md: no 'Current traction' table found" in problems + + +def test_the_real_repo_agrees(): + assert check.check() == [] diff --git a/tests/test_cli.py b/tests/test_cli.py index 78f0548b..0e8b8edd 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1472,10 +1472,16 @@ def test_python_dash_m_faircode_runs_the_cli(): assert json.loads(done.stdout)["n_rows"] > 0 bad = subprocess.run([sys.executable, "-m", "faircode", "profile", "--no-such-flag"], capture_output=True, text=True, cwd=root) assert bad.returncode == 2 + # #1096: only the console script that belongs to this interpreter counts; a stray `faircode` on PATH + # may be installed for another Python and cannot import this checkout. + import os import shutil - script = shutil.which("faircode") + exe_dir = Path(sys.executable).parent # bin/ in a venv, the install root (with Scripts/) for a Windows system Python + script = shutil.which("faircode", path=os.pathsep.join([str(exe_dir), str(exe_dir / "Scripts")])) if script: # the installed console script behaves the same assert subprocess.run([script, "--version"], capture_output=True, text=True).returncode == 0 + else: + print(f"no faircode console script next to {sys.executable}; skipping the console-script check") @pytest.mark.parametrize("flags,message", [ diff --git a/tests/test_diff_results.py b/tests/test_diff_results.py index 2faf9f33..62620293 100644 --- a/tests/test_diff_results.py +++ b/tests/test_diff_results.py @@ -115,16 +115,53 @@ def test_summary_counts_rows_and_largest_delta_per_audit(tmp_path, capsys): assert dr.main([*args, "--summary", "--json"]) == 0 summary = json.loads(capsys.readouterr().out)["fairness"] assert [s["audit"] for s in summary] == ["a", "b"] - assert summary[0] == {"audit": "a", "compared": 1, "flagged": 0, "max_abs_delta": 0.0} + assert summary[0] == {"audit": "a", "compared": 1, "flagged": 0, "max_abs_delta": 0.0, "different_sample": 0} assert (summary[1]["compared"], summary[1]["flagged"]) == (2, 2) assert summary[1]["max_abs_delta"] == pytest.approx(0.2) assert dr.main([*args, "--summary", "--audit", "b"]) == 0 out = capsys.readouterr().out - assert "b: 2 compared, 2 flagged, largest |delta| 0.2000" in out + assert "b: 2 compared, 2 flagged, 0 on different test sets, largest |delta| 0.2000" in out assert "a:" not in out def test_summary_largest_delta_is_none_without_numeric_deltas(): diff = {"fairness": [{"audit": "a", "status": "only_current", "delta": None}]} - assert dr.summarize(diff) == {"fairness": [{"audit": "a", "compared": 1, "flagged": 1, "max_abs_delta": None}]} + assert dr.summarize(diff) == {"fairness": [{"audit": "a", "compared": 1, "flagged": 1, "max_abs_delta": None, "different_sample": 0}]} + + +SIZE_FAIR_COLS = FAIR_COLS + ["n_disadvantaged", "n_advantaged"] + + +def test_rows_measured_on_different_test_sets_are_marked(): + # #1095: same value change, but the second row was scored on different people. + ref = pd.DataFrame([["a", "b", "lr", "sex", "dp", 0.10, 100, 200], ["a", "b", "lr", "sex", "eo", 0.10, 100, 200]], + columns=SIZE_FAIR_COLS) + cur = pd.DataFrame([["a", "b", "lr", "sex", "dp", 0.20, 100, 200], ["a", "b", "lr", "sex", "eo", 0.20, 90, 210]], + columns=SIZE_FAIR_COLS) + rows = {r["metric"]: r for r in dr.diff_table(cur, ref, dr.KINDS["fairness"][1], size_columns=dr.SIZE_COLUMNS["fairness"])} + assert rows["dp"]["sample"] == "same" and "sample_sizes" not in rows["dp"] + assert rows["eo"]["sample"] == "different" + assert rows["eo"]["sample_sizes"] == {"current": [90, 210], "reference": [100, 200]} + + +def test_sample_is_unknown_without_size_columns_or_for_one_sided_rows(): + ref = _fair([["a", "b", "lr", "sex", "dp", 0.1], ["a", "b", "lr", "sex", "gone", 0.3]]) + cur = _fair([["a", "b", "lr", "sex", "dp", 0.2]]) + rows = {r["metric"]: r for r in dr.diff_table(cur, ref, dr.KINDS["fairness"][1], size_columns=dr.SIZE_COLUMNS["fairness"])} + assert rows["dp"]["sample"] == "unknown" and rows["gone"]["sample"] == "unknown" + + +def test_different_test_sets_show_in_the_text_json_and_summary(tmp_path): + perf_cols = ["audit", "strategy", "model", "metric", "value", "n"] + for name, n in (("ref", 1000), ("cur", 950)): + _write(tmp_path / name, _fair([["a", "b", "lr", "sex", "dp", 0.1]]), + pd.DataFrame([["a", "b", "lr", "accuracy", 0.7, n]], columns=perf_cols)) + diff = dr.diff_dirs(tmp_path / "cur", tmp_path / "ref") + assert diff["performance"][0]["sample"] == "different" + assert diff["performance"][0]["sample_sizes"] == {"current": [950], "reference": [1000]} + text = dr.render(diff, show_all=True) + assert "1 measured on different test sets" in text and "[different test set: reference [1000], current [950]]" in text + assert dr.summarize(diff)["performance"][0]["different_sample"] == 1 + assert "1 on different test sets" in dr.render_summary(dr.summarize(diff)) + assert json.loads(json.dumps(diff))["performance"][0]["sample"] == "different" From 85b9b8afbf51c0755bfba259cea86d6c48599cbe Mon Sep 17 00:00:00 2001 From: propcgamer20-png <276521524+propcgamer20-png@users.noreply.github.com> Date: Wed, 21 Oct 2026 23:51:08 +0530 Subject: [PATCH 2/2] Raise requires-python to 3.10 after the 3.9 CI run found real breakages The new 3.9 job failed on code that needs 3.10: X | None evaluated at runtime in scripts/build_explainers.py, Path.write_text(newline=) in a test, and a stdlib-module check in tests/test_declared_dependencies.py. 3.10 and 3.13 pass, and the mcp extra needs 3.10 anyway, so the minimum is now 3.10 as #848 asked ('what actually passes'). Also fixes the CHANGELOG line I garbled earlier. --- .github/workflows/audits.yml | 9 ++++----- CHANGELOG.md | 5 +++-- CONTRIBUTING.md | 2 +- README.md | 4 ++-- pyproject.toml | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/audits.yml b/.github/workflows/audits.yml index c83f316d..78fafb8b 100644 --- a/.github/workflows/audits.yml +++ b/.github/workflows/audits.yml @@ -104,9 +104,9 @@ jobs: echo "::endgroup::" python-versions: - # The package declares requires-python >=3.9 (#848); this runs the fast suite on the oldest and + # The package declares requires-python >=3.10 (#848); this runs the fast suite on the oldest and # newest supported interpreters so a change that only works on the version `profiler` uses is - # caught. The mcp extra needs 3.10+, so it is installed only there (its tests skip without it). + # caught. A 3.9 run found real breakages, which is why the minimum is 3.10. needs: changes if: needs.changes.outputs.code_relevant == 'true' runs-on: ubuntu-latest @@ -114,7 +114,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.13"] + python-version: ["3.10", "3.13"] steps: - uses: actions/checkout@v7 @@ -130,8 +130,7 @@ jobs: - name: Install faircode + pytest run: | python -m pip install --upgrade pip - pip install -e ".[excel,parquet,proxy]" pytest - if python -c "import sys; sys.exit(0 if sys.version_info >= (3, 10) else 1)"; then pip install -e ".[mcp]"; fi + pip install -e ".[excel,parquet,proxy,mcp]" pytest - name: Run the fast suite run: python -m pytest tests/ -q -m "not slow" diff --git a/CHANGELOG.md b/CHANGELOG.md index ce934af6..791cac6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,7 @@ All notable changes to Fair Code are documented here, newest first. - `scripts/check_traction.py` (#1097), run by `make lint`: fails, naming the files and values, when the stars, forks, contributors, watching, social reach, countries or audits figures differ between the README, ROADMAP and METRICS, or when the README's "Show N completed items" differs from its `- [x]` lines. It compares the files with each other, so it works offline. - `scripts/diff_results.py` marks rows measured on different test sets (#1095): each row has a `sample` field (`same`, `different` or `unknown`, with `sample_sizes` when different), different rows are marked `~` in the text output and counted in `--summary`. - CONTRIBUTING.md lists the prerequisites for `make check` and a "Checks you may meet" table saying what each failing check means and how to fix it (#989). -- CI runs the fast suite on Python 3.9, 3.10 and 3.13 (#848). +- CI runs the fast suite on Python 3.10 and 3.13 (#848). - Fourteen more explainers, each run against a real audit with the benchmark's current `results/` rows as the baseline where one exists: [Adversarial Debiasing](explainers/adversarial-debiasing.md), [Fairness Regularization](explainers/fairness-regularization.md) and [Fair Representation Learning](explainers/fair-representation-learning.md) (close #1083); [Measurement Bias](explainers/measurement-bias.md), [Aggregation Bias](explainers/aggregation-bias.md) and [Survivorship Bias](explainers/survivorship-bias.md) (close #1084); [Disaggregated Evaluation](explainers/disaggregated-evaluation.md), [Leave-One-Group-Out Evaluation](explainers/leave-one-group-out-evaluation.md) and [Choosing a Fairness Metric](explainers/choosing-a-fairness-metric.md) (close #1085); [Model Cards](explainers/model-cards.md), [Datasheets for Datasets](explainers/datasheets-for-datasets.md) and [Algorithmic Impact Assessments](explainers/algorithmic-impact-assessment.md) (close #1086); [Inferring Missing Protected Attributes](explainers/inferring-protected-attributes.md) and [Fairness Without Demographics](explainers/fairness-without-demographics.md) (close #1087). The explainer count goes from 68 to 82, with generated pages, OG images, sitemap entries, `llms.txt`, `llms-full.txt` and the MCP mirror. - `faircode.benchmark.load_audit(folder, strategy=None, random_state=None)` returns an `AuditData` prepared by the same `prepare_audit()` that `run_audit` uses, so explainers, notebooks and tests reproduce `results/` rows without copying the loader, encoding and split (closes #1078, SPEC section 11). - Browser tests (Chromium) that a 20,000-row file with about 18,000 distinct labels keeps the profile and compare flag lists at 51 items or fewer (closes #1080). @@ -44,7 +44,8 @@ All notable changes to Fair Code are documented here, newest first. ### Changed -- `requires-python` is now `>=3.9` and - The no-op `frozen-files.yml` workflow is removed; the repo ruleset no longer requires it (closes #1061). +- `requires-python` is now `>=3.10` and ruff targets `py310` (#848). A Python 3.9 CI run found real breakages (`X | None` evaluated at runtime, `Path.write_text(newline=)`), and the `mcp` extra needs 3.10 anyway. +- The no-op `frozen-files.yml` workflow is removed; the repo ruleset no longer requires it (closes #1061). - The profile scaling test compares growth when the number of distinct labels quadruples instead of a wall-clock limit, so it no longer fails on a slow CI runner. - Per-group profile flags past 10 of the same kind in one dimension (or intersection) collapse into one summary flag, and the terminal, HTML and web views list at most 50 flags and then "and N more"; the full per-group detail stays in `--json` and the CSV (closes #1043, SPEC section 6). - `compare` summarises more than 10 appeared or disappeared groups in one dimension as one flag, like the profile side (closes #1052, SPEC section 8). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 606d7974..e86889c4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -77,7 +77,7 @@ make favicons # regenerate favicon.ico, apple-touch-icon.png, icon-{192 make lint # em-dash-free check + broken internal doc link check + issue-number check + explainer counts + traction figures + ruff ``` -Prerequisites for the full `make check`: Python 3.9 or newer (3.10+ for the `mcp` extra), Node 18 or newer (`key-fold-check` and the JavaScript parity tests run the browser engines under `node`), and the GitHub CLI `gh` signed in (`scripts/check_issue_refs.py` asks GitHub whether an issue or PR number cited in the docs exists; it is skipped offline, and the lint workflow runs it as informational only). +Prerequisites for the full `make check`: Python 3.10 or newer, Node 18 or newer (`key-fold-check` and the JavaScript parity tests run the browser engines under `node`), and the GitHub CLI `gh` signed in (`scripts/check_issue_refs.py` asks GitHub whether an issue or PR number cited in the docs exists; it is skipped offline, and the lint workflow runs it as informational only). ### Checks you may meet diff --git a/README.md b/README.md index a36dd41a..5b48ed08 100644 --- a/README.md +++ b/README.md @@ -1022,8 +1022,8 @@ orientation" error pointing you at `split`, which round-trips unambiguously. `.xlsx` needs the optional `excel` extra, `.parquet` needs the optional `parquet` extra. -Python 3.9 or newer is required (Pillow 11.1, a core dependency, does not support 3.8), and the `mcp` extra -needs Python 3.10 or newer. CI runs the suite on 3.9, 3.10, 3.12 and 3.13. +Python 3.10 or newer is required (the code uses 3.10 features such as `X | None` annotations and +`Path.write_text(newline=...)`, and the `mcp` extra needs 3.10 too). CI runs the suite on 3.10, 3.12 and 3.13. ```bash pip install -e . # installs the faircode console script diff --git a/pyproject.toml b/pyproject.toml index 90adac3e..837cc90c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "faircode" version = "2.3.0" description = "Diagnostic profiler that audits a tabular dataset for demographic representation." readme = "README.md" -requires-python = ">=3.9" +requires-python = ">=3.10" license = { text = "MIT" } authors = [{ name = "Yash Kewlani" }] keywords = ["fairness", "bias", "dataset", "profiling", "demographics", "audit"] @@ -54,7 +54,7 @@ markers = [ ] [tool.ruff] -target-version = "py39" +target-version = "py310" [tool.ruff.lint] # Real bugs only (unused imports, undefined names, redefinitions) - Pyflakes'