From c8e82b1c4e0e0eea01d8f3453c532f077c446a47 Mon Sep 17 00:00:00 2001 From: minutechreview <141821420+minutechreview@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:24:16 +0300 Subject: [PATCH] Check that each audit's dataset link agrees between the README and its scripts Signed-off-by: minutechreview <141821420+minutechreview@users.noreply.github.com> --- CONTRIBUTING.md | 3 +- Makefile | 3 +- scripts/check_dataset_sources.py | 93 +++++++++++++++++++++++++++++ tests/test_check_dataset_sources.py | 77 ++++++++++++++++++++++++ 4 files changed, 174 insertions(+), 2 deletions(-) create mode 100644 scripts/check_dataset_sources.py create mode 100644 tests/test_check_dataset_sources.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 77560a8..e91be8d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -76,7 +76,7 @@ make fix-explainer-count # correct any stale "N explainers" mention in README/C make key-fold # regenerate the JS case-folding table (assets/profiler-engine.js KEY_FOLD) after a Unicode upgrade make favicons # regenerate favicon.ico, apple-touch-icon.png, icon-{192,512}.png after editing logo.svg make external-links # report dead external links (a report, never a gate); a link that is correct but cannot pass an anonymous check goes in scripts/external-links-ignore.txt as `URL reason`, and the report still counts it -make lint # em-dash-free check + broken internal doc link check + issue-number check + explainer counts + traction figures + CONTRIBUTORS.md vs merged PRs + ruff +make lint # em-dash-free check + broken internal doc link check + issue-number check + explainer counts + traction figures + dataset source links + CONTRIBUTORS.md vs merged PRs + ruff ``` Prerequisites for the full `make check`: Python 3.10 or newer, Node 18 or newer (`key-fold-check` and the JavaScript parity tests run the browser engines under `node`), and the GitHub CLI `gh` signed in (`scripts/check_issue_refs.py` asks GitHub whether an issue or PR number cited in the docs exists; it is skipped offline, and the lint workflow runs it as informational only). @@ -91,6 +91,7 @@ Prerequisites for the full `make check`: Python 3.10 or newer, Node 18 or newer | `check_explainer_count.py` | An "N explainers" figure, or an explainer listing in the README or `llms.txt`, is stale | `make fix-explainer-count` (a missing listing has to be added by hand) | | `check_explainer_structure.py` | An explainer is missing a required section (definition, why it matters, concrete example, detection code, limitations, related concepts, further reading), has no `python` block, has fewer than two Further Reading entries, or lacks the footer line | Add the section (the check accepts the usual heading variants, e.g. "Why This Matters"); pages with a known gap are listed in `KNOWN_EXCEPTIONS` in the script, and an exception is removed when its page is fixed | | `check_traction.py` | Stars, forks, contributors, watching, social reach, countries or audits differ between README, ROADMAP and METRICS, or the README's "Show N completed items" differs from its `- [x]` lines | Edit the figure in the file(s) the message names so they agree; the README's "External Contributors" is one below the total; `--live` (opt-in, needs `gh`, run weekly by `traction-live.yml`) also compares stars, forks, watching and contributors with GitHub | +| `check_dataset_sources.py` | A URL in a comment of an audit's `unfair.py` or `fair.py` is not on that audit's `**Dataset:**` line in the README | Change the link in both places so they match (a script with no link is fine) | | `check_contributors.py` | An entry in CONTRIBUTORS.md has a merged-PR count or first-merged date that differs from `gh pr list --state merged` (counted only up to the PR the snapshot line names), an entry is out of order (PRs, then commits, then first merge), or a merged-PR author has no entry | Edit the entry's `**N merged PRs · M commits · first merged DATE**` line or move it; run it offline with `--prs FILE` (JSON from `gh pr list --state merged --limit 5000 --json number,author,mergedAt`); it is skipped when `gh` is missing | | `key-fold-check` | The JavaScript case-folding table `KEY_FOLD` no longer matches Python | `make key-fold`, then commit `assets/profiler-engine.js` | | A test that runs a README command | Every `faircode ...` line in the README, CONTRIBUTING and SPEC code blocks is checked against the CLI's argument parser; one of them uses a flag or value that no longer exists | Fix the example or the CLI | diff --git a/Makefile b/Makefile index e36e5f4..20c39de 100644 --- a/Makefile +++ b/Makefile @@ -48,13 +48,14 @@ fix-explainer-count: ## Correct stale "N explainers" mentions in README/CONTRIB external-links: ## Report dead external links in the docs and explainers (a report, never a gate; the weekly workflow runs the same script) $(PY) scripts/check_external_links.py -lint: ## Enforce the em-dash-free rule + check for broken doc links + issue numbers in docs (needs gh, skipped offline) + explainer counts and listings + explainer structure + traction figures agreeing across README/ROADMAP/METRICS + CONTRIBUTORS.md matching the merged PRs (needs gh, skipped offline) + ruff (mirrors the lint workflow) +lint: ## Enforce the em-dash-free rule + check for broken doc links + issue numbers in docs (needs gh, skipped offline) + explainer counts and listings + explainer structure + traction figures agreeing across README/ROADMAP/METRICS + dataset source links agreeing between the README and the audit scripts + CONTRIBUTORS.md matching the merged PRs (needs gh, skipped offline) + ruff (mirrors the lint workflow) $(PY) scripts/check_em_dash.py $(PY) scripts/check_broken_links.py $(PY) scripts/check_issue_refs.py $(PY) scripts/check_explainer_count.py $(PY) scripts/check_explainer_structure.py $(PY) scripts/check_traction.py + $(PY) scripts/check_dataset_sources.py $(PY) scripts/check_contributors.py ruff check faircode scripts tests diff --git a/scripts/check_dataset_sources.py b/scripts/check_dataset_sources.py new file mode 100644 index 0000000..86d0027 --- /dev/null +++ b/scripts/check_dataset_sources.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Checks that each audit's dataset source link agrees between README.md and the audit's scripts (#1208). + +The source address of a dataset is written in the README's project block (the `**Dataset:**` line under +`### NN · - ...`) and again in a comment in the audit's `unfair.py` and `fair.py`. When one +copy is edited and the other is not (the NIJ link in #1175 went dead in one place only), they drift. This +reads every URL in a full-line `#` comment of each script and fails, naming the file, when it is not one of +the URLs on that audit's README `**Dataset:**` line. A script with no URL in its comments is not an error. +It reads files only, so it works offline. + + python3 scripts/check_dataset_sources.py + +Exit code: 0 = every script's source link matches the README, 1 = at least one disagreement. +""" +from __future__ import annotations + +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +SCRIPTS = ("unfair.py", "fair.py") + +HEADING = re.compile(r"^### \d+ · (.+?) - ", re.MULTILINE) +DATASET_LINE = re.compile(r"^\*\*Dataset:\*\*.*$", re.MULTILINE) +MARKDOWN_URL = re.compile(r"\]\((https?://[^)\s]+)\)") +COMMENT_URL = re.compile(r"https?://[^\s)>\"']+") + + +def _clean(url: str) -> str: + return url.rstrip(".,;").rstrip("/") + + +def readme_sources(readme_text: str) -> dict[str, set[str]]: + """{audit folder: URLs on its README `**Dataset:**` line} for each numbered project block.""" + sources: dict[str, set[str]] = {} + headings = list(HEADING.finditer(readme_text)) + for i, heading in enumerate(headings): + end = headings[i + 1].start() if i + 1 < len(headings) else len(readme_text) + block = readme_text[heading.end():end] + next_section = re.search(r"^## ", block, re.MULTILINE) + if next_section: + block = block[:next_section.start()] + line = DATASET_LINE.search(block) + urls = MARKDOWN_URL.findall(line.group(0)) if line else [] + sources[heading.group(1).strip()] = {_clean(url) for url in urls} + return sources + + +def script_sources(script_text: str) -> set[str]: + """URLs that appear in the script's full-line `#` comments.""" + urls: set[str] = set() + for line in script_text.splitlines(): + if line.lstrip().startswith("#"): + urls.update(_clean(url) for url in COMMENT_URL.findall(line)) + return urls + + +def disagreements(root: Path | None = None) -> list[str]: + """One message per script comment URL that the README's dataset line for that audit does not have.""" + root = ROOT if root is None else root + sources = readme_sources((root / "README.md").read_text(encoding="utf-8")) + if not sources: + return ["README.md: no '### NN · - ...' project blocks found, so nothing was compared"] + problems = [] + for audit, readme_urls in sources.items(): + folder = root / audit + if not folder.is_dir(): + problems.append(f"README.md: project block '{audit}' has no folder of that name") + continue + for name in SCRIPTS: + path = folder / name + if not path.is_file(): + continue + for url in sorted(script_sources(path.read_text(encoding="utf-8")) - readme_urls): + listed = ", ".join(sorted(readme_urls)) or "no link" + problems.append(f"{audit}/{name}: cites {url}, but the README's Dataset line for {audit} has {listed}") + return problems + + +def main() -> int: + problems = disagreements() + if problems: + print("Dataset source links disagree between README.md and the audit scripts:") + for problem in problems: + print(f" {problem}") + return 1 + print("Dataset source links agree between README.md and the audit scripts.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_check_dataset_sources.py b/tests/test_check_dataset_sources.py new file mode 100644 index 0000000..7e62a1c --- /dev/null +++ b/tests/test_check_dataset_sources.py @@ -0,0 +1,77 @@ +"""scripts/check_dataset_sources.py compares each audit's dataset link in README.md and its scripts (#1208).""" + +import importlib +from pathlib import Path + +check = importlib.import_module("scripts.check_dataset_sources") + +KAGGLE = "https://www.kaggle.com/datasets/ppb00x/credit-risk-customers" +NEW = "https://www.kaggle.com/datasets/someone/credit-risk-moved" + + +def _repo(root: Path, readme_url, unfair_url, fair_url=None) -> Path: + link = f" · [Kaggle source]({readme_url})" if readme_url else "" + (root / "README.md").write_text( + "## Projects\n\n" + "### 01 · German Credit Lending - Lending Bias\n\n" + f"**Dataset:** `credit_customers.csv` - German Credit (1,000 records){link}\n\n" + "### 02 · COMPAS - Criminal Justice Bias\n\n" + "**Dataset:** `compas-scores-raw.csv` - ProPublica's public COMPAS dataset\n\n" + "## Why This Matters\n\n[elsewhere](https://example.org/not-a-dataset)\n", + encoding="utf-8", + ) + for name, url in (("unfair.py", unfair_url), ("fair.py", fair_url)): + folder = root / "German Credit Lending" + folder.mkdir(exist_ok=True) + header = f"# Source: German Credit\n# {url}\n" if url else "# Load dataset\n" + (folder / name).write_text(header + "import pandas as pd\n", encoding="utf-8") + (root / "COMPAS").mkdir() + (root / "COMPAS" / "unfair.py").write_text("# 1. Load your uploaded file\n", encoding="utf-8") + return root + + +def test_matching_links_pass(tmp_path): + assert check.disagreements(_repo(tmp_path, KAGGLE, KAGGLE, KAGGLE)) == [] + + +def test_script_without_a_link_passes(tmp_path): + assert check.disagreements(_repo(tmp_path, KAGGLE, KAGGLE, None)) == [] + + +def test_link_changed_in_readme_only_fails(tmp_path): + problems = check.disagreements(_repo(tmp_path, NEW, KAGGLE, KAGGLE)) + assert [p.split(":")[0] for p in problems] == ["German Credit Lending/unfair.py", "German Credit Lending/fair.py"] + assert all(KAGGLE in p and NEW in p for p in problems) + + +def test_link_changed_in_one_script_only_fails(tmp_path): + problems = check.disagreements(_repo(tmp_path, KAGGLE, NEW, KAGGLE)) + assert len(problems) == 1 + assert problems[0].startswith(f"German Credit Lending/unfair.py: cites {NEW}") + + +def test_readme_without_a_link_fails_when_a_script_has_one(tmp_path): + problems = check.disagreements(_repo(tmp_path, None, KAGGLE)) + assert problems == [ + f"German Credit Lending/unfair.py: cites {KAGGLE}, but the README's Dataset line for German Credit Lending has no link" + ] + + +def test_trailing_slash_and_full_stop_are_ignored(tmp_path): + assert check.disagreements(_repo(tmp_path, KAGGLE + "/", KAGGLE + ".")) == [] + + +def test_project_block_without_a_folder_is_reported(tmp_path): + root = _repo(tmp_path, KAGGLE, KAGGLE) + (root / "COMPAS" / "unfair.py").unlink() + (root / "COMPAS").rmdir() + assert check.disagreements(root) == ["README.md: project block 'COMPAS' has no folder of that name"] + + +def test_readme_with_no_project_blocks_fails(tmp_path): + (tmp_path / "README.md").write_text("# Fair-Code\n", encoding="utf-8") + assert len(check.disagreements(tmp_path)) == 1 + + +def test_repository_links_agree(): + assert check.disagreements() == []