test: guard social-card copy against audit drift

scripts/generate_og_images.py

@@ -193,6 +193,15 @@ def render_card_both_themes(slug, kicker, title, subtitle, force=True):
     return written
 
 
+def home_card_subtitle():
+    return ("Open-source audits across real-world domains, with measurable "
+            "before/after fairness results.")
+
+
+def benchmark_card_subtitle():
+    return "Explore fairness results across seven audits, five mitigation strategies, and three model families."
+
+
 def main(force=False):
     for colors in THEMES.values():
         colors["out_dir"].mkdir(parents=True, exist_ok=True)
@@ -206,9 +215,7 @@ def render(*args, **kwargs):
         "home",
         kicker="Algorithmic bias research · Open source",
         title="The bias is real. So is the fix.",
-        subtitle="Open-source audits across criminal justice, hiring, lending, "
-                 "healthcare, welfare, and tenant screening - with measurable "
-                 "before/after fairness results.",
+        subtitle=home_card_subtitle(),
     )
 
     render(
@@ -223,7 +230,7 @@ def render(*args, **kwargs):
         "benchmark",
         kicker="Fair Code · Benchmark Results",
         title="Benchmark Results Dashboard",
-        subtitle="Explore fairness results across seven audits, five mitigation strategies, and three model families.",
+        subtitle=benchmark_card_subtitle(),
     )
 
     entries = json.loads(DATA_JSON.read_text(encoding="utf-8"))

tests/test_og_card_consistency.py

@@ -0,0 +1,23 @@
+"""Keep the generated benchmark card aligned with the benchmark definitions."""
+
+import importlib
+import pytest
+
+pytest.importorskip("sklearn", reason="benchmark definitions require the benchmark extra")
+pytest.importorskip("fairlearn", reason="benchmark definitions require the benchmark extra")
+
+from faircode.models import MODEL_FAMILIES  # noqa: E402
+from faircode.strategies import STRATEGIES  # noqa: E402
+generate_og_images = importlib.import_module("scripts.generate_og_images")
+
+
+def test_og_card_copy_tracks_audit_and_benchmark_definitions():
+    audits = list(generate_og_images.ROOT.glob("*/audit.yaml"))
+    count_words = {1: "one", 2: "two", 3: "three", 4: "four", 5: "five",
+                   6: "six", 7: "seven", 8: "eight", 9: "nine", 10: "ten"}
+    subtitle = generate_og_images.benchmark_card_subtitle()
+    assert f"{count_words[len(audits)]} audits" in subtitle
+    assert f"{count_words[len(STRATEGIES)]} mitigation strategies" in subtitle
+    assert f"{count_words[len(MODEL_FAMILIES)]} model families" in subtitle
+    assert "real-world domains" in generate_og_images.home_card_subtitle()
+    assert "criminal justice, hiring, lending" not in generate_og_images.home_card_subtitle()