Docs site: Training section generated from the training folder

.github/workflows/gh-pages.yml

@@ -3,6 +3,10 @@ on:
   push:
     branches:
       - main
+  # Pull requests build the site without deploying it. The build fails when a published
+  # route has no page and no redirect, or when a new route is missing from
+  # website/published-routes.txt (see website/README.md).
+  pull_request:
 
 jobs:
   build-site:
@@ -39,6 +43,7 @@ jobs:
 
   deploy-site:
     needs: build-site
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
     permissions:
       pages: write
       id-token: write

.gitignore

@@ -181,8 +181,8 @@ website/node_modules/
 # Notebook -> Markdown conversion output, generated by scripts/docs/convert_notebooks.py
 docs/notebooks/
 
-# Training pages for the site, generated by scripts/docs/build_training_pages.py
-website/static/training/
+# Training section of the site, generated by scripts/docs/build_training_site.py
+website/generated/
 
 # schema cache written by cookbook check_metadata.py
 website/static/cookbook-files/**/wb-schemas/

scripts/docs/build_training_pages.py

@@ -1,14 +1,12 @@
 #!/usr/bin/env python3
-"""Build the standalone HTML pages of the training series for the docs site.
+"""Document collection data and pages for the training series.
 
-Reads the worked-example documents of session 01 and writes a page that shows each
-document with its metadata, its role in the session, and the chunks that notebook 02
-makes of it, each labelled with the citation the application writes when it uses the
-chunk. The site build runs this script before Docusaurus, and the output lands under
-website/static so that it is served as is.
+Reads the worked-example documents of a session with their roles (assets/documents/roles.json)
+and splits each document into the chunks that notebook 02 makes of it, each labelled with the
+citation the application writes when it uses the chunk. build_training_site.py calls
+viewer_data() to build the Document collection page of the Training section.
 
-    python3 scripts/docs/build_training_pages.py                 # website/static/training/session-01/documents.html
-    python3 scripts/docs/build_training_pages.py --artifact OUT  # body-only page with embedded logos, for a Claude artifact
+    python3 scripts/docs/build_training_pages.py --artifact OUT  # standalone page with embedded logos, for a Claude artifact
 
 No dependencies beyond the standard library: the documents use headings, paragraphs
 and bullet lists only, and the chunking mirrors workshop.split_sections (one chunk per
@@ -26,34 +24,14 @@
 REPO = Path(__file__).resolve().parents[2]
 SESSION = REPO / "training" / "session-01-building-ai-applications"
 DOCS = SESSION / "assets" / "documents"
-SITE_OUT = REPO / "website" / "static" / "training" / "session-01" / "documents.html"
 LOGO = REPO / "website" / "static" / "img" / "program-logo.png"
 LOGO_DARK = REPO / "website" / "static" / "img" / "program-logo-dark.png"
 
-ROLES = [
-    ("evidence", "Relevant evidence",
-     "The four country briefs. Their figures are rounded from the indicator data, and a correct answer to the main question cites them.",
-     ["kenya-digital", "rwanda-digital", "kenya-economy", "rwanda-economy"]),
-    ("definition", "Definition",
-     "A note on the definition of the internet-users indicator and on the comparability of its values across years.",
-     ["internet-indicator"]),
-    ("outdated", "Outdated",
-     "A 2016 summary with figures for 2015 and expectations that later data did not confirm. It outranks the current briefs for a question about the latest figures, and its date in the metadata removes it.",
-     ["regional-2016"]),
-    ("distractor", "Distractors",
-     "Two notes on related topics without the requested indicators. The coffee note is the retrieval failure case; the mobile-money note is a plausible wrong passage.",
-     ["coffee", "mobile-money"]),
-]
-NOTE = {
-    "kenya-digital": "Relevant evidence for internet use and mobile subscriptions in Kenya.",
-    "rwanda-digital": "Relevant evidence for internet use and mobile subscriptions in Rwanda.",
-    "kenya-economy": "Relevant evidence for GDP per capita and growth in Kenya.",
-    "rwanda-economy": "Relevant evidence for GDP per capita and growth in Rwanda.",
-    "internet-indicator": "Definition of the indicator; the source for the question on its metadata.",
-    "regional-2016": "Outdated figures; shows why dates in metadata matter.",
-    "coffee": "Irrelevant to the question; shows a retrieval failure.",
-    "mobile-money": "Related topic without the requested indicators; a plausible distractor.",
-}
+# The role of each document in the session, and the text of the Document collection page.
+ROLES_FILE = DOCS / "roles.json"
+_roles = json.loads(ROLES_FILE.read_text(encoding="utf-8"))
+ROLES = [(g["key"], g["label"], g["description"], [d["id"] for d in g["documents"]]) for g in _roles["groups"]]
+NOTE = {d["id"]: d["note"] for g in _roles["groups"] for d in g["documents"]}
 META_KEYS = ["id", "type", "country", "date", "source", "note"]
 
 
@@ -263,23 +241,37 @@ def build(docs: list[dict], logo_light: str, logo_dark: str, site_links: bool) -
 """
 
 
+def viewer_data(docs_dir: Path) -> dict:
+    """A session's document collection as data, for the Document collection page of the Training section.
+
+    `docs_dir` holds the documents (*.md) and roles.json, which groups them by their role in the session."""
+    roles = json.loads((docs_dir / "roles.json").read_text(encoding="utf-8"))
+    by = {d["metadata"]["id"]: d for d in (read_document(p) for p in sorted(docs_dir.glob("*.md")))}
+    groups = []
+    for g in roles["groups"]:
+        docs = []
+        for entry in g["documents"]:
+            doc = by[entry["id"]]
+            m = doc["metadata"]
+            docs.append({
+                "id": entry["id"], "title": m.get("title", entry["id"]), "year": str(m.get("date", ""))[:4],
+                "note": entry.get("note", ""),
+                "metadata": [[k, m[k]] for k in META_KEYS if k in m],
+                "chunks": [{"citation": c["citation"], "characters": len(c["text"]), "html": render_markdown(c["text"])}
+                           for c in split_sections(doc)],
+            })
+        groups.append({"key": g["key"], "label": g["label"], "description": g.get("description", ""), "documents": docs})
+    return {"groups": groups}
+
+
 def main() -> int:
     ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
-    ap.add_argument("--artifact", type=Path, help="write a body-only page with embedded logos to this path instead of the site page")
+    ap.add_argument("--artifact", type=Path, required=True, help="write a standalone page with embedded logos to this path")
     args = ap.parse_args()
     docs = [read_document(p) for p in sorted(DOCS.glob("*.md"))]
-    if args.artifact:
-        page = build(docs, data_uri(LOGO), data_uri(LOGO_DARK), site_links=True)
-        args.artifact.write_text(page, encoding="utf-8")
-        print(f"wrote {args.artifact} ({len(page):,} bytes)")
-        return 0
-    page = build(docs, "/ai4data/img/program-logo.png", "/ai4data/img/program-logo-dark.png", site_links=True)
-    full = ('<!doctype html>\n<html lang="en">\n<head>\n<meta charset="utf-8">\n<meta name="viewport" content="width=device-width, initial-scale=1">\n'
-            '<meta name="description" content="The eight documents of the worked example in session 01 of the AI for Data training series, with their metadata, their role, and their retrieval chunks.">\n'
-            '<link rel="icon" href="/ai4data/img/favicon.ico">\n' + page.split("\n", 1)[0] + "\n</head>\n<body>\n" + page.split("\n", 1)[1] + "</body>\n</html>\n")
-    SITE_OUT.parent.mkdir(parents=True, exist_ok=True)
-    SITE_OUT.write_text(full, encoding="utf-8")
-    print(f"wrote {SITE_OUT.relative_to(REPO)} ({len(full):,} bytes)")
+    page = build(docs, data_uri(LOGO), data_uri(LOGO_DARK), site_links=True)
+    args.artifact.write_text(page, encoding="utf-8")
+    print(f"wrote {args.artifact} ({len(page):,} bytes)")
     return 0