diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 2cb5036a..41dbc777 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -3,6 +3,10 @@ on: push: branches: - main + # Pull requests build the site without deploying it. The build fails when a published + # route has no page and no redirect, or when a new route is missing from + # website/published-routes.txt (see website/README.md). + pull_request: jobs: build-site: @@ -39,6 +43,7 @@ jobs: deploy-site: needs: build-site + if: github.event_name == 'push' && github.ref == 'refs/heads/main' permissions: pages: write id-token: write diff --git a/.gitignore b/.gitignore index 9fb02e15..48745969 100644 --- a/.gitignore +++ b/.gitignore @@ -181,8 +181,8 @@ website/node_modules/ # Notebook -> Markdown conversion output, generated by scripts/docs/convert_notebooks.py docs/notebooks/ -# Training pages for the site, generated by scripts/docs/build_training_pages.py -website/static/training/ +# Training section of the site, generated by scripts/docs/build_training_site.py +website/generated/ # schema cache written by cookbook check_metadata.py website/static/cookbook-files/**/wb-schemas/ diff --git a/scripts/docs/build_training_pages.py b/scripts/docs/build_training_pages.py index 6c31b1a8..1e6f2f53 100644 --- a/scripts/docs/build_training_pages.py +++ b/scripts/docs/build_training_pages.py @@ -1,14 +1,12 @@ #!/usr/bin/env python3 -"""Build the standalone HTML pages of the training series for the docs site. +"""Document collection data and pages for the training series. -Reads the worked-example documents of session 01 and writes a page that shows each -document with its metadata, its role in the session, and the chunks that notebook 02 -makes of it, each labelled with the citation the application writes when it uses the -chunk. The site build runs this script before Docusaurus, and the output lands under -website/static so that it is served as is. +Reads the worked-example documents of a session with their roles (assets/documents/roles.json) +and splits each document into the chunks that notebook 02 makes of it, each labelled with the +citation the application writes when it uses the chunk. build_training_site.py calls +viewer_data() to build the Document collection page of the Training section. - python3 scripts/docs/build_training_pages.py # website/static/training/session-01/documents.html - python3 scripts/docs/build_training_pages.py --artifact OUT # body-only page with embedded logos, for a Claude artifact + python3 scripts/docs/build_training_pages.py --artifact OUT # standalone page with embedded logos, for a Claude artifact No dependencies beyond the standard library: the documents use headings, paragraphs and bullet lists only, and the chunking mirrors workshop.split_sections (one chunk per @@ -26,34 +24,14 @@ REPO = Path(__file__).resolve().parents[2] SESSION = REPO / "training" / "session-01-building-ai-applications" DOCS = SESSION / "assets" / "documents" -SITE_OUT = REPO / "website" / "static" / "training" / "session-01" / "documents.html" LOGO = REPO / "website" / "static" / "img" / "program-logo.png" LOGO_DARK = REPO / "website" / "static" / "img" / "program-logo-dark.png" -ROLES = [ - ("evidence", "Relevant evidence", - "The four country briefs. Their figures are rounded from the indicator data, and a correct answer to the main question cites them.", - ["kenya-digital", "rwanda-digital", "kenya-economy", "rwanda-economy"]), - ("definition", "Definition", - "A note on the definition of the internet-users indicator and on the comparability of its values across years.", - ["internet-indicator"]), - ("outdated", "Outdated", - "A 2016 summary with figures for 2015 and expectations that later data did not confirm. It outranks the current briefs for a question about the latest figures, and its date in the metadata removes it.", - ["regional-2016"]), - ("distractor", "Distractors", - "Two notes on related topics without the requested indicators. The coffee note is the retrieval failure case; the mobile-money note is a plausible wrong passage.", - ["coffee", "mobile-money"]), -] -NOTE = { - "kenya-digital": "Relevant evidence for internet use and mobile subscriptions in Kenya.", - "rwanda-digital": "Relevant evidence for internet use and mobile subscriptions in Rwanda.", - "kenya-economy": "Relevant evidence for GDP per capita and growth in Kenya.", - "rwanda-economy": "Relevant evidence for GDP per capita and growth in Rwanda.", - "internet-indicator": "Definition of the indicator; the source for the question on its metadata.", - "regional-2016": "Outdated figures; shows why dates in metadata matter.", - "coffee": "Irrelevant to the question; shows a retrieval failure.", - "mobile-money": "Related topic without the requested indicators; a plausible distractor.", -} +# The role of each document in the session, and the text of the Document collection page. +ROLES_FILE = DOCS / "roles.json" +_roles = json.loads(ROLES_FILE.read_text(encoding="utf-8")) +ROLES = [(g["key"], g["label"], g["description"], [d["id"] for d in g["documents"]]) for g in _roles["groups"]] +NOTE = {d["id"]: d["note"] for g in _roles["groups"] for d in g["documents"]} META_KEYS = ["id", "type", "country", "date", "source", "note"] @@ -263,23 +241,37 @@ def build(docs: list[dict], logo_light: str, logo_dark: str, site_links: bool) - """ +def viewer_data(docs_dir: Path) -> dict: + """A session's document collection as data, for the Document collection page of the Training section. + + `docs_dir` holds the documents (*.md) and roles.json, which groups them by their role in the session.""" + roles = json.loads((docs_dir / "roles.json").read_text(encoding="utf-8")) + by = {d["metadata"]["id"]: d for d in (read_document(p) for p in sorted(docs_dir.glob("*.md")))} + groups = [] + for g in roles["groups"]: + docs = [] + for entry in g["documents"]: + doc = by[entry["id"]] + m = doc["metadata"] + docs.append({ + "id": entry["id"], "title": m.get("title", entry["id"]), "year": str(m.get("date", ""))[:4], + "note": entry.get("note", ""), + "metadata": [[k, m[k]] for k in META_KEYS if k in m], + "chunks": [{"citation": c["citation"], "characters": len(c["text"]), "html": render_markdown(c["text"])} + for c in split_sections(doc)], + }) + groups.append({"key": g["key"], "label": g["label"], "description": g.get("description", ""), "documents": docs}) + return {"groups": groups} + + def main() -> int: ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) - ap.add_argument("--artifact", type=Path, help="write a body-only page with embedded logos to this path instead of the site page") + ap.add_argument("--artifact", type=Path, required=True, help="write a standalone page with embedded logos to this path") args = ap.parse_args() docs = [read_document(p) for p in sorted(DOCS.glob("*.md"))] - if args.artifact: - page = build(docs, data_uri(LOGO), data_uri(LOGO_DARK), site_links=True) - args.artifact.write_text(page, encoding="utf-8") - print(f"wrote {args.artifact} ({len(page):,} bytes)") - return 0 - page = build(docs, "/ai4data/img/program-logo.png", "/ai4data/img/program-logo-dark.png", site_links=True) - full = ('\n\n\n\n\n' - '\n' - '\n' + page.split("\n", 1)[0] + "\n\n\n" + page.split("\n", 1)[1] + "\n\n") - SITE_OUT.parent.mkdir(parents=True, exist_ok=True) - SITE_OUT.write_text(full, encoding="utf-8") - print(f"wrote {SITE_OUT.relative_to(REPO)} ({len(full):,} bytes)") + page = build(docs, data_uri(LOGO), data_uri(LOGO_DARK), site_links=True) + args.artifact.write_text(page, encoding="utf-8") + print(f"wrote {args.artifact} ({len(page):,} bytes)") return 0 diff --git a/scripts/docs/build_training_site.py b/scripts/docs/build_training_site.py new file mode 100644 index 00000000..2f9168f0 --- /dev/null +++ b/scripts/docs/build_training_site.py @@ -0,0 +1,551 @@ +#!/usr/bin/env python3 +"""Generate the Training section of the docs site from the training/ folder. + +The training folder stays the source: its READMEs, participant resources, and +notebooks are what GitHub shows and what reviewers edit. This script writes a +Markdown copy of them into website/generated/training/ (gitignored), which the +`training` docs instance of the site reads. The site build runs it before +Docusaurus, through `npm run convert-notebooks`. + + python3 scripts/docs/build_training_site.py # warnings are printed + python3 scripts/docs/build_training_site.py --strict # warnings fail the run + +The conventions a session folder follows are described in training/AUTHORING.md, and +check_session() reports the ones the build depends on. + +What the script does to the source: + +- The series README becomes the section's index page, and each session README + becomes the session's overview page. Sections listed in OMIT_SECTIONS are + left off the site. +- Each notebook becomes a page with its committed outputs, an "Open in Colab" + button, and a link to the notebook on GitHub. Colab section links + (`#scrollTo=section-N`) become heading anchors, and Colab links between the + session's notebooks become links between the pages. +- Relative links to files that are on the site are rewritten to the generated + pages. Relative links to anything else (data files, scripts, folders) point + to the file on GitHub. Absolute links to the published site become site- + internal links. +- Notebook HTML outputs keep their markup. Inline scripts are removed; the + site's pcnDetail client module provides the behaviour they had in Jupyter. + +No dependencies beyond the standard library. +""" +from __future__ import annotations + +import base64 +import hashlib +import html +import json +import re +import shutil +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +SRC = REPO / "training" +OUT = REPO / "website" / "generated" / "training" +GITHUB = "https://github.com/worldbank/ai4data" +BLOB = f"{GITHUB}/blob/main" +TREE = f"{GITHUB}/tree/main" +EDIT = f"{GITHUB}/edit/main" +COLAB = "https://colab.research.google.com/github/worldbank/ai4data/blob/main" +SITE = "https://worldbank.github.io/ai4data" + +# Sections of a README that concern the team preparing a session and stay on GitHub. +OMIT_SECTIONS = { + "training/README.md": ["Layout of a session folder", "Review", "Adding a session"], + "session": ["Roles", "Definition of ready"], +} + +# Fixed pages of a session: (source, output, sidebar label, sidebar position). Positions +# are within the session: the overview is 1, the preface 1, the slides 1.5, the notebooks 2, +# the participant resources 3, the document collection 3.5, and the worked-example data 4. +SESSION_PAGES = [ + ("preface.md", "preface.md", "Preface", 1), + ("assets/README.md", "worked-example-data.md", "Worked-example data", 4), +] + +WARNINGS: list[str] = [] + + +def warn(path: Path, message: str) -> None: + WARNINGS.append(f"{path.relative_to(REPO).as_posix()}: {message}") + + +def resource_pages(folder: Path) -> list[tuple[Path, str, str | None, int]]: + """Every Markdown file in resources/, as (source, output, sidebar label, position). + + The README is the category's first page, under the label "Participant guide". The other + files follow in the order in which the README first links to them, and files that it + does not link to follow in alphabetical order. A file's name is its page's URL.""" + res = folder / "resources" + if not res.is_dir(): + return [] + for other in sorted(res.iterdir()): + if other.suffix != ".md": + warn(other, "only Markdown files in resources/ become pages; this file stays on GitHub") + files = sorted(p for p in res.glob("*.md") if p.name != "README.md") + readme = res / "README.md" + linked = [] + if readme.exists(): + for target in re.findall(r"\]\(([^)#\s]+\.md)(?:#[^)]*)?\)", readme.read_text(encoding="utf-8")): + path = (res / target).resolve() + if path.parent == res.resolve() and path.exists() and path not in linked: + linked.append(path) + else: + warn(res, "resources/ has no README.md, so the Participant resources category has no first page") + order = linked + [p.resolve() for p in files if p.resolve() not in linked] + pages = [(readme, "resources/index.md", "Participant guide", 1)] if readme.exists() else [] + pages += [(p, f"resources/{p.stem}.md", None, i) for i, p in enumerate(order, start=2)] + return pages + + +def slugify(text: str) -> str: + text = re.sub(r"[^\w\s-]", "", text.lower(), flags=re.UNICODE) + return re.sub(r"[\s_]+", "-", text).strip("-") + + +def yaml_str(text: str) -> str: + return json.dumps(text, ensure_ascii=False) + + +def front_matter(**fields) -> str: + lines = ["---"] + for key, value in fields.items(): + if value is None: + continue + lines.append(f"{key}: {value if isinstance(value, (int, float, bool)) else yaml_str(str(value))}") + lines.append("format: md") + lines.append("---\n") + return "\n".join(lines) + "\n" + + +def split_title(markdown: str) -> tuple[str, str]: + match = re.search(r"^#\s+(.+?)\s*$", markdown, re.MULTILINE) + if not match: + return "Untitled", markdown + body = markdown[: match.start()] + markdown[match.end():] + return match.group(1), body.lstrip("\n") + + +def omit_sections(markdown: str, headings: list[str]) -> str: + """Remove each `## heading` section up to the next heading of level 2 or higher.""" + for heading in headings: + pattern = rf"^##\s+{re.escape(heading)}\s*\n.*?(?=^#{{1,2}}\s|\Z)" + markdown = re.sub(pattern, "", markdown, flags=re.MULTILINE | re.DOTALL) + return markdown.rstrip() + "\n" + + +class Site: + """Maps source files to generated pages and rewrites links between them.""" + + def __init__(self) -> None: + self.pages: dict[Path, Path] = {} # source file -> generated file + self.colab: dict[str, Path] = {} # Colab URL of a notebook -> generated file + + def add(self, src: Path, out: Path) -> None: + self.pages[src.resolve()] = out + if src.suffix == ".ipynb": + self.colab[f"{COLAB}/{src.relative_to(REPO).as_posix()}"] = out + + def rewrite_links(self, markdown: str, src: Path, out: Path, *, colab_to_page: bool) -> str: + def target(url: str) -> str: + url = url.strip() + if url.startswith(SITE + "/"): + path = "/" + url[len(SITE) + 1:] + # Static .html files are not routes; pathname:// keeps the link as is. + return f"pathname://{path}" if re.search(r"\.html(#|$)", path) else path + if url.startswith(SITE): + return "/" + if colab_to_page: + base, _, frag = url.partition("#") + if base in self.colab: + frag = frag.removeprefix("scrollTo=") + return self.relative(out, self.colab[base]) + (f"#{frag}" if frag else "") + if url.startswith("#scrollTo="): + return "#" + url[len("#scrollTo="):] + if re.match(r"^[a-z][a-z0-9+.-]*:", url) or url.startswith("#") or url.startswith("/"): + return url + path, _, frag = url.partition("#") + resolved = (src.parent / path).resolve() + frag = f"#{frag}" if frag else "" + if resolved in self.pages: + return self.relative(out, self.pages[resolved]) + frag + rel = resolved.relative_to(REPO).as_posix() + return f"{TREE if resolved.is_dir() else BLOB}/{rel}{frag}" + + def link(match: re.Match) -> str: + return f"{match.group(1)}({target(match.group(2))}{match.group(3) or ''})" + + parts = re.split(r"(^```.*?^```)", markdown, flags=re.MULTILINE | re.DOTALL) + for i in range(0, len(parts), 2): + parts[i] = re.sub(r"(\]\s?)\(([^()\s]+)(\s+\"[^\"]*\")?\)", link, parts[i]) + parts[i] = re.sub(r'href="([^"]+)"', lambda m: f'href="{target(m.group(1))}"', parts[i]) + return "".join(parts) + + @staticmethod + def relative(frm: Path, to: Path) -> str: + rel = Path(*([".."] * (len(frm.parent.relative_to(OUT).parts)))) / to.relative_to(OUT) + return rel.as_posix() if rel.as_posix().startswith(".") else "./" + rel.as_posix() + + +def write(path: Path, text: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +def markdown_page(site: Site, src: Path, out: Path, *, position: int, label: str | None = None, + omit: list[str] | None = None, slug: str | None = None, note: str | None = None) -> None: + title, body = split_title(src.read_text(encoding="utf-8")) + if omit: + body = omit_sections(body, omit) + if note: + body = body.rstrip() + "\n\n## Source files\n\n" + note + "\n" + body = site.rewrite_links(body, src, out, colab_to_page=False) + rel = src.relative_to(REPO).as_posix() + write(out, front_matter(title=title, sidebar_label=label, sidebar_position=position, slug=slug, + custom_edit_url=f"{EDIT}/{rel}") + body) + + +def slides_page(src: Path, out: Path, *, position: float) -> None: + """A page that shows the session's Canva deck in Canva's embedded viewer.""" + deck = json.loads(src.read_text(encoding="utf-8")) + view = deck["view_url"] + embed = view + ("&" if "?" in view else "?") + "embed" + rel = src.relative_to(REPO).as_posix() + body = ( + "This page shows the slides of the session in the Canva viewer. " + "The Open in Canva button opens the deck in a separate tab.\n\n" + f'

' + "Open in Canva

\n\n" + f'
\n' + ) + write(out, front_matter(title=deck["title"], sidebar_label="Slides", sidebar_position=position, + custom_edit_url=f"{BLOB}/{rel}") + body) + + +def documents_page(site: Site, docs_dir: Path, out: Path, *, position: float) -> None: + """The Document collection page: a session's worked-example documents in the DocumentViewer component. + + `docs_dir/roles.json` groups the documents by role and holds the page's text: `intro`, a + list of paragraphs, and `sources`, a paragraph on where the figures come from.""" + sys.path.insert(0, str(Path(__file__).resolve().parent)) + import build_training_pages + roles_file = docs_dir / "roles.json" + roles = json.loads(roles_file.read_text(encoding="utf-8")) + write(out.with_suffix(".json"), json.dumps(build_training_pages.viewer_data(docs_dir), indent=1, ensure_ascii=False)) + rel = docs_dir.relative_to(REPO).as_posix() + text = "\n\n".join(roles.get("intro", [])) + sources = f"The source files are in [`{rel}`]({TREE}/{rel}) on GitHub." + if roles.get("sources"): + sources += " " + roles["sources"] + write(out, "\n".join([ + "---", + "title: Document collection", + "sidebar_label: Document collection", + f"sidebar_position: {position}", + f"custom_edit_url: {BLOB}/{roles_file.relative_to(REPO).as_posix()}", + "---", + "", + "import DocumentViewer from '@site/src/components/training/DocumentViewer';", + f"import data from './{out.with_suffix('.json').name}';", + "", + site.rewrite_links(text, roles_file, out, colab_to_page=False), + "", + "", + "", + site.rewrite_links(sources, roles_file, out, colab_to_page=False), + "", + ])) + + +# --- Notebooks ---------------------------------------------------------------- + +DATA_IMAGE = re.compile(r"!\[([^\]]*)\]\(data:image/(svg\+xml|png|jpeg);base64,([A-Za-z0-9+/=]+)\)") +EXT = {"svg+xml": "svg", "png": "png", "jpeg": "jpg"} + + +def source_text(cell: dict) -> str: + src = cell.get("source", "") + return "".join(src) if isinstance(src, list) else src + + +def fence(text: str, lang: str = "") -> str: + ticks = "```" + while ticks in text: + ticks += "`" + return f"{ticks}{lang}\n{text.rstrip()}\n{ticks}" + + +OUTPUT_CSS = REPO / "website" / "generated" / "training-outputs.css" +STYLES: dict[str, None] = {} # ", markup, flags=re.DOTALL | re.IGNORECASE): + STYLES.setdefault(css.strip(), None) + markup = re.sub(r"", "", markup, flags=re.DOTALL | re.IGNORECASE) + # A blank line ends a CommonMark HTML block, so lines are joined without blank lines. + markup = re.sub(r"\n[ \t]*\n+", "\n", markup.strip()) + # Whitespace between table tags becomes text nodes that React rejects inside a table. + table = r"/?(?:table|thead|tbody|tfoot|tr|td|th|colgroup|col)\b" + markup = re.sub(rf"(<{table}[^>]*>)\s+(?=<)", r"\1", markup) + markup = re.sub(rf"\s+(?=<{table})", "", markup) + return f'
\n{markup}\n
' + + +def notebook_page(site: Site, src: Path, out: Path, *, position: int, assets_dir: Path) -> None: + nb = json.loads(src.read_text(encoding="utf-8")) + rel = src.relative_to(REPO).as_posix() + blocks: list[str] = [] + title = None + + def save_image(kind: str, data: bytes) -> str: + name = hashlib.sha1(data).hexdigest()[:12] + "." + EXT[kind] + (assets_dir).mkdir(parents=True, exist_ok=True) + (assets_dir / name).write_bytes(data) + return Site.relative(out, assets_dir / name) + + for cell in nb["cells"]: + text = source_text(cell) + if cell["cell_type"] == "markdown": + if title is None: + found, text = split_title(text) + if found != "Untitled": + title = found + cell_id = cell.get("metadata", {}).get("id") or cell.get("id", "") + if cell_id.startswith("section-"): + text = re.sub(r"^(#{2,3}\s+.+?)\s*$", rf"\1 {{#{cell_id}}}", text, count=1, flags=re.MULTILINE) + text = site.rewrite_links(text, src, out, colab_to_page=True) + # Embedded images become files next to the page; done after link rewriting, + # which would otherwise resolve their relative paths against the source folder. + text = DATA_IMAGE.sub( + lambda m: f"![{m.group(1)}]({save_image(m.group(2), base64.b64decode(m.group(3)))})", text) + blocks.append(text) + elif cell["cell_type"] == "code": + if text.strip(): + blocks.append(fence(text, "python")) + for output in cell.get("outputs", []): + blocks.extend(render_output(output, save_image)) + title = title or src.stem + header = ( + '

' + f'Open in Colab ' + f'View the notebook on GitHub' + "

\n\n" + "The outputs on this page are the ones committed with the notebook. Without an API key, " + "the notebook replays recorded model answers, and the same recordings appear here.\n" + ) + short = re.sub(r"^\d+\s*·\s*", "", title) + write(out, front_matter(title=title, sidebar_label=title.split(" · ")[0] + " · " + short, + sidebar_position=position, slug=out.stem, custom_edit_url=f"{BLOB}/{rel}") + + header + "\n" + "\n\n".join(b.strip("\n") for b in blocks if b.strip()) + "\n") + + +def render_output(output: dict, save_image) -> list[str]: + kind = output.get("output_type") + if kind == "stream": + text = source_text({"source": output.get("text", "")}) + return [fence(text, "text")] if text.strip() else [] + if kind == "error": + return [fence("\n".join(output.get("traceback", [])) or output.get("evalue", ""), "text")] + data = output.get("data", {}) + if "application/vnd.jupyter.widget-view+json" in data: + return [] + if "text/html" in data: + return [html_block(source_text({"source": data["text/html"]}))] + for mime, ext in (("image/svg+xml", "svg+xml"), ("image/png", "png"), ("image/jpeg", "jpeg")): + if mime in data: + raw = data[mime] + raw = "".join(raw) if isinstance(raw, list) else raw + payload = raw.encode() if mime == "image/svg+xml" else base64.b64decode(raw) + return [f"![Output image]({save_image(ext, payload)})"] + if "text/markdown" in data: + return [source_text({"source": data["text/markdown"]})] + if "text/plain" in data: + return [fence(source_text({"source": data["text/plain"]}), "text")] + return [] + + +# --- Checks ------------------------------------------------------------------- +# The conventions of training/AUTHORING.md that the build depends on. A session that breaks +# one still builds, with a page missing or a link that does not resolve, so each is reported. + +def cell_ids(nb: dict) -> set[str]: + return {c.get("metadata", {}).get("id") or c.get("id", "") for c in nb["cells"]} - {""} + + +def check_session(folder: Path) -> None: + number = int(re.match(r"session-(\d+)", folder.name).group(1)) + readme = folder / "README.md" + if not readme.exists(): + warn(folder, "the session folder has no README.md, which is the session's overview page") + else: + title, _ = split_title(readme.read_text(encoding="utf-8")) + m = re.match(r"Session (\d+) · \S", title) + if not m or int(m.group(1)) != number: + warn(readme, f"the title should read 'Session {number:02d} · ', found '{title}'") + series = (SRC / "README.md").read_text(encoding="utf-8") + if f"]({folder.name}/README.md)" not in series: + warn(SRC / "README.md", f"the series table does not link to {folder.name}/README.md") + if not (folder / "preface.md").exists(): + warn(folder, "the session has no preface.md") + + nb_dir = folder / "notebooks" + notebooks = {} + for path in sorted(nb_dir.glob("*.ipynb")) if nb_dir.is_dir() else []: + if not re.match(r"\d\d-", path.name): + warn(path, "only notebooks named NN-short-name.ipynb become pages") + continue + notebooks[path] = json.loads(path.read_text(encoding="utf-8")) + for path, nb in notebooks.items(): + cells = nb["cells"] + found, _ = split_title(source_text(cells[0])) if cells else ("Untitled", "") + if not re.match(rf"{path.name[:2]} · \S", found): + warn(path, f"the first cell should hold the title '# {path.name[:2]} · <title>', found '{found}'") + ids = cell_ids(nb) + for cell in cells: + if cell["cell_type"] != "markdown": + continue + text = source_text(cell) + cid = cell.get("metadata", {}).get("id") or cell.get("id", "") + if cid.startswith("section-") and not re.search(r"^#{2,3}\s", text, re.MULTILINE): + warn(path, f"cell {cid} has no ## or ### heading, so its section has no anchor on the site") + for url, frag in re.findall(r"\]\(([^)#\s]*)#scrollTo=([^)\s]+)\)", text): + if not url: + target = path + elif url.startswith(COLAB + "/"): + target = REPO / url[len(COLAB) + 1:] + else: + continue + if target not in notebooks: + warn(path, f"a link points to {target.name}, which is not a notebook of this session") + elif frag not in cell_ids(notebooks[target]): + warn(path, f"a link points to #scrollTo={frag} in {target.name}, which has no cell with that id") + for url in re.findall(r"\]\((" + re.escape(COLAB) + r"/[^)#\s]+)", text): + if not (REPO / url[len(COLAB) + 1:]).exists(): + warn(path, f"a Colab link points to a file that does not exist: {url[len(COLAB) + 1:]}") + + for page in [folder / "preface.md", *sorted((folder / "resources").glob("*.md"))]: + if page.exists() and split_title(page.read_text(encoding="utf-8"))[0] == "Untitled": + warn(page, "the page has no '# Title' line") + + canva = folder / "slides" / "canva.json" + if canva.exists(): + deck = json.loads(canva.read_text(encoding="utf-8")) + url = deck.get("view_url", "") + if "title" not in deck or not url: + warn(canva, "canva.json needs a 'title' and a 'view_url'") + elif not re.match(r"https://www\.canva\.com/design/[^/]+/[^/]+/view", url): + warn(canva, "view_url should be a public view link of the form https://www.canva.com/design/<id>/<token>/view") + + docs_dir = folder / "assets" / "documents" + if docs_dir.is_dir(): + roles_file = docs_dir / "roles.json" + files = {p.stem for p in docs_dir.glob("*.md")} + if not roles_file.exists(): + warn(docs_dir, "the documents have no roles.json, so the session has no Document collection page") + else: + roles = json.loads(roles_file.read_text(encoding="utf-8")) + listed = [d["id"] for g in roles.get("groups", []) for d in g.get("documents", [])] + for missing in sorted(set(listed) - files): + warn(roles_file, f"lists '{missing}', but there is no {missing}.md") + for unlisted in sorted(files - set(listed)): + warn(roles_file, f"does not list {unlisted}.md, so the page does not show it") + for doc in sorted(docs_dir.glob("*.md")): + head = re.match(r"---\n(.*?)\n---\n", doc.read_text(encoding="utf-8"), re.S) + keys = {line.partition(":")[0].strip() for line in head.group(1).splitlines()} if head else set() + for key in ("id", "title", "date"): + if key not in keys: + warn(doc, f"the document's front matter has no '{key}'") + + +# --- Build -------------------------------------------------------------------- + +def session_slug(folder: Path) -> str: + match = re.match(r"session-(\d+)", folder.name) + return f"session-{match.group(1)}" if match else folder.name + + +def build() -> None: + if OUT.exists(): + shutil.rmtree(OUT) + STYLES.clear() + site = Site() + sessions = sorted(p for p in SRC.iterdir() if p.is_dir() and re.match(r"session-\d+", p.name)) + + # Register every page first so that links between them resolve in any order. + plan = [] + site.add(SRC / "README.md", OUT / "index.md") + for folder in sessions: + dest = OUT / session_slug(folder) + site.add(folder / "README.md", dest / "index.md") + notebooks = sorted((folder / "notebooks").glob("[0-9][0-9]-*.ipynb")) + nb_out = [] + for nb in notebooks: + title = nb.stem + first = json.loads(nb.read_text(encoding="utf-8"))["cells"][0] + found, _ = split_title(source_text(first)) + if found != "Untitled": + title = found + out = dest / "notebooks" / f"{slugify(title)}.md" + site.add(nb, out) + nb_out.append((nb, out)) + pages = [] + fixed = [(folder / src_rel, out_rel, label, position) for src_rel, out_rel, label, position in SESSION_PAGES] + for src, out_rel, label, position in fixed + resource_pages(folder): + if src.exists(): + site.add(src, dest / out_rel) + pages.append((src, dest / out_rel, label, position)) + plan.append((folder, dest, nb_out, pages)) + + markdown_page(site, SRC / "README.md", OUT / "index.md", position=1, label="Training series", + omit=OMIT_SECTIONS["training/README.md"], slug="/", + note=f"The pages of this section are generated from the [training folder]({TREE}/training) " + "of the repository. The slide decks, the demonstration scripts, and the scripts that " + "regenerate the notebooks are kept there.") + for folder, dest, nb_out, pages in plan: + check_session(folder) + title, _ = split_title((folder / "README.md").read_text(encoding="utf-8")) + write(dest / "_category_.json", json.dumps( + {"label": title.replace("Session ", "", 1), "position": int(re.sub(r"\D", "", dest.name) or 99) + 1, + "collapsed": False}, indent=2)) + markdown_page(site, folder / "README.md", dest / "index.md", position=1, + label=title.split(" · ", 1)[-1], + omit=OMIT_SECTIONS["session"], + note=f"The session's slides, demonstration script, worked-example files, and notebook " + f"sources are in the [session folder]({TREE}/{folder.relative_to(REPO).as_posix()}) " + "on GitHub.") + write(dest / "notebooks" / "_category_.json", + json.dumps({"label": "Notebooks", "position": 2, "collapsed": False}, indent=2)) + for i, (nb, out) in enumerate(nb_out, start=1): + notebook_page(site, nb, out, position=i, assets_dir=out.parent / "img") + resources = [p for p in pages if p[1].parent.name == "resources"] + if resources: + write(dest / "resources" / "_category_.json", json.dumps( + {"label": "Participant resources", "position": 3, "collapsed": True, + "link": {"type": "doc", "id": f"{dest.name}/resources/index"}}, indent=2)) + for src, out, label, position in pages: + markdown_page(site, src, out, position=position, label=label) + docs_dir = folder / "assets" / "documents" + if (docs_dir / "roles.json").exists(): + documents_page(site, docs_dir, dest / "documents.mdx", position=3.5) + if (folder / "slides" / "canva.json").exists(): + slides_page(folder / "slides" / "canva.json", dest / "slides.md", position=1.5) + print(f"training site: {dest.relative_to(REPO)} ({len(nb_out)} notebooks, {len(pages)} pages)") + write(OUTPUT_CSS, "/* Styles of the training notebook outputs, generated by " + "scripts/docs/build_training_site.py. */\n" + "\n".join(STYLES) + "\n") + + +if __name__ == "__main__": + build() + for line in WARNINGS: + print(f"training site: warning: {line}") + if WARNINGS and "--strict" in sys.argv: + raise SystemExit(f"training site: {len(WARNINGS)} warnings (see training/AUTHORING.md)") diff --git a/training/AUTHORING.md b/training/AUTHORING.md new file mode 100644 index 00000000..43e96f86 --- /dev/null +++ b/training/AUTHORING.md @@ -0,0 +1,118 @@ +# Authoring a training session + +This guide describes the folder layout, file names, and title formats of a training session, and the page that each file becomes in the Training section of the docs site. It also covers the writing rules for the session's text and the checks to run before a pull request. The site pages are generated from the session folder by `scripts/docs/build_training_site.py` at every site build, so a session that follows this guide appears on the site without changes to the site's code. + +## Folder layout + +``` +training/ + README.md series overview; the Training index page + AUTHORING.md this guide (GitHub only) + session-NN-short-name/ + README.md session overview page + preface.md Preface page + slides/canva.json Slides page (optional) + notebooks/NN-short-name.ipynb one page per notebook + resources/README.md Participant guide, the first page of Participant resources + resources/*.md further pages of Participant resources + assets/README.md Worked-example data page + assets/documents/*.md the session's documents (optional) + assets/documents/roles.json Document collection page + demo/ GitHub only +``` + +`NN` is the two-digit session or notebook number. Everything else in the folder, such as decks in `.pptx` form, data files, and scripts, stays on GitHub, and the site pages link to it there. + +## How the files become pages + +| Source | Page in the sidebar | Requirements | +|---|---|---| +| `README.md` | the session's category, labelled `NN · <title>`, and its overview page | The first line is `# Session NN · <title>`. The sections `Roles` and `Definition of ready` stay on GitHub. | +| `preface.md` | Preface | A `# Title` line. | +| `slides/canva.json` | Slides, with the deck in Canva's embedded viewer | See [Slides](#slides). | +| `notebooks/NN-*.ipynb` | Notebooks, one page each, in file-name order | See [Notebooks](#notebooks). | +| `resources/README.md` | Participant guide | A `# Title` line. | +| `resources/<name>.md` | a page at `resources/<name>` | A `# Title` line. Pages follow the order in which `resources/README.md` first links to them; files without a link follow in alphabetical order. | +| `assets/documents/roles.json` | Document collection | See [Document collection](#document-collection). | +| `assets/README.md` | Worked-example data | A `# Title` line. | + +The session also needs a row in the table of `training/README.md` that links to its README. A planned session has a row with its title and the format `Planned`, without a link. + +Links between these files are written as relative links to the source files, for example `[glossary](resources/glossary.md)` or `[notebook 03](notebooks/03-retrieval-augmented-generation.ipynb)`. They work on GitHub, and the generator rewrites them to the corresponding site pages. Relative links to any other file point to that file on GitHub. + +## Notebooks + +- **File name and title.** A notebook is named `NN-short-name.ipynb`, and its first cell begins with `# NN · <title>`. The page URL is formed from the title, so a new title after the session is published needs a redirect from the old URL (see [Addresses of published pages](#addresses-of-published-pages)). +- **Sections.** Each numbered section starts a Markdown cell whose metadata `id` is `section-N` and whose first line is `## N. <section title>`. The site gives that heading the anchor `#section-N`. +- **Links to sections.** A contents list links to a section as `[N. <title>](#scrollTo=section-N)`, which Colab follows inside the notebook. A link to another notebook of the session uses its Colab address on the `main` branch, `https://colab.research.google.com/github/worldbank/ai4data/blob/main/training/<session folder>/notebooks/<file>`, with an optional `#scrollTo=section-N`. On the site, both kinds of link lead to the corresponding page and heading. +- **Outputs.** Notebooks are committed with their outputs, and the site shows those outputs. Without an API key, every notebook runs in a fresh Colab environment and replays recorded model answers. +- **Output types.** Text and stream outputs appear as text blocks, Markdown outputs as Markdown, and images as image files. HTML outputs keep their markup. Scripts inside HTML outputs are removed, and an interactive output needs a client module in `website/src/clientModules/` for its behaviour on the site; `pcnDetail.js` is the example. `<style>` blocks are moved to a generated stylesheet. Widget outputs are left out. + +## Slides + +The Slides page shows a deck kept in Canva. `slides/canva.json` holds the page title and the public view link of the deck: + +```json +{ + "title": "Slides", + "view_url": "https://www.canva.com/design/<id>/<token>/view?utm_content=...&utlId=..." +} +``` + +The link is the one Canva gives under **Share → Public view link**, or the address that a `canva.link` short link redirects to. Its query string is kept, because Canva attributes views by it. Anyone with the link can view the deck, so the deck holds only material that may be public. The names of colleagues appear only with their agreement. + +## Document collection + +When a session searches a collection of documents, the documents are Markdown files in `assets/documents/`, each with front matter. The page shows the fields `id`, `type`, `country`, `date`, `source`, and `note`; `id`, `title`, and `date` are required. The application cites a chunk as `[id · section · year]`, and the page labels each chunk the same way. + +`assets/documents/roles.json` groups the documents by their role in the session and holds the text of the page: + +```json +{ + "intro": ["First paragraph of the page.", "Second paragraph."], + "sources": "A sentence on where the figures in the documents come from.", + "groups": [ + { + "key": "evidence", + "label": "Relevant evidence", + "description": "What the documents of this group are and how the session uses them.", + "documents": [{"id": "kenya-digital", "note": "The role of this document in one sentence."}] + } + ] +} +``` + +The group keys `evidence`, `definition`, `outdated`, and `distractor` have their own colours on the page; other keys are shown without a colour. Every document in the folder is listed in one group. The intro and sources text is Markdown, and relative links in it are resolved from `assets/documents/`. + +## Writing + +The writing rules of the [cookbook authoring guide](../cookbook/authoring/index.md) apply to every page of a session: descriptive titles, plain statements, no slogans, and no antithesis as a habit. Three further rules apply to training material. + +- **Audience.** A session is written for a general audience of researchers, analysts, people who maintain data and documents, reviewers, and developers. The preface and the overview describe the session from its content. +- **Preface.** The preface explains the problem that the session addresses and the readers for whom it matters. It then sets out the ideas that the notebooks develop, how readers at different technical levels can use the session, and the contribution of each notebook in one or two sentences. Each paragraph explains; none of them announces the structure of the page. +- **Notebook text.** Markdown cells are explanations in connected paragraphs. Each cell says what the next code does, what its result shows, and what that result means for the application. + +The materials contain no names of colleagues, no API keys or other credentials, no confidential documents, and no delivery dates. + +## Addresses of published pages + +Once a session is on the site, its page addresses are listed in `website/published-routes.txt`, and the site build fails when one of them stops resolving. A page moves when its source file is renamed or moved, when a notebook title changes, or when a resource file is renamed. In each case, an entry in `website/redirects.json` maps the old address to the new one. The process is described in `website/README.md`, under "Changing or removing a page". + +## Checks before a pull request + +The site build runs the generator, which prints a warning for each convention of this guide that a session breaks. The strict run turns the warnings into an error: + +```bash +python3 scripts/docs/build_training_site.py --strict +cd website && npm run build +npx docusaurus serve --port 3400 --no-open +``` + +The served site is at `http://localhost:3400/ai4data/training/`. Before the pull request: + +1. The strict run reports no warnings, and the site build succeeds. +2. `npm run routes:update` has added the new pages to `website/published-routes.txt`, and every moved page has an entry in `website/redirects.json`. +3. Every page of the session opens from the sidebar, and the section links of each notebook lead to their headings. +4. The notebooks run in a fresh Colab environment without an API key. +5. The session has a row in the table of `training/README.md`. +6. The text of the touched files has been checked against the writing rules above. diff --git a/training/README.md b/training/README.md index 22c8ba82..9f7d2c40 100644 --- a/training/README.md +++ b/training/README.md @@ -1,41 +1,47 @@ # Training series -Materials for the training series of the AI for Data – Data for AI program: session plans, slide decks, notebooks, worked-example assets, demonstration scripts, and participant resources. Each session lives in its own folder and is self-contained. +Materials for the training series of the AI for Data – Data for AI program: session plans, slide decks, notebooks, worked-example assets, demonstration scripts, and participant resources. Each session has its own folder, and that folder holds all of the session's materials. | Session | Title | Format | |---|---|---| | 01 | [Building AI applications for research and data](session-01-building-ai-applications/README.md) | 90 minutes, slides and guided notebooks, plus 15 to 30 minutes of questions | +| 02 | Building specialized small AI using synthetic data | Planned | +| 03 | Evaluating AI systems | Planned | +| 04 | Applying AI to metadata curation and data quality | Planned | +| 05 | Building trustworthy AI applications | Planned | +| 06 | All about context: Why AI is really a search problem | Planned | ### Session 01 notebooks -They open in Google Colab from the `main` branch of `worldbank/ai4data`, and each ends with a link to the next one. For the lab (07), choose a T4 GPU runtime before you run it; the notebook explains how. +They open in Google Colab from the `main` branch of `worldbank/ai4data`, and each ends with a link to the next one. The lab (07) needs a T4 GPU runtime, and the notebook explains how to select one. | Notebook | Open in Colab | |---|---| -| 01 · From a language model to an application | [Open](https://colab.research.google.com/github/worldbank/ai4data/blob/main/training/session-01-building-ai-applications/notebooks/01-from-model-to-application.ipynb) | -| 02 · Preparing knowledge for an application | [Open](https://colab.research.google.com/github/worldbank/ai4data/blob/main/training/session-01-building-ai-applications/notebooks/02-preparing-knowledge.ipynb) | -| 03 · Retrieval-augmented generation | [Open](https://colab.research.google.com/github/worldbank/ai4data/blob/main/training/session-01-building-ai-applications/notebooks/03-retrieval-augmented-generation.ipynb) | -| 04 · Tools and structured outputs | [Open](https://colab.research.google.com/github/worldbank/ai4data/blob/main/training/session-01-building-ai-applications/notebooks/04-tools-and-structured-outputs.ipynb) | -| 05 · Workflows and agents | [Open](https://colab.research.google.com/github/worldbank/ai4data/blob/main/training/session-01-building-ai-applications/notebooks/05-workflows-and-agents.ipynb) | -| 06 · Verification | [Open](https://colab.research.google.com/github/worldbank/ai4data/blob/main/training/session-01-building-ai-applications/notebooks/06-verification-and-evaluation.ipynb) | -| 07 · Lab: chat with your documents | [Open](https://colab.research.google.com/github/worldbank/ai4data/blob/main/training/session-01-building-ai-applications/notebooks/07-lab-chat-with-your-documents.ipynb) | +| [01 · From a language model to an application](session-01-building-ai-applications/notebooks/01-from-model-to-application.ipynb) | [Open](https://colab.research.google.com/github/worldbank/ai4data/blob/main/training/session-01-building-ai-applications/notebooks/01-from-model-to-application.ipynb) | +| [02 · Preparing knowledge for an application](session-01-building-ai-applications/notebooks/02-preparing-knowledge.ipynb) | [Open](https://colab.research.google.com/github/worldbank/ai4data/blob/main/training/session-01-building-ai-applications/notebooks/02-preparing-knowledge.ipynb) | +| [03 · Retrieval-augmented generation](session-01-building-ai-applications/notebooks/03-retrieval-augmented-generation.ipynb) | [Open](https://colab.research.google.com/github/worldbank/ai4data/blob/main/training/session-01-building-ai-applications/notebooks/03-retrieval-augmented-generation.ipynb) | +| [04 · Tools and structured outputs](session-01-building-ai-applications/notebooks/04-tools-and-structured-outputs.ipynb) | [Open](https://colab.research.google.com/github/worldbank/ai4data/blob/main/training/session-01-building-ai-applications/notebooks/04-tools-and-structured-outputs.ipynb) | +| [05 · Workflows and agents](session-01-building-ai-applications/notebooks/05-workflows-and-agents.ipynb) | [Open](https://colab.research.google.com/github/worldbank/ai4data/blob/main/training/session-01-building-ai-applications/notebooks/05-workflows-and-agents.ipynb) | +| [06 · Verification](session-01-building-ai-applications/notebooks/06-verification-and-evaluation.ipynb) | [Open](https://colab.research.google.com/github/worldbank/ai4data/blob/main/training/session-01-building-ai-applications/notebooks/06-verification-and-evaluation.ipynb) | +| [07 · Lab: chat with your documents](session-01-building-ai-applications/notebooks/07-lab-chat-with-your-documents.ipynb) | [Open](https://colab.research.google.com/github/worldbank/ai4data/blob/main/training/session-01-building-ai-applications/notebooks/07-lab-chat-with-your-documents.ipynb) | ## Layout of a session folder ``` session-NN-short-name/ README.md session plan: audience, objectives, timing, what to prepare, definition of ready - slides/ the deck (.pptx) and its outline (outline.md), which is the reviewable text of every slide + preface.md the problem the session addresses, its readers, its ideas, and the notebooks + slides/ the deck and its outline, or canva.json with the public view link of a deck kept in Canva notebooks/ the notebooks, a shared module they import, and the script that regenerates them - assets/ the worked example: data, metadata, documents, questions, recorded model outputs + assets/ the worked example: data, metadata, documents with their roles.json, questions, recorded model outputs demo/ scripts for live demonstrations, with pretested inputs and fallbacks - resources/ what participants take home: setup guide, glossary, exercises, next steps + resources/ what participants take home: a README as the participant guide, glossary, exercises, next steps ``` ## Conventions - **One worked example per session.** Every notebook and every slide of a session uses the same question, the same data, and the same documents, so that what changes between steps is the capability being added. -- **Runnable without credentials.** Every notebook runs in a fresh Colab environment with no API key. Model calls fall back to recorded outputs that are labelled as such; everything else, from the vector store to the MCP server, runs for real. A live key, when available, is read from the environment and never written into a notebook. +- **Runnable without credentials.** Every notebook runs in a fresh Colab environment with no API key. Model calls fall back to recorded outputs that are labelled as such; everything else, from the vector store to the MCP server, runs in full. A live key, when available, is read from the environment and never written into a notebook. - **Popular libraries over custom code.** Sessions use the libraries participants will meet in their own work (LangChain, LangGraph, FastMCP, Pydantic, and the program's own packages such as `pcn-core`), and write custom code only for what those do not provide. - **Real data, labelled.** Statistical values come from a named source with the retrieval date recorded. Documents written for a workshop say so in their header, state where their figures come from, and are never presented as publications. - **Cached outputs.** Notebooks are committed with their outputs so that reviewers and participants can read them without running them. The regeneration script reruns them in fallback mode. @@ -49,4 +55,4 @@ A session is ready for delivery when its definition-of-ready checklist in the se ## Adding a session -Copy the layout above, write the session README first, agree the worked example, then build the notebooks and the deck against it. +Copy the layout above, write the session README first, agree the worked example, then build the notebooks and the deck against it. [AUTHORING.md](AUTHORING.md) describes the file names, titles, and formats that the Training section of the docs site expects, the writing rules for the session's text, and the checks to run before a pull request. diff --git a/training/session-01-building-ai-applications/README.md b/training/session-01-building-ai-applications/README.md index 861c3896..fc2139cc 100644 --- a/training/session-01-building-ai-applications/README.md +++ b/training/session-01-building-ai-applications/README.md @@ -2,6 +2,8 @@ 90 minutes of slides and guided notebook demonstrations, followed by 15 to 30 minutes of questions. +The [preface](preface.md) explains the problem addressed in the session and the ideas developed in its notebooks. It also describes how readers with different backgrounds and levels of technical experience can use the session. + ## Who it is for Two audiences at once: @@ -46,17 +48,17 @@ Questions are taken at the notebook transitions as well as at the end. Notebooks | Notebook | Open in Colab | |---|---| -| 01 · From a language model to an application | [Open](https://colab.research.google.com/github/worldbank/ai4data/blob/main/training/session-01-building-ai-applications/notebooks/01-from-model-to-application.ipynb) | -| 02 · Preparing knowledge for an application | [Open](https://colab.research.google.com/github/worldbank/ai4data/blob/main/training/session-01-building-ai-applications/notebooks/02-preparing-knowledge.ipynb) | -| 03 · Retrieval-augmented generation | [Open](https://colab.research.google.com/github/worldbank/ai4data/blob/main/training/session-01-building-ai-applications/notebooks/03-retrieval-augmented-generation.ipynb) | -| 04 · Tools and structured outputs | [Open](https://colab.research.google.com/github/worldbank/ai4data/blob/main/training/session-01-building-ai-applications/notebooks/04-tools-and-structured-outputs.ipynb) | -| 05 · Workflows and agents | [Open](https://colab.research.google.com/github/worldbank/ai4data/blob/main/training/session-01-building-ai-applications/notebooks/05-workflows-and-agents.ipynb) | -| 06 · Verification | [Open](https://colab.research.google.com/github/worldbank/ai4data/blob/main/training/session-01-building-ai-applications/notebooks/06-verification-and-evaluation.ipynb) | -| 07 · Lab: chat with your documents | [Open](https://colab.research.google.com/github/worldbank/ai4data/blob/main/training/session-01-building-ai-applications/notebooks/07-lab-chat-with-your-documents.ipynb) | +| [01 · From a language model to an application](notebooks/01-from-model-to-application.ipynb) | [Open](https://colab.research.google.com/github/worldbank/ai4data/blob/main/training/session-01-building-ai-applications/notebooks/01-from-model-to-application.ipynb) | +| [02 · Preparing knowledge for an application](notebooks/02-preparing-knowledge.ipynb) | [Open](https://colab.research.google.com/github/worldbank/ai4data/blob/main/training/session-01-building-ai-applications/notebooks/02-preparing-knowledge.ipynb) | +| [03 · Retrieval-augmented generation](notebooks/03-retrieval-augmented-generation.ipynb) | [Open](https://colab.research.google.com/github/worldbank/ai4data/blob/main/training/session-01-building-ai-applications/notebooks/03-retrieval-augmented-generation.ipynb) | +| [04 · Tools and structured outputs](notebooks/04-tools-and-structured-outputs.ipynb) | [Open](https://colab.research.google.com/github/worldbank/ai4data/blob/main/training/session-01-building-ai-applications/notebooks/04-tools-and-structured-outputs.ipynb) | +| [05 · Workflows and agents](notebooks/05-workflows-and-agents.ipynb) | [Open](https://colab.research.google.com/github/worldbank/ai4data/blob/main/training/session-01-building-ai-applications/notebooks/05-workflows-and-agents.ipynb) | +| [06 · Verification](notebooks/06-verification-and-evaluation.ipynb) | [Open](https://colab.research.google.com/github/worldbank/ai4data/blob/main/training/session-01-building-ai-applications/notebooks/06-verification-and-evaluation.ipynb) | +| [07 · Lab: chat with your documents](notebooks/07-lab-chat-with-your-documents.ipynb) | [Open](https://colab.research.google.com/github/worldbank/ai4data/blob/main/training/session-01-building-ai-applications/notebooks/07-lab-chat-with-your-documents.ipynb) | Each notebook ends with a link to the next. Notebook 07 runs a small open model, so choose **Runtime > Change runtime type > T4 GPU** before you run it. -The eight documents of the worked example, with their metadata, their role in the session, and the chunks that notebook 02 makes of them, are shown at [worldbank.github.io/ai4data/training/session-01/documents.html](https://worldbank.github.io/ai4data/training/session-01/documents.html). The page is generated from `assets/documents/` by `scripts/docs/build_training_pages.py` when the site is built. +The eight documents of the worked example, with their metadata, their role in the session, and the chunks that notebook 02 makes of them, are shown on the [Document collection](https://worldbank.github.io/ai4data/training/session-01/documents) page of the docs site. The page is generated from `assets/documents/` when the site is built. ## Materials diff --git a/training/session-01-building-ai-applications/assets/documents/roles.json b/training/session-01-building-ai-applications/assets/documents/roles.json new file mode 100644 index 00000000..90ffdd2d --- /dev/null +++ b/training/session-01-building-ai-applications/assets/documents/roles.json @@ -0,0 +1,69 @@ +{ + "intro": [ + "This page shows the eight documents of the worked example. They were written for the workshop from World Development Indicators values retrieved on 13 October 2026, and none of them is an official publication. The application built in notebooks 02 to 07 searches only this collection.", + "Notebook 02 splits each document at its headings into chunks. When the application uses a chunk in an answer, it cites the chunk as `[document id · section · year]`. Each chunk on this page is labelled with that citation." + ], + "sources": "The figures in the briefs are rounded from [`indicators.csv`](../indicators.csv), and the 2016 summary reproduces the 2015 values.", + "groups": [ + { + "key": "evidence", + "label": "Relevant evidence", + "description": "The four country briefs. Their figures are rounded from the indicator data, and a correct answer to the main question cites them.", + "documents": [ + { + "id": "kenya-digital", + "note": "Relevant evidence for internet use and mobile subscriptions in Kenya." + }, + { + "id": "rwanda-digital", + "note": "Relevant evidence for internet use and mobile subscriptions in Rwanda." + }, + { + "id": "kenya-economy", + "note": "Relevant evidence for GDP per capita and growth in Kenya." + }, + { + "id": "rwanda-economy", + "note": "Relevant evidence for GDP per capita and growth in Rwanda." + } + ] + }, + { + "key": "definition", + "label": "Definition", + "description": "A note on the definition of the internet-users indicator and on the comparability of its values across years.", + "documents": [ + { + "id": "internet-indicator", + "note": "Definition of the indicator; the source for the question on its metadata." + } + ] + }, + { + "key": "outdated", + "label": "Outdated", + "description": "A 2016 summary with figures for 2015 and expectations that later data did not confirm. It outranks the current briefs for a question about the latest figures, and its date in the metadata removes it.", + "documents": [ + { + "id": "regional-2016", + "note": "Outdated figures; shows why dates in metadata matter." + } + ] + }, + { + "key": "distractor", + "label": "Distractors", + "description": "Two notes on related topics without the requested indicators. The coffee note is the retrieval failure case; the mobile-money note is a plausible wrong passage.", + "documents": [ + { + "id": "coffee", + "note": "Irrelevant to the question; shows a retrieval failure." + }, + { + "id": "mobile-money", + "note": "Related topic without the requested indicators; a plausible distractor." + } + ] + } + ] +} diff --git a/training/session-01-building-ai-applications/preface.md b/training/session-01-building-ai-applications/preface.md new file mode 100644 index 00000000..6a64bb5d --- /dev/null +++ b/training/session-01-building-ai-applications/preface.md @@ -0,0 +1,41 @@ +# How AI chatbots work + +This preface explains the problem addressed in session 01, the readers for whom that problem matters, and the three ideas developed in the notebooks. It ends with notes on running the notebooks and a short description of each one. + +## The problem + +A language model can write fluent text on almost any subject, and the text often contains figures such as percentages, totals, and years. The quality of the writing gives no information about the origin of those figures. A figure produced from training alone may be correct, out of date, or invented, and the answer gives the reader no means of finding out. Session 01 builds an application that bases its answers on identified evidence and checks each figure against its source. + +## Who the session is for + +Researchers and analysts receive figures from applications of this kind and need to know whether each figure can be traced to a source. The session shows them which questions to ask of an answer. People who maintain collections of documents or data affect what an application can find through the metadata, dates, and structure of their documents, and notebooks 02 and 03 show that effect. Managers and reviewers who decide whether an application is ready for use need to examine how it works. The session divides an application into components that can be examined one at a time. Developers and data scientists build the software that decides which material reaches the model, and the notebooks implement each pattern with LangChain, LangGraph, FastMCP, and Pydantic. + +## The application around the model + +An AI chatbot consists of a language model and the software around it, which the session calls the application. The application decides which material reaches the model with each question, and it processes the answer afterwards. The session builds such an application in stages and adds one component in each notebook. + +### Grounding + +An answer is grounded when its statements and figures come from evidence supplied by the application. In notebook 01, background information is written into the prompt. Notebooks 02 and 03 add passages retrieved from a collection of documents, and notebook 04 obtains exact figures from a database of indicators through a tool. + +### Tracing failures to components + +A wrong answer can usually be traced to one component, and the correction depends on which component failed. In notebook 03, a 2016 summary ranks highly for a question about the latest figures, because the similarity score ignores dates. A filter on the publication date corrects the ranking. Notebook 04 shows a request for data that do not exist. In notebook 05, a limit on the number of steps stops an agent that keeps calling tools. + +### Separating writing from checking + +A grounded answer can still contain a wrong figure, for instance a value altered in copying. In notebook 06, the model tags each figure with its source record, and a verifier written in ordinary code compares the figure with that record. The verifier marks a figure as verified when it matches and flags it when it differs. A figure without a tag receives no mark. The reader still judges whether a record is the right one for the question. + +## Readers at different technical levels + +Readers without programming experience can follow the session from the explanations and the outputs published on this site. The notebooks run in Google Colab or any Python environment. An API key for a language model is optional. Without a key, the notebooks replay recorded answers from the model and run all other steps in full. + +## The notebooks + +- **01 · From a language model to an application.** A model answers the same question without context and then with background information in the prompt, and the notebook measures the cost of that context in tokens. +- **02 · Preparing knowledge for an application.** Eight documents receive metadata and are divided into chunks and embedded. A badly prepared copy of the collection shows the effect of poor preparation on a search. +- **03 · Retrieval-augmented generation.** The application retrieves passages for each question and passes them to the model. A filter on dates removes an outdated passage, and a test checks the search on eight questions without the model. +- **04 · Tools and structured outputs.** The model requests exact figures through a tool, and the answer is returned in a fixed structure. The last section offers the tool through the Model Context Protocol (MCP). +- **05 · Workflows and agents.** The application is built once as a workflow, with the order of the steps fixed in code, and once as an agent that chooses its own steps. A table compares the two designs. +- **06 · Verification.** Each figure is tagged with its source record and checked by a verifier, and the notebook describes the limits of that check. +- **07 · Lab: chat with your documents.** Participants build a chat application over documents of their own choice. diff --git a/training/session-01-building-ai-applications/resources/README.md b/training/session-01-building-ai-applications/resources/README.md index 10d3c21b..3f36613a 100644 --- a/training/session-01-building-ai-applications/resources/README.md +++ b/training/session-01-building-ai-applications/resources/README.md @@ -57,5 +57,5 @@ Each notebook ends with one exercise. In recorded mode some exercises ask for a ## Further reading - [Glossary](glossary.md) -- [Testing on fixed questions](evaluation-examples.md): the eight questions of the session, and how an application is tested against them +- [Testing on fixed questions](testing-on-fixed-questions.md): the eight questions of the session, and how an application is tested against them - [Next steps](next-steps.md): where to go after the session, by role diff --git a/training/session-01-building-ai-applications/resources/next-steps.md b/training/session-01-building-ai-applications/resources/next-steps.md index 19e27bce..72cb3f25 100644 --- a/training/session-01-building-ai-applications/resources/next-steps.md +++ b/training/session-01-building-ai-applications/resources/next-steps.md @@ -48,4 +48,8 @@ An application that cannot answer these is a demonstration. ## Later sessions in the series -To be announced. Suggested topics from this session's questions are collected in the session README of the next session when it is created. +The [series page](https://worldbank.github.io/ai4data/training/) lists the planned sessions. Three of them take up topics that this session introduces. + +- **Session 03 · Evaluating AI systems.** Notebook 03, section 6, tests the search on fixed questions, and notebook 06 leaves tests of a whole application out of scope. Session 03 covers the testing of complete applications. +- **Session 05 · Building trustworthy AI applications.** This session extends the verification of figures in notebook 06. +- **Session 06 · All about context: Why AI is really a search problem.** This session returns to the preparation of documents in notebook 02 and the selection of passages in notebook 03. diff --git a/training/session-01-building-ai-applications/resources/evaluation-examples.md b/training/session-01-building-ai-applications/resources/testing-on-fixed-questions.md similarity index 100% rename from training/session-01-building-ai-applications/resources/evaluation-examples.md rename to training/session-01-building-ai-applications/resources/testing-on-fixed-questions.md diff --git a/training/session-01-building-ai-applications/slides/README.md b/training/session-01-building-ai-applications/slides/README.md index 40c6dbd1..dd95fff9 100644 --- a/training/session-01-building-ai-applications/slides/README.md +++ b/training/session-01-building-ai-applications/slides/README.md @@ -3,6 +3,7 @@ - `deck.pptx`: the deck, 20 slides. - `outline.md`: the text of every slide with its demonstration pointer and speaker notes. This is the file to review and comment on. - `build_deck.js`: the source of both. Edit the script, then regenerate. +- `canva.json`: the public view link of the deck presented in the session, which is kept in Canva. The docs site shows it on the session's Slides page in Canva's embedded viewer. ## Regenerating diff --git a/training/session-01-building-ai-applications/slides/canva.json b/training/session-01-building-ai-applications/slides/canva.json new file mode 100644 index 00000000..5b7bf9eb --- /dev/null +++ b/training/session-01-building-ai-applications/slides/canva.json @@ -0,0 +1,4 @@ +{ + "title": "Slides", + "view_url": "https://www.canva.com/design/DAHYlQWlsiE/ifr1TPS8BJBhOssSUuzc6A/view?utm_content=DAHYlQWlsiE&utm_campaign=designshare&utm_medium=link2&utm_source=uniquelinks&utlId=ha8f47a4a1c" +} diff --git a/website/README.md b/website/README.md index bce6eb2f..f7dd7f68 100644 --- a/website/README.md +++ b/website/README.md @@ -26,6 +26,18 @@ npm run build This command generates static content into the `build` directory and can be served using any static contents hosting service. +## Changing or removing a page + +Every page route the site has published is listed in `published-routes.txt`, and each one keeps working after a page moves. A plugin, `plugins/published-routes.mjs`, checks this after every build. + +- **Moving or renaming a page.** An entry in `redirects.json` maps the old route to the new one, for example `{"from": "/training/session-01/documents.html", "to": "/training/session-01/documents"}`. Routes are written without the base URL. The build writes a redirect page at the old route, and the redirect keeps any `#anchor` of the old link. +- **Adding a page.** After `npm run build`, `npm run routes:update` adds the routes of the build to `published-routes.txt`, and the updated file is committed with the change. +- **Retiring a page without a successor.** Its line is deleted from `published-routes.txt` in the same pull request, so that the removal is visible in review. + +The build fails when a published route has neither a page nor a redirect, and when a redirect points to a route that is not a page. A new route that is missing from `published-routes.txt` is a warning in a local build and an error in CI. Pull requests run the same build in the "Publish to GitHub Pages" workflow, without deploying. + +The local preview server (`npm run serve`) rewrites addresses that end in `.html`, so redirects from `.html` routes are tested with a plain static server, for example `python3 -m http.server` in a folder where `ai4data` links to `build`. + ## Deployment Using SSH: diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index d37ae761..d4cdc398 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -4,6 +4,7 @@ import path from 'node:path'; import {fileURLToPath} from 'node:url'; import {themes as prismThemes} from 'prism-react-renderer'; import codeImport from 'remark-code-import'; +import publishedRoutes from './plugins/published-routes.mjs'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const repoRoot = path.resolve(__dirname, '..'); @@ -76,7 +77,10 @@ const config = { }, }, themes: ['@docusaurus/theme-mermaid'], - clientModules: [path.join(__dirname, 'src/clientModules/fonts.js')], + clientModules: [ + path.join(__dirname, 'src/clientModules/fonts.js'), + path.join(__dirname, 'src/clientModules/pcnDetail.js'), + ], i18n: { defaultLocale: 'en', @@ -111,7 +115,14 @@ const config = { ], }, theme: { - customCss: './src/css/custom.css', + // The second stylesheet holds the styles of the training notebook outputs and is + // written by scripts/docs/build_training_site.py before the build. + customCss: [ + './src/css/custom.css', + ...(fs.existsSync(path.join(__dirname, 'generated/training-outputs.css')) + ? ['./generated/training-outputs.css'] + : []), + ], }, }), ], @@ -160,6 +171,12 @@ const config = { position: 'left', activeBaseRegex: '/cookbook/', }, + { + to: '/training/', + label: 'Training', + position: 'left', + activeBaseRegex: '/training/', + }, { type: 'docSidebar', sidebarId: 'docsSidebar', @@ -241,6 +258,7 @@ const config = { plugins: [ hideCookbookFiles, + publishedRoutes, [ '@docusaurus/plugin-content-docs', /** @type {import('@docusaurus/plugin-content-docs').Options} */ @@ -271,6 +289,17 @@ const config = { editUrl: 'https://github.com/worldbank/ai4data/edit/main/cookbook/', }), ], + [ + '@docusaurus/plugin-content-docs', + /** @type {import('@docusaurus/plugin-content-docs').Options} */ + ({ + // Generated from training/ by scripts/docs/build_training_site.py. + id: 'training', + path: 'generated/training', + routeBasePath: 'training', + sidebarPath: path.join(__dirname, 'sidebars-training.js'), + }), + ], [ '@easyops-cn/docusaurus-search-local', /** @type {import('@easyops-cn/docusaurus-search-local').PluginOptions} */ @@ -279,7 +308,7 @@ const config = { indexDocs: true, indexBlog: false, indexPages: true, - docsRouteBasePath: ['/docs', '/pift-toolkit', '/cookbook'], + docsRouteBasePath: ['/docs', '/pift-toolkit', '/cookbook', '/training'], // Unlisted pages and hidden cookbooks stay out of the site search. ignoreFiles: [ 'modern-nso', diff --git a/website/package.json b/website/package.json index 6155c74f..98dade08 100644 --- a/website/package.json +++ b/website/package.json @@ -4,9 +4,10 @@ "private": true, "scripts": { "docusaurus": "docusaurus", - "convert-notebooks": "python3 ../scripts/docs/convert_notebooks.py && python3 ../scripts/docs/build_training_pages.py", + "convert-notebooks": "python3 ../scripts/docs/convert_notebooks.py && python3 ../scripts/docs/build_training_site.py", "start": "npm run convert-notebooks && docusaurus start", "build": "npm run convert-notebooks && docusaurus build", + "routes:update": "node plugins/published-routes.mjs --update", "swizzle": "docusaurus swizzle", "deploy": "docusaurus deploy", "clear": "docusaurus clear", diff --git a/website/plugins/published-routes.mjs b/website/plugins/published-routes.mjs new file mode 100644 index 00000000..1618e41b --- /dev/null +++ b/website/plugins/published-routes.mjs @@ -0,0 +1,107 @@ +// Keeps published addresses of the site working. +// +// published-routes.txt lists every page route the site has published. redirects.json maps +// an old route to its new one. After each build, this plugin writes a redirect page at +// every old route and fails the build when a published route has neither a page nor a +// redirect. A built route that is missing from published-routes.txt fails the build in CI +// and is a warning locally; `npm run routes:update` adds the routes of the last build. +// See website/README.md, "Changing or removing a page". +import fs from 'node:fs'; +import path from 'node:path'; +import {fileURLToPath} from 'node:url'; + +const siteDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const ROUTES_FILE = path.join(siteDir, 'published-routes.txt'); +const REDIRECTS_FILE = path.join(siteDir, 'redirects.json'); + +// Routes of the HTML pages in a build folder, without the base URL: /x for x/index.html, +// /x.html for any other HTML file. The 404 page and the redirect pages written here are +// not routes. +export function builtRoutes(outDir) { + const routes = new Set(); + const walk = (dir) => { + for (const entry of fs.readdirSync(dir, {withFileTypes: true})) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) walk(full); + else if (entry.name.endsWith('.html')) { + const rel = path.relative(outDir, full).split(path.sep).join('/'); + if (rel === '404.html' || fs.readFileSync(full, 'utf8').includes('content="published-routes redirect"')) continue; + routes.add(rel.endsWith('index.html') ? '/' + rel.slice(0, -'index.html'.length).replace(/\/$/, '') : '/' + rel); + } + } + }; + walk(outDir); + return routes; +} + +function readRoutes() { + if (!fs.existsSync(ROUTES_FILE)) return new Set(); + return new Set( + fs.readFileSync(ROUTES_FILE, 'utf8').split('\n').map((l) => l.trim()).filter((l) => l && !l.startsWith('#')), + ); +} + +function writeRoutes(routes) { + const header = + '# Page routes the site has published, without the base URL. Each one must stay a page\n' + + '# or have an entry in redirects.json. `npm run routes:update` adds the routes of a build;\n' + + '# a route is removed from this file only when the page is retired without a successor.\n'; + fs.writeFileSync(ROUTES_FILE, header + [...routes].sort().join('\n') + '\n'); +} + +function redirectPage(target) { + const t = target.replace(/"/g, '"'); + return `<!doctype html> +<html lang="en"><head><meta charset="utf-8"> +<title>Moved + + + + + +

This page has moved to ${t}.

+`; +} + +export default function publishedRoutes(context) { + return { + name: 'published-routes', + async postBuild({outDir}) { + const base = context.baseUrl.replace(/\/$/, ''); + const built = builtRoutes(outDir); + const redirects = fs.existsSync(REDIRECTS_FILE) ? JSON.parse(fs.readFileSync(REDIRECTS_FILE, 'utf8')) : []; + const errors = []; + const redirected = new Set(); + for (const {from, to} of redirects) { + const [toPath] = to.split('#'); + if (built.has(from)) errors.push(`redirects.json: ${from} is still a page, so it cannot redirect`); + else if (!built.has(toPath.replace(/\/$/, '') || '/')) errors.push(`redirects.json: ${from} redirects to ${to}, which is not a page`); + else { + const file = from.endsWith('.html') ? path.join(outDir, from) : path.join(outDir, from, 'index.html'); + fs.mkdirSync(path.dirname(file), {recursive: true}); + fs.writeFileSync(file, redirectPage(base + to)); + redirected.add(from); + } + } + const published = readRoutes(); + const missing = [...published].filter((r) => !built.has(r) && !redirected.has(r)).sort(); + for (const r of missing) errors.push(`${r} was published and is no longer a page; add a redirect in redirects.json`); + const unlisted = [...built].filter((r) => !published.has(r)).sort(); + if (unlisted.length) { + const message = `${unlisted.length} new route(s) are not in published-routes.txt; run \`npm run routes:update\`:\n ` + unlisted.join('\n '); + if (process.env.CI) errors.push(message); + else console.warn(`[published-routes] ${message}`); + } + if (errors.length) throw new Error('[published-routes]\n ' + errors.join('\n ')); + console.log(`[published-routes] ${built.size} pages, ${redirected.size} redirects, all published routes resolve`); + }, + }; +} + +// node plugins/published-routes.mjs --update [build folder] +if (process.argv[1] === fileURLToPath(import.meta.url) && process.argv[2] === '--update') { + const outDir = path.resolve(process.argv[3] || path.join(siteDir, 'build')); + const routes = new Set([...readRoutes(), ...builtRoutes(outDir)]); + writeRoutes(routes); + console.log(`published-routes.txt: ${routes.size} routes`); +} diff --git a/website/published-routes.txt b/website/published-routes.txt new file mode 100644 index 00000000..7c33479a --- /dev/null +++ b/website/published-routes.txt @@ -0,0 +1,261 @@ +# Page routes the site has published, without the base URL. Each one must stay a page +# or have an entry in redirects.json. `npm run routes:update` adds the routes of a build; +# a route is removed from this file only when the page is retired without a successor. +/ +/ai-readiness-assessment +/ai-readiness-in-practice +/assessment-framework +/cookbook +/cookbook/ai-assistants-at-work +/cookbook/ai-assistants-at-work/asking +/cookbook/ai-assistants-at-work/checking +/cookbook/ai-assistants-at-work/contributing +/cookbook/ai-assistants-at-work/glossary +/cookbook/ai-assistants-at-work/inputs +/cookbook/ai-assistants-at-work/record +/cookbook/ai-assistants-at-work/standards +/cookbook/ai-assistants-at-work/start-here +/cookbook/ai-assistants-at-work/tasks +/cookbook/ai-assistants-at-work/unit +/cookbook/ai-ready-dissemination +/cookbook/ai-ready-dissemination/ask +/cookbook/ai-ready-dissemination/contribute +/cookbook/ai-ready-dissemination/data-types +/cookbook/ai-ready-dissemination/evaluate +/cookbook/ai-ready-dissemination/find +/cookbook/ai-ready-dissemination/glossary +/cookbook/ai-ready-dissemination/govern +/cookbook/ai-ready-dissemination/monitor-use +/cookbook/ai-ready-dissemination/retrieve +/cookbook/ai-ready-dissemination/standards +/cookbook/ai-ready-dissemination/start-here +/cookbook/ai-ready-dissemination/sustain +/cookbook/ai-ready-dissemination/trust +/cookbook/ai-ready-dissemination/understand +/cookbook/assessment-to-pathway +/cookbook/assessment-to-pathway/contributing +/cookbook/assessment-to-pathway/effort +/cookbook/assessment-to-pathway/evidence +/cookbook/assessment-to-pathway/glossary +/cookbook/assessment-to-pathway/pathway +/cookbook/assessment-to-pathway/profile +/cookbook/assessment-to-pathway/reassess +/cookbook/assessment-to-pathway/sequence +/cookbook/assessment-to-pathway/standards +/cookbook/assessment-to-pathway/start-here +/cookbook/assessment-to-pathway/targets +/cookbook/authoring +/cookbook/data-from-documents +/cookbook/data-from-documents/cite +/cookbook/data-from-documents/contributing +/cookbook/data-from-documents/extract +/cookbook/data-from-documents/glossary +/cookbook/data-from-documents/inventory +/cookbook/data-from-documents/locate +/cookbook/data-from-documents/operate +/cookbook/data-from-documents/publish +/cookbook/data-from-documents/scanned +/cookbook/data-from-documents/series +/cookbook/data-from-documents/standards +/cookbook/data-from-documents/start-here +/cookbook/data-from-documents/verify +/cookbook/evaluation-suites +/cookbook/evaluation-suites/answers +/cookbook/evaluation-suites/contributing +/cookbook/evaluation-suites/extraction +/cookbook/evaluation-suites/gates +/cookbook/evaluation-suites/glossary +/cookbook/evaluation-suites/inventory +/cookbook/evaluation-suites/judges +/cookbook/evaluation-suites/production +/cookbook/evaluation-suites/questions +/cookbook/evaluation-suites/reporting +/cookbook/evaluation-suites/retrieval +/cookbook/evaluation-suites/standards +/cookbook/evaluation-suites/start-here +/cookbook/governance-and-risk +/cookbook/governance-and-risk/contributing +/cookbook/governance-and-risk/decisions +/cookbook/governance-and-risk/glossary +/cookbook/governance-and-risk/licensing +/cookbook/governance-and-risk/position +/cookbook/governance-and-risk/providers +/cookbook/governance-and-risk/register +/cookbook/governance-and-risk/review +/cookbook/governance-and-risk/risk +/cookbook/governance-and-risk/standards +/cookbook/governance-and-risk/start-here +/cookbook/governance-and-risk/statement +/cookbook/language-models-in-production +/cookbook/language-models-in-production/assurance +/cookbook/language-models-in-production/coding +/cookbook/language-models-in-production/commentary +/cookbook/language-models-in-production/contributing +/cookbook/language-models-in-production/design +/cookbook/language-models-in-production/editing +/cookbook/language-models-in-production/glossary +/cookbook/language-models-in-production/map +/cookbook/language-models-in-production/running +/cookbook/language-models-in-production/standards +/cookbook/language-models-in-production/start-here +/cookbook/metadata-curation-with-llms +/cookbook/metadata-curation-with-llms/assess +/cookbook/metadata-curation-with-llms/contributing +/cookbook/metadata-curation-with-llms/draft +/cookbook/metadata-curation-with-llms/glossary +/cookbook/metadata-curation-with-llms/improve +/cookbook/metadata-curation-with-llms/migrate +/cookbook/metadata-curation-with-llms/review +/cookbook/metadata-curation-with-llms/scope +/cookbook/metadata-curation-with-llms/standardize +/cookbook/metadata-curation-with-llms/standards +/cookbook/metadata-curation-with-llms/start-here +/cookbook/metadata-curation-with-llms/translate +/cookbook/microdata-documentation +/cookbook/microdata-documentation/access +/cookbook/microdata-documentation/concepts +/cookbook/microdata-documentation/contributing +/cookbook/microdata-documentation/find +/cookbook/microdata-documentation/fitness +/cookbook/microdata-documentation/glossary +/cookbook/microdata-documentation/navigate +/cookbook/microdata-documentation/produce +/cookbook/microdata-documentation/rounds +/cookbook/microdata-documentation/standards +/cookbook/microdata-documentation/start-here +/cookbook/microdata-documentation/variables +/cookbook/ml-ready-datasets +/cookbook/ml-ready-datasets/contributing +/cookbook/ml-ready-datasets/describe +/cookbook/ml-ready-datasets/document +/cookbook/ml-ready-datasets/glossary +/cookbook/ml-ready-datasets/license +/cookbook/ml-ready-datasets/purpose +/cookbook/ml-ready-datasets/represent +/cookbook/ml-ready-datasets/shape +/cookbook/ml-ready-datasets/splits +/cookbook/ml-ready-datasets/standards +/cookbook/ml-ready-datasets/start-here +/cookbook/ml-ready-datasets/version +/cookbook/monitoring-data-use +/cookbook/monitoring-data-use/access +/cookbook/monitoring-data-use/act +/cookbook/monitoring-data-use/citations +/cookbook/monitoring-data-use/collect +/cookbook/monitoring-data-use/collect-web +/cookbook/monitoring-data-use/contributing +/cookbook/monitoring-data-use/define +/cookbook/monitoring-data-use/detect +/cookbook/monitoring-data-use/glossary +/cookbook/monitoring-data-use/harmonize +/cookbook/monitoring-data-use/report +/cookbook/monitoring-data-use/standards +/cookbook/monitoring-data-use/start-here +/cookbook/serving-statistics-to-agents +/cookbook/serving-statistics-to-agents/access +/cookbook/serving-statistics-to-agents/context +/cookbook/serving-statistics-to-agents/contributing +/cookbook/serving-statistics-to-agents/distribute +/cookbook/serving-statistics-to-agents/evaluate +/cookbook/serving-statistics-to-agents/glossary +/cookbook/serving-statistics-to-agents/operate +/cookbook/serving-statistics-to-agents/provenance +/cookbook/serving-statistics-to-agents/purpose +/cookbook/serving-statistics-to-agents/safety +/cookbook/serving-statistics-to-agents/standards +/cookbook/serving-statistics-to-agents/start-here +/cookbook/serving-statistics-to-agents/tools +/cookbook/small-and-open-models +/cookbook/small-and-open-models/adaptation +/cookbook/small-and-open-models/candidates +/cookbook/small-and-open-models/comparison +/cookbook/small-and-open-models/contributing +/cookbook/small-and-open-models/cost +/cookbook/small-and-open-models/glossary +/cookbook/small-and-open-models/security +/cookbook/small-and-open-models/serving +/cookbook/small-and-open-models/standards +/cookbook/small-and-open-models/start-here +/cookbook/small-and-open-models/when +/cookbook/synthetic-data-for-sharing +/cookbook/synthetic-data-for-sharing/contributing +/cookbook/synthetic-data-for-sharing/glossary +/cookbook/synthetic-data-for-sharing/method +/cookbook/synthetic-data-for-sharing/purpose +/cookbook/synthetic-data-for-sharing/relational +/cookbook/synthetic-data-for-sharing/release +/cookbook/synthetic-data-for-sharing/risk +/cookbook/synthetic-data-for-sharing/standards +/cookbook/synthetic-data-for-sharing/start-here +/cookbook/synthetic-data-for-sharing/utility +/cookbook/synthetic-data-for-sharing/workflow +/docs/CODE_OF_CONDUCT +/docs/CONTRIBUTING +/docs/ai-ready-framework +/docs/anomaly-detection +/docs/anomaly-detection/feedback-system +/docs/anomaly/explanation +/docs/anomaly/explanation/elicitation-pipeline +/docs/anomaly/explanation/motivation +/docs/data-discoverability +/docs/data-discoverability/embedding-fine-tuning +/docs/data_use +/docs/gsbpm-mapping +/docs/inclusive-ai +/docs/introduction +/docs/mcp +/docs/mcp/mcp-for-official-statistics +/docs/metadata-augmentation +/docs/metadata-augmentation/methodology +/docs/metadata-quality/generative-ai-for-metadata-quality +/docs/metadata-reviewer/agentic-approach +/docs/metadata-reviewer/implementation +/docs/metadata-reviewer/overview +/docs/metadata-reviewer/user-manual +/docs/metadata-reviewer/user-manual/advanced-usage +/docs/metadata-reviewer/user-manual/appendix-agents-manifest +/docs/metadata-reviewer/user-manual/appendix-api-reference +/docs/metadata-reviewer/user-manual/appendix-glossary +/docs/metadata-reviewer/user-manual/client-api +/docs/metadata-reviewer/user-manual/core-concepts +/docs/metadata-reviewer/user-manual/end-to-end-workflow +/docs/metadata-reviewer/user-manual/installation +/docs/metadata-reviewer/user-manual/introduction +/docs/metadata-reviewer/user-manual/jobs +/docs/metadata-reviewer/user-manual/quick-start +/docs/metadata-reviewer/user-manual/review-board +/docs/metadata-reviewer/user-manual/troubleshooting +/docs/notebooks/data-anomaly/timeseries-anomaly-explanation-with-llms +/docs/notebooks/metadata-quality-assessment-with-llm +/docs/partnerships +/docs/pcn +/docs/reference-implementation +/docs/repo-structure +/hub +/modern-nso +/nso-roadmap +/pift-toolkit/configuration +/pift-toolkit/deployment +/pift-toolkit/method +/pift-toolkit/pipeline +/search +/tools +/training +/training/session-01 +/training/session-01/documents +/training/session-01/documents.html +/training/session-01/notebooks/01-from-a-language-model-to-an-application +/training/session-01/notebooks/02-preparing-knowledge-for-an-application +/training/session-01/notebooks/03-retrieval-augmented-generation +/training/session-01/notebooks/04-tools-and-structured-outputs +/training/session-01/notebooks/05-workflows-and-agents +/training/session-01/notebooks/06-verification +/training/session-01/notebooks/07-lab-chat-with-your-documents +/training/session-01/preface +/training/session-01/resources +/training/session-01/resources/glossary +/training/session-01/resources/next-steps +/training/session-01/resources/testing-on-fixed-questions +/training/session-01/slides +/training/session-01/worked-example-data +/unlisted diff --git a/website/redirects.json b/website/redirects.json new file mode 100644 index 00000000..7c24b0b9 --- /dev/null +++ b/website/redirects.json @@ -0,0 +1,3 @@ +[ + {"from": "/training/session-01/documents.html", "to": "/training/session-01/documents"} +] diff --git a/website/sidebars-training.js b/website/sidebars-training.js new file mode 100644 index 00000000..596f768e --- /dev/null +++ b/website/sidebars-training.js @@ -0,0 +1,12 @@ +// @ts-check + +// The Training section is generated from training/ by +// scripts/docs/build_training_site.py; its order comes from the generated +// front matter and _category_.json files. + +/** @type {import('@docusaurus/plugin-content-docs').SidebarsConfig} */ +const sidebars = { + training: [{type: 'autogenerated', dirName: '.'}], +}; + +export default sidebars; diff --git a/website/src/clientModules/pcnDetail.js b/website/src/clientModules/pcnDetail.js new file mode 100644 index 00000000..75b77025 --- /dev/null +++ b/website/src/clientModules/pcnDetail.js @@ -0,0 +1,28 @@ +// Fills the detail panel under a verified chat answer on the training notebook +// pages. In Jupyter an inline script does this; the site removes inline scripts +// from notebook outputs, so the same behaviour is attached here after each route. + +function wire() { + document.querySelectorAll('[id^="pcn-chat-"]').forEach((card) => { + if (card.dataset.pcnWired) return; + card.dataset.pcnWired = '1'; + const panel = card.querySelector('.pcn-detail'); + if (!panel) return; + card.querySelectorAll('.pcn-mark').forEach((mark) => { + const tip = mark.querySelector('.pcn-tip'); + if (!tip) return; + const show = () => { + panel.innerHTML = tip.innerHTML; + panel.style.color = '#16315a'; + }; + mark.addEventListener('mouseenter', show); + mark.addEventListener('focus', show); + }); + }); +} + +export function onRouteDidUpdate() { + if (typeof document !== 'undefined') { + window.requestAnimationFrame(wire); + } +} diff --git a/website/src/components/training/DocumentViewer/index.js b/website/src/components/training/DocumentViewer/index.js new file mode 100644 index 00000000..c44b164b --- /dev/null +++ b/website/src/components/training/DocumentViewer/index.js @@ -0,0 +1,92 @@ +import React, {useEffect, useState} from 'react'; +import clsx from 'clsx'; +import styles from './styles.module.css'; + +// The worked-example documents of a training session, grouped by their role in the +// session. The list selects a document; the sheet shows its metadata and its chunks, +// each labelled with the citation the application writes for it. The selection is +// kept in the URL hash so that a document can be linked to. +export default function DocumentViewer({data}) { + const ids = data.groups.flatMap((g) => g.documents.map((d) => d.id)); + const [current, setCurrent] = useState(ids[0]); + + useEffect(() => { + const fromHash = () => { + const id = window.location.hash.replace('#', ''); + if (ids.includes(id)) setCurrent(id); + }; + fromHash(); + window.addEventListener('hashchange', fromHash); + return () => window.removeEventListener('hashchange', fromHash); + }, []); + + const select = (id) => { + setCurrent(id); + try { + window.history.replaceState(null, '', `#${id}`); + } catch (e) {} + }; + + return ( +
+ +
+ {data.groups.flatMap((g) => + g.documents.map((d) => ( + + )), + )} +
+
+ ); +} diff --git a/website/src/components/training/DocumentViewer/styles.module.css b/website/src/components/training/DocumentViewer/styles.module.css new file mode 100644 index 00000000..abe9f749 --- /dev/null +++ b/website/src/components/training/DocumentViewer/styles.module.css @@ -0,0 +1,246 @@ +/* Colours follow the site theme (Infima variables); the role chips carry their own + light and dark pairs. */ +.viewer { + --chip-evidence: #e3eefa; + --chip-evidence-fg: #0058b0; + --chip-definition: #ddf4f0; + --chip-definition-fg: #0a7a6e; + --chip-outdated: #f3e9da; + --chip-outdated-fg: #7a4a0c; + --chip-distractor: #eceef1; + --chip-distractor-fg: #565f68; + display: grid; + grid-template-columns: 260px minmax(0, 1fr); + gap: 24px; + align-items: start; + margin: 1.5rem 0; +} + +:global([data-theme='dark']) .viewer { + --chip-evidence: #1e3a5f; + --chip-evidence-fg: #cfe1fa; + --chip-definition: #164840; + --chip-definition-fg: #c7f0e9; + --chip-outdated: #4a3614; + --chip-outdated-fg: #f3ddb4; + --chip-distractor: #2e353d; + --chip-distractor-fg: #d5dae0; +} + +.list { + position: sticky; + top: calc(var(--ifm-navbar-height) + 16px); + display: flex; + flex-direction: column; + gap: 16px; +} + +.list ul { + list-style: none; + margin: 0; + padding: 0; + display: flex; + flex-direction: column; + gap: 4px; +} + +.list li + li { + margin-top: 0; +} + +.groupHeading { + margin: 0 0 4px; +} + +.groupText { + margin: 0 0 8px; + font-size: 0.8rem; + line-height: 1.4; + color: var(--ifm-color-emphasis-700); +} + +.doc { + width: 100%; + text-align: left; + font: inherit; + color: var(--ifm-font-color-base); + background: transparent; + border: 1px solid transparent; + border-radius: 8px; + padding: 6px 10px; + display: flex; + flex-direction: column; + gap: 2px; + cursor: pointer; +} + +.doc:hover { + border-color: var(--ifm-color-emphasis-300); + background: var(--ifm-background-surface-color); +} + +.doc[aria-current='true'] { + background: var(--ifm-background-surface-color); + border-color: var(--ifm-color-primary); + box-shadow: inset 3px 0 0 var(--ifm-color-primary); +} + +.doc:focus-visible { + outline: 2px solid var(--ifm-color-primary); + outline-offset: 2px; +} + +.docTitle { + font-size: 0.9rem; + font-weight: 500; + line-height: 1.3; +} + +.docId { + font-family: var(--ifm-font-family-monospace); + font-size: 0.72rem; + color: var(--ifm-color-emphasis-600); +} + +.chip { + display: inline-block; + font-size: 0.68rem; + font-weight: 600; + letter-spacing: 0.06em; + text-transform: uppercase; + padding: 2px 8px; + border-radius: 999px; +} + +.evidence { background: var(--chip-evidence); color: var(--chip-evidence-fg); } +.definition { background: var(--chip-definition); color: var(--chip-definition-fg); } +.outdated { background: var(--chip-outdated); color: var(--chip-outdated-fg); } +.distractor { background: var(--chip-distractor); color: var(--chip-distractor-fg); } + +.sheet { + border: 1px solid var(--ifm-color-emphasis-300); + border-radius: 12px; + padding: clamp(16px, 3vw, 28px); + background: var(--ifm-background-surface-color); + min-width: 0; +} + +.role { + margin: 0 0 12px; + font-size: 0.85rem; + color: var(--ifm-color-emphasis-700); + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 8px; +} + +.meta { + margin: 0 0 16px; + border-top: 1px solid var(--ifm-color-emphasis-300); + border-bottom: 1px solid var(--ifm-color-emphasis-300); +} + +.row { + display: grid; + grid-template-columns: 80px minmax(0, 1fr); + gap: 12px; + padding: 5px 0; + border-bottom: 1px dashed var(--ifm-color-emphasis-300); + font-size: 0.82rem; +} + +.row:last-child { + border-bottom: 0; +} + +.row dt { + font-family: var(--ifm-font-family-monospace); + font-weight: 400; + font-size: 0.75rem; + color: var(--ifm-color-emphasis-600); + padding-top: 2px; + margin: 0; +} + +.row dd { + margin: 0; + min-width: 0; + overflow-wrap: anywhere; +} + +.chunkNote { + margin: 0 0 8px; + font-size: 0.82rem; + color: var(--ifm-color-emphasis-700); +} + +.chunk { + display: grid; + gap: 4px; + padding: 12px 0 12px 16px; + border-left: 3px solid var(--ifm-color-primary); + margin: 10px 0; +} + +.cite { + display: flex; + flex-wrap: wrap; + align-items: baseline; + gap: 10px; + font-size: 0.75rem; + color: var(--ifm-color-emphasis-700); +} + +.cite code { + font-size: 0.75rem; + color: var(--ifm-color-primary-darker); + background: transparent; + border: 0; + padding: 0; +} + +:global([data-theme='dark']) .cite code { + color: var(--ifm-color-primary-lighter); +} + +.prose { + font-size: 0.95rem; + line-height: 1.6; + max-width: 70ch; + min-width: 0; +} + +.prose h2 { + font-size: 1.3rem; + margin: 0 0 8px; +} + +.prose h3 { + font-size: 1.05rem; + margin: 0 0 6px; +} + +.prose p { + margin: 0 0 10px; +} + +.prose p:last-child, +.prose ul:last-child { + margin-bottom: 0; +} + +.prose ul { + margin: 0 0 10px; + padding-left: 22px; +} + +@media (max-width: 996px) { + .viewer { + grid-template-columns: 1fr; + } + + .list { + position: static; + } +} diff --git a/website/src/css/custom.css b/website/src/css/custom.css index 6a4d76c3..c279ede7 100644 --- a/website/src/css/custom.css +++ b/website/src/css/custom.css @@ -381,3 +381,40 @@ div[class*='codeBlockContainer'] { height: 38px; width: auto; } + +/* ---------- Training notebook pages ---------- */ + +/* Generated by scripts/docs/build_training_site.py: the buttons at the top of a + notebook page and the HTML outputs of its cells. */ +.training-actions { + display: flex; + flex-wrap: wrap; + gap: 0.5rem; +} +.markdown .training-actions a.button--primary { + color: #ffffff; +} +.markdown .training-actions a.button--secondary { + color: var(--ifm-color-secondary-contrast-foreground); +} +.nb-output { + margin-bottom: var(--ifm-leading); + max-width: 100%; + overflow-x: auto; +} + +.training-slides { + aspect-ratio: 16 / 9; + width: 100%; + margin: 1rem 0 1.5rem; + border: 1px solid var(--ifm-color-emphasis-300); + border-radius: 8px; + overflow: hidden; +} + +.training-slides iframe { + display: block; + width: 100%; + height: 100%; + border: 0; +}