From 12a8e6d83efedb83680339f7328387fba9c1593f Mon Sep 17 00:00:00 2001 From: avsolatorio Date: Thu, 22 Oct 2026 12:57:03 +0000 Subject: [PATCH 1/4] Docs site: hide the microdata documentation cookbook; links into it from public pages become plain text --- .../ai-ready-dissemination/understand.mdx | 2 +- cookbook/evaluation-suites/retrieval.mdx | 2 +- .../language-models-in-production/design.mdx | 6 +- .../standardize.mdx | 2 +- cookbook/ml-ready-datasets/license.mdx | 6 +- cookbook/ml-ready-datasets/shape.mdx | 4 +- cookbook/ml-ready-datasets/version.mdx | 2 +- cookbook/monitoring-data-use/act.mdx | 4 +- .../serving-statistics-to-agents/access.mdx | 2 +- .../serving-statistics-to-agents/purpose.mdx | 2 +- .../serving-statistics-to-agents/safety.mdx | 2 +- cookbook/synthetic-data-for-sharing/index.mdx | 2 +- .../synthetic-data-for-sharing/purpose.mdx | 8 +- .../synthetic-data-for-sharing/relational.mdx | 2 +- .../synthetic-data-for-sharing/release.mdx | 2 +- cookbook/synthetic-data-for-sharing/risk.mdx | 4 +- .../synthetic-data-for-sharing/standards.md | 2 +- .../synthetic-data-for-sharing/workflow.mdx | 8 +- docs/metadata-augmentation/index.md | 2 +- docs/reference-implementation/index.md | 2 +- website/cookbook-visibility.json | 3 +- website/src/content/investment.json | 260 ++---------------- 22 files changed, 65 insertions(+), 264 deletions(-) diff --git a/cookbook/ai-ready-dissemination/understand.mdx b/cookbook/ai-ready-dissemination/understand.mdx index 41d553cb..303a0821 100644 --- a/cookbook/ai-ready-dissemination/understand.mdx +++ b/cookbook/ai-ready-dissemination/understand.mdx @@ -239,7 +239,7 @@ only the values file sees the flags without them. ## Resources - [SDMX Content-Oriented Guidelines](https://sdmx.org/?page_id=4345): cross-domain concepts and code lists to reuse. -- [DDI Codebook 2.5](https://ddialliance.org/Specification/DDI-Codebook/2.5/) and [DDI Lifecycle 3.3](https://ddialliance.org/Specification/DDI-Lifecycle/3.3/): variable-level documentation and reusable questions and concepts; the [microdata cookbook](/cookbook/microdata-documentation/) applies them. +- [DDI Codebook 2.5](https://ddialliance.org/Specification/DDI-Codebook/2.5/) and [DDI Lifecycle 3.3](https://ddialliance.org/Specification/DDI-Lifecycle/3.3/): variable-level documentation and reusable questions and concepts; the microdata cookbook applies them. - [UN classifications registry](https://unstats.un.org/unsd/classifications/): ISIC, CPC, COICOP, and correspondence tables; [ISCED](https://uis.unesco.org/en/topic/international-standard-classification-education-isced) and [ISCO](https://www.ilo.org/public/english/bureau/stat/isco/). - [SKOS](https://www.w3.org/TR/skos-reference/), [XKOS](https://ddialliance.org/xkos), and [GSIM](https://unece.org/statistics/modernstats/gsim): concept schemes, classifications, and the names of statistical information objects. - [SDMX `OBS_STATUS` code list](https://sdmx.org/?page_id=3215): the status codes for breaks, revisions, and estimates in recipe 3.3. diff --git a/cookbook/evaluation-suites/retrieval.mdx b/cookbook/evaluation-suites/retrieval.mdx index b11b5e78..a39e0765 100644 --- a/cookbook/evaluation-suites/retrieval.mdx +++ b/cookbook/evaluation-suites/retrieval.mdx @@ -215,7 +215,7 @@ puts them in. - The [PI-FT toolkit](/pift-toolkit/pipeline) reports Recall@k, MRR, and nDCG@k and compares embedding models on the same queries; its [method page](/pift-toolkit/method) describes graded evaluation. - [Data Discoverability](/docs/data-discoverability/) evaluates search by meaning over structured records with multilingual queries. -- The [microdata cookbook](/cookbook/microdata-documentation/navigate) applies the same measures to variable-level search. +- The microdata cookbook applies the same measures to variable-level search. ## Resources diff --git a/cookbook/language-models-in-production/design.mdx b/cookbook/language-models-in-production/design.mdx index 2b804d55..2ebab669 100644 --- a/cookbook/language-models-in-production/design.mdx +++ b/cookbook/language-models-in-production/design.mdx @@ -123,7 +123,7 @@ the model judges each one. a different concept. Ask for a confidence and a reason. 3. The methodologist confirms or rejects each proposal. A confirmed match links the new variable to the concept - ([microdata cookbook, chapter 4](/cookbook/microdata-documentation/concepts)) + (microdata cookbook, chapter 4) and inherits its definition. 4. Track the share of proposals confirmed at each level of confidence. Stop showing proposals below the level at which few are confirmed. @@ -143,7 +143,7 @@ definition, and the comparability note records the added category. **What this does not do.** A match says that two questions measure the same concept. It does not make their data comparable; that depends on the survey mode, the population, and the categories -([microdata cookbook, chapter 6](/cookbook/microdata-documentation/rounds)). +(microdata cookbook, chapter 6). @@ -208,7 +208,7 @@ No draft is published without review. - [Cross-Cultural Survey Guidelines](https://ccsg.isr.umich.edu/): questionnaire design, translation, and testing practice the checklist draws on. - [DDI Lifecycle 3.3](https://ddialliance.org/Specification/DDI-Lifecycle/3.3/): reusable questions and concepts, the form of a question bank. - [Drafting](/cookbook/metadata-curation-with-llms/draft) and [translation](/cookbook/metadata-curation-with-llms/translate) chapters of the curation cookbook: the workflows and checks for recipe 2.3. -- [Concepts chapter of the microdata cookbook](/cookbook/microdata-documentation/concepts): linking variables to concepts and classifications. +- Concepts chapter of the microdata cookbook: linking variables to concepts and classifications. ## Common mistakes diff --git a/cookbook/metadata-curation-with-llms/standardize.mdx b/cookbook/metadata-curation-with-llms/standardize.mdx index dd6f03bc..245d911d 100644 --- a/cookbook/metadata-curation-with-llms/standardize.mdx +++ b/cookbook/metadata-curation-with-llms/standardize.mdx @@ -180,7 +180,7 @@ filter for every user. - **Vocabulary format:** SKOS for concepts and labels, XKOS for classifications; a CSV with preferred label, URI, and alternates as the working form. - **Mapping:** exact and fuzzy string matching (this chapter); embeddings for semantic matching; a model for proposals with a confidence. - **Storage:** `keywords` and `topics` with vocabulary and URI in the World Bank schemas; the Metadata Editor's controlled lists. -- **International vocabularies:** the SDG indicator framework, the classifications of [chapter 4 of the microdata cookbook](/cookbook/microdata-documentation/concepts), and thematic thesauri, as sources of URIs. +- **International vocabularies:** the SDG indicator framework, the classifications of chapter 4 of the microdata cookbook, and thematic thesauri, as sources of URIs. ## World Bank examples diff --git a/cookbook/ml-ready-datasets/license.mdx b/cookbook/ml-ready-datasets/license.mdx index a7880fed..d08dcc5d 100644 --- a/cookbook/ml-ready-datasets/license.mdx +++ b/cookbook/ml-ready-datasets/license.mdx @@ -156,7 +156,7 @@ treatment of the source records is stated in the card. 1. Establish the source. A dataset built from a public use file inherits its disclosure control - ([microdata cookbook, chapter 8](/cookbook/microdata-documentation/access)). + (microdata cookbook, chapter 8). A dataset built from the full file is a new product and gets a new review, with the key variables, the uniqueness measures, and the recoding the unit requires. @@ -202,7 +202,7 @@ text is reviewed again when a new version adds records. ## World Bank examples - The World Bank's [terms of use for datasets](https://www.worldbank.org/en/about/legal/terms-of-use-for-datasets) default to CC BY 4.0, with the licence in the schema's `license` field and on every page. -- The [microdata cookbook's access chapter](/cookbook/microdata-documentation/access) gives the tiers, the disclosure control workflow, and the rules for automated and AI clients that a training dataset inherits. +- The microdata cookbook's access chapter gives the tiers, the disclosure control workflow, and the rules for automated and AI clients that a training dataset inherits. ## Resources @@ -210,7 +210,7 @@ text is reviewed again when a new version adds records. - [Understanding CC licenses and generative AI](https://creativecommons.org/2023/08/18/understanding-cc-licenses-and-generative-ai/) (Creative Commons, 2023): what the licences do and do not say about training. - [Responsible AI Licenses](https://www.licenses.ai/): the RAIL family, for comparison when the organization considers use restrictions. - [sdcMicro practice guide](https://sdcpractice.readthedocs.io/): the disclosure control workflow the unit applies to a new product. -- [Microdata cookbook, chapter 8](/cookbook/microdata-documentation/access): the access policy with tiers and the rules for AI clients. +- Microdata cookbook, chapter 8: the access policy with tiers and the rules for AI clients. ## Common mistakes diff --git a/cookbook/ml-ready-datasets/shape.mdx b/cookbook/ml-ready-datasets/shape.mdx index 5fa38395..b5875141 100644 --- a/cookbook/ml-ready-datasets/shape.mdx +++ b/cookbook/ml-ready-datasets/shape.mdx @@ -109,7 +109,7 @@ predicts without asking anyone. say how the dataset was made or what it covers, which is the card of [chapter 4](./document.mdx), and it does not check the data against the description, which the -[microdata cookbook's dictionary check](/cookbook/microdata-documentation/variables) +microdata cookbook's dictionary check does. @@ -225,7 +225,7 @@ as released. - [Apache Parquet](https://parquet.apache.org/): the columnar file format for large tabular datasets. - [Data Package standard](https://datapackage.org/): a container format for a dataset's files with a table schema per resource, and the `frictionless` tools that validate it. - [Tidy data layout in the dissemination cookbook](/cookbook/ai-ready-dissemination/retrieve): one row per observation, codes in the file, labels in the code lists. -- [Dictionary check of the microdata cookbook](/cookbook/microdata-documentation/variables): the check that a dictionary is complete and consistent. +- Dictionary check of the microdata cookbook: the check that a dictionary is complete and consistent. ## Common mistakes diff --git a/cookbook/ml-ready-datasets/version.mdx b/cookbook/ml-ready-datasets/version.mdx index e05751e2..71501249 100644 --- a/cookbook/ml-ready-datasets/version.mdx +++ b/cookbook/ml-ready-datasets/version.mdx @@ -231,7 +231,7 @@ terms ask for citation and the card makes it easy. - [DataCite metadata schema](https://schema.datacite.org/): the fields for a dataset version's DOI, including the relation to other versions. - [Monitoring cookbook, chapter 3](/cookbook/monitoring-data-use/citations): collecting and merging citation events for a DOI. - [Hugging Face dataset cards](https://huggingface.co/docs/hub/datasets-cards): where a dataset's card links a paper, and where model cards name the datasets a model was trained on. -- [Microdata cookbook, chapter 7](/cookbook/microdata-documentation/fitness): versioning each release and publishing errata, the same practice for microdata. +- Microdata cookbook, chapter 7: versioning each release and publishing errata, the same practice for microdata. ## Common mistakes diff --git a/cookbook/monitoring-data-use/act.mdx b/cookbook/monitoring-data-use/act.mdx index 74bd039b..eaa937c7 100644 --- a/cookbook/monitoring-data-use/act.mdx +++ b/cookbook/monitoring-data-use/act.mdx @@ -61,7 +61,7 @@ them. status: the completeness check of the [dissemination cookbook](/cookbook/ai-ready-dissemination/find) and, for surveys, the dictionary check of the - [microdata cookbook](/cookbook/microdata-documentation/variables). + microdata cookbook. 2. Order the work: datasets used most and documented least first. Among them, those used outside research first, since policy and media users read the record and nothing else. @@ -183,7 +183,7 @@ matches only; "contains" and fuzzy matches wait for review. - [Proof-Carrying Numbers](https://arxiv.org/abs/2509.06902): the program's method for checking a quoted figure against its source, applied to misquotes. - [NADA](https://nada.ihsn.org/): the catalog's citations feature, where "used in" lists are published per study. - [World Bank metadata schemas](https://github.com/worldbank/metadata-schemas): `data_sources` in document records and `aliases` in series records, the fields that close the loop. -- [Findability chapter of the dissemination cookbook](/cookbook/ai-ready-dissemination/find) and [chapter 3 of the microdata cookbook](/cookbook/microdata-documentation/variables): the completeness and dictionary checks that use counts prioritize. +- [Findability chapter of the dissemination cookbook](/cookbook/ai-ready-dissemination/find) and chapter 3 of the microdata cookbook: the completeness and dictionary checks that use counts prioritize. - [Evaluation chapter of the dissemination cookbook](/cookbook/ai-ready-dissemination/evaluate): the question sets that misquoted figures feed. ## Common mistakes diff --git a/cookbook/serving-statistics-to-agents/access.mdx b/cookbook/serving-statistics-to-agents/access.mdx index f890b99f..493db575 100644 --- a/cookbook/serving-statistics-to-agents/access.mdx +++ b/cookbook/serving-statistics-to-agents/access.mdx @@ -202,7 +202,7 @@ missing scope. The public series need no token at all. **What this does not do.** Authorization identifies users and their scopes. It does not decide whether a licensed product may be served to an agent at all. That decision belongs to the access policy of the -[microdata cookbook](/cookbook/microdata-documentation/access). +microdata cookbook. diff --git a/cookbook/serving-statistics-to-agents/purpose.mdx b/cookbook/serving-statistics-to-agents/purpose.mdx index a51f9226..2a565538 100644 --- a/cookbook/serving-statistics-to-agents/purpose.mdx +++ b/cookbook/serving-statistics-to-agents/purpose.mdx @@ -88,7 +88,7 @@ should have stayed behind a licence. scope at launch. Microdata are a special case: the interface may serve their metadata, so that an assistant can tell a user that a survey exists and how to apply for it, but it never serves the data - ([chapter 8 of the microdata cookbook](/cookbook/microdata-documentation/access)). + (chapter 8 of the microdata cookbook). 3. State what the interface does not do. It does not estimate, forecast, or combine series into new figures. The assistant reports what the organization publishes, and the statement says so in those words. diff --git a/cookbook/serving-statistics-to-agents/safety.mdx b/cookbook/serving-statistics-to-agents/safety.mdx index b870c966..567dddab 100644 --- a/cookbook/serving-statistics-to-agents/safety.mdx +++ b/cookbook/serving-statistics-to-agents/safety.mdx @@ -72,7 +72,7 @@ answer. 1. Give the server a credential to the API and the catalog that can read public records and values and nothing else. With that credential, a licensed microdata file is unreachable, which is the access rule of - [chapter 8 of the microdata cookbook](/cookbook/microdata-documentation/access). + chapter 8 of the microdata cookbook. 2. Implement no tool that writes, deletes, submits, or triggers anything. Mark every tool read-only in its annotations and say so in the manifest. The annotation is a hint that clients may display; the diff --git a/cookbook/synthetic-data-for-sharing/index.mdx b/cookbook/synthetic-data-for-sharing/index.mdx index 72e46b85..032171f3 100644 --- a/cookbook/synthetic-data-for-sharing/index.mdx +++ b/cookbook/synthetic-data-for-sharing/index.mdx @@ -143,7 +143,7 @@ scripts here measure, check, and document what they produce. |---|---| | This guide | How to produce, measure, document, and use synthetic microdata, in a sequence of practical steps. | | [REaLTabFormer](https://github.com/worldbank/REaLTabFormer) | The program's generator for tabular and relational synthetic data. | -| [Practical Guide to AI-Ready Microdata Documentation](/cookbook/microdata-documentation/) | The documentation, access tiers, and disclosure control the synthetic file sits beside. | +| Practical Guide to AI-Ready Microdata Documentation | The documentation, access tiers, and disclosure control the synthetic file sits beside. | | [Practical Guide to Monitoring Data Use](/cookbook/monitoring-data-use/) | Counting downloads and finding mentions of the synthetic file. | | [Code](https://github.com/worldbank/ai4data) | Open-source software to deploy or adapt. | diff --git a/cookbook/synthetic-data-for-sharing/purpose.mdx b/cookbook/synthetic-data-for-sharing/purpose.mdx index efb82956..98455be6 100644 --- a/cookbook/synthetic-data-for-sharing/purpose.mdx +++ b/cookbook/synthetic-data-for-sharing/purpose.mdx @@ -75,7 +75,7 @@ of what its synthetic files are for and what they must not be used for. synthetic in its name, in its catalog record, and inside the data itself. 4. Publish the statement with the access policy of the - [microdata cookbook](/cookbook/microdata-documentation/access), and + microdata cookbook, and repeat it in every synthesis record ([chapter 6](./release.mdx)). **Worked example.** The example organization's statement fits on one @@ -149,7 +149,7 @@ that users do analyse. inference, never the identification of a respondent in the file. 2. Where the organization uses partial synthesis as disclosure control, document it in the public use file's disclosure control record - ([microdata cookbook, chapter 8](/cookbook/microdata-documentation/access)), + (microdata cookbook, chapter 8), with utility measured for the estimates users will make. 3. Keep the two products apart in the catalog, with different labels, records, and terms of use. @@ -182,14 +182,14 @@ other; what differs is the documentation and the use. ## World Bank examples - The program's synthetic data workstream develops [REaLTabFormer](https://github.com/worldbank/REaLTabFormer), a transformer-based generator for tabular and relational data, placed under disclosure control (GSBPM 6.4) in the [GSBPM mapping](/docs/gsbpm-mapping). -- The [Microdata Library](https://microdata.worldbank.org/) access tiers and the [microdata cookbook's access chapter](/cookbook/microdata-documentation/access) set the workflow a synthetic file joins. +- The [Microdata Library](https://microdata.worldbank.org/) access tiers and the microdata cookbook's access chapter set the workflow a synthetic file joins. ## Resources - [UNECE, Synthetic Data for Official Statistics: A Starter Guide](https://unece.org/statistics/publications/synthetic-data-official-statistics-starter-guide): the reference introduction for statistical organizations, with use cases and cautions. - [REaLTabFormer](https://github.com/worldbank/REaLTabFormer): the program's generator for tabular and relational synthetic data (MIT). - [synthpop](https://cran.r-project.org/package=synthpop): the R package for sequential synthesis with CART and parametric methods, with its own utility and disclosure measures. -- [Access chapter of the microdata cookbook](/cookbook/microdata-documentation/access): the access policy and tiers the synthetic file sits beside. +- Access chapter of the microdata cookbook: the access policy and tiers the synthetic file sits beside. ## Common mistakes diff --git a/cookbook/synthetic-data-for-sharing/relational.mdx b/cookbook/synthetic-data-for-sharing/relational.mdx index c9452ae0..1daf9118 100644 --- a/cookbook/synthetic-data-for-sharing/relational.mdx +++ b/cookbook/synthetic-data-for-sharing/relational.mdx @@ -207,7 +207,7 @@ person histories is the alternative for panels with many waves. ## World Bank examples - [REaLTabFormer](https://github.com/worldbank/REaLTabFormer) was built for relational data: a parent table model and a sequence-to-sequence child model, evaluated on household-person and similar structures. -- The [microdata cookbook's comparability chapter](/cookbook/microdata-documentation/rounds) documents the structure across rounds that a synthetic panel has to reproduce. +- The microdata cookbook's comparability chapter documents the structure across rounds that a synthetic panel has to reproduce. ## Resources diff --git a/cookbook/synthetic-data-for-sharing/release.mdx b/cookbook/synthetic-data-for-sharing/release.mdx index 8f148eda..12021440 100644 --- a/cookbook/synthetic-data-for-sharing/release.mdx +++ b/cookbook/synthetic-data-for-sharing/release.mdx @@ -218,7 +218,7 @@ label, and the terms of use ask users to keep both. ## World Bank examples - The [Metadata Editor](https://github.com/worldbank/metadata-editor) and [NADA](https://nada.ihsn.org/) hold the record and the access type, with documentation files attached. -- The [microdata cookbook's access chapter](/cookbook/microdata-documentation/access) sets the tiers and the policy the release decision applies. +- The microdata cookbook's access chapter sets the tiers and the policy the release decision applies. ## Resources diff --git a/cookbook/synthetic-data-for-sharing/risk.mdx b/cookbook/synthetic-data-for-sharing/risk.mdx index d1a2c3f7..529da4c2 100644 --- a/cookbook/synthetic-data-for-sharing/risk.mdx +++ b/cookbook/synthetic-data-for-sharing/risk.mdx @@ -153,7 +153,7 @@ recorded review against the limits. in a small area with a rare combination that occurs once in the real file. 2. Record the limits in the survey's disclosure control documentation - ([microdata cookbook, chapter 8](/cookbook/microdata-documentation/access)). + (microdata cookbook, chapter 8). 3. Review each file's report against the limits. Record the reviewer, the date, and the decision in the synthesis record. @@ -220,7 +220,7 @@ use, which address misuse, a different risk from disclosure. ## World Bank examples - [REaLTabFormer](https://github.com/worldbank/REaLTabFormer) includes a distance-to-closest-record stopping criterion that limits memorization during training, the closest-record check applied at training time. -- The [microdata cookbook's access chapter](/cookbook/microdata-documentation/access) and the [sdcMicro practice guide](https://sdcpractice.readthedocs.io/) set the disclosure control practice the unit applies. +- The microdata cookbook's access chapter and the [sdcMicro practice guide](https://sdcpractice.readthedocs.io/) set the disclosure control practice the unit applies. ## Resources diff --git a/cookbook/synthetic-data-for-sharing/standards.md b/cookbook/synthetic-data-for-sharing/standards.md index d629889c..7d9b0520 100644 --- a/cookbook/synthetic-data-for-sharing/standards.md +++ b/cookbook/synthetic-data-for-sharing/standards.md @@ -36,7 +36,7 @@ the program's workstreams provide a tool, the guide uses it. | [Metadata Editor](https://worldbank.github.io/metadata-editor-docs) and [NADA](https://nada.ihsn.org/) | Creating the record and publishing the file with its access type | Chapter 6 | | [Creative Commons BY 4.0](https://creativecommons.org/licenses/by/4.0/) | The licence for open synthetic files | Chapter 6 | | [UNECE, Synthetic Data for Official Statistics: A Starter Guide](https://unece.org/statistics/publications/synthetic-data-official-statistics-starter-guide) | The reference introduction and use cases for statistical organizations | Chapters [1](./purpose.mdx) and [7](./workflow.mdx) | -| [Access chapter of the microdata cookbook](/cookbook/microdata-documentation/access) | Tiers, disclosure control records, and the access workflow | Chapters 1, 4, 6, and 7 | +| Access chapter of the microdata cookbook | Tiers, disclosure control records, and the access workflow | Chapters 1, 4, 6, and 7 | ## Corrections diff --git a/cookbook/synthetic-data-for-sharing/workflow.mdx b/cookbook/synthetic-data-for-sharing/workflow.mdx index 73f69d4f..61681fc4 100644 --- a/cookbook/synthetic-data-for-sharing/workflow.mdx +++ b/cookbook/synthetic-data-for-sharing/workflow.mdx @@ -107,7 +107,7 @@ program from the synthetic file to the real one by changing one path. variable names, types, value codes, and file layout as the real file, so that a program runs on the real data after changing one file path. Provide the data dictionary of the - [microdata cookbook](/cookbook/microdata-documentation/variables) for + microdata cookbook for both files. 3. State the differences a user will see on the real file: different numbers, small groups present, weights. @@ -180,13 +180,13 @@ followed up through the monitoring cookbook. ## World Bank examples -- The [Microdata Library](https://microdata.worldbank.org/) access workflow (public use, licensed, enclave) is where the synthetic file is offered, as the [microdata cookbook's access chapter](/cookbook/microdata-documentation/access) describes. +- The [Microdata Library](https://microdata.worldbank.org/) access workflow (public use, licensed, enclave) is where the synthetic file is offered, as the microdata cookbook's access chapter describes. - The [monitoring cookbook](/cookbook/monitoring-data-use/) counts access and finds mentions, which measure the synthetic file's use. ## Resources -- [Access chapter of the microdata cookbook](/cookbook/microdata-documentation/access): the tiers and workflow the file joins. -- [Variables chapter of the microdata cookbook](/cookbook/microdata-documentation/variables): the data dictionary that makes the switch-over possible. +- Access chapter of the microdata cookbook: the tiers and workflow the file joins. +- Variables chapter of the microdata cookbook: the data dictionary that makes the switch-over possible. - [Access statistics chapter of the monitoring cookbook](/cookbook/monitoring-data-use/access): counting downloads of the synthetic file consistently. - [UNECE starter guide](https://unece.org/statistics/publications/synthetic-data-official-statistics-starter-guide): use cases from statistical organizations. diff --git a/docs/metadata-augmentation/index.md b/docs/metadata-augmentation/index.md index 6e4db53d..ec9b4001 100644 --- a/docs/metadata-augmentation/index.md +++ b/docs/metadata-augmentation/index.md @@ -138,6 +138,6 @@ An API key for your chosen LLM provider must be set as an environment variable ( - Groups that fail the self-consistency check are kept in the output and flagged, so every output needs review before it enters a catalog. - Cluster counts and group quality vary with the embedding model, the dictionary language, and the length of variable labels; short or coded labels cluster poorly without question text. -- The pipeline proposes variable groups. It does not change variable labels, concepts, or classifications, which the [microdata documentation cookbook](/cookbook/microdata-documentation/concepts) covers. +- The pipeline proposes variable groups. It does not change variable labels, concepts, or classifications, which the microdata documentation cookbook covers. - An API key for a model provider is required, or a local model served through a compatible endpoint. - Status on the [tools catalogue](/tools#metadata-augmentation): pre-release. diff --git a/docs/reference-implementation/index.md b/docs/reference-implementation/index.md index 1963aed8..ebe0fe56 100644 --- a/docs/reference-implementation/index.md +++ b/docs/reference-implementation/index.md @@ -94,5 +94,5 @@ The tests in `tests/test_reference_end_to_end.py` run the pipeline in both modes - [Implementation hub](/hub): the areas the steps belong to. - [Dissemination cookbook](/cookbook/ai-ready-dissemination/): findability, retrieval, meaning, and trust, with the completeness checker the validation step follows. -- [Microdata documentation cookbook](/cookbook/microdata-documentation/): the survey record and its catalog. +- Microdata documentation cookbook: the survey record and its catalog. - [Agents cookbook](/cookbook/serving-statistics-to-agents/): the full design of a server over official statistics. diff --git a/website/cookbook-visibility.json b/website/cookbook-visibility.json index 5eda5ab4..8dfe5aae 100644 --- a/website/cookbook-visibility.json +++ b/website/cookbook-visibility.json @@ -4,7 +4,8 @@ "hidden": [ "governance-and-risk", "assessment-to-pathway", - "ai-assistants-at-work" + "ai-assistants-at-work", + "microdata-documentation" ], "indexed": [] } diff --git a/website/src/content/investment.json b/website/src/content/investment.json index 6abe796b..9c5e6d05 100644 --- a/website/src/content/investment.json +++ b/website/src/content/investment.json @@ -3,8 +3,8 @@ "Foundational": { "roles": [ "Developer or data engineer", - "Data curator", "Subject-matter staff", + "Data curator", "Catalog or data manager", "Methodologist", "Dissemination officer", @@ -15,16 +15,12 @@ 5 ], "per_period": [ - "minutes after that", - "minutes on each release" + "minutes after that" ], "per_item": [ "About 30 minutes per indicator", "An hour per questionnaire", "Half an hour per document", - "Minutes per file", - "One hour per survey", - "Two to four hours per survey", "seconds per record" ], "recipes": [ @@ -40,30 +36,6 @@ "time": "About 30 minutes per indicator", "skills": "Subject-matter statistician" }, - { - "title": "3.1 Check the data dictionary before release", - "chapter": "microdata-documentation/variables", - "time": "One hour the first time; minutes on each release", - "skills": "Data curator; Python" - }, - { - "title": "2.1 Draft a dictionary from the data file", - "chapter": "microdata-documentation/produce", - "time": "Minutes per file", - "skills": "Data curator; Python" - }, - { - "title": "2.2 Import files with their labels into the Metadata Editor", - "chapter": "microdata-documentation/produce", - "time": "One hour per survey", - "skills": "Data curator" - }, - { - "title": "4.1 Attach the question text to every collected variable", - "chapter": "microdata-documentation/concepts", - "time": "Two to four hours per survey", - "skills": "Data curator with the questionnaire" - }, { "title": "6.1 Build the vocabulary", "chapter": "metadata-curation-with-llms/standardize", @@ -124,34 +96,30 @@ "roles": [ "Developer or data engineer", "Data curator", + "Catalog or data manager", "Subject-matter staff", "Methodologist", - "Catalog or data manager", - "Management", "IT operations and security" ], "effort_days": [ - 6, - 6 + 4, + 4 ], "per_period": [ "Minutes per period", "Part of each round", "an hour per period to review", - "an hour per quarter", "minutes per run", "part of each release" ], "per_item": [ "Half a day per source", - "Half a day per survey", "Half an hour per document", "Half an hour per table", "Minutes per document", "Minutes per source", "Minutes per variable", "One to two days per code list", - "Two to four hours per survey", "minutes per questionnaire" ], "recipes": [ @@ -173,36 +141,6 @@ "time": "Half a day to set up; part of each release", "skills": "Data manager" }, - { - "title": "3.2 Convert the dictionary into the DDI-based schema", - "chapter": "microdata-documentation/variables", - "time": "One hour", - "skills": "Python" - }, - { - "title": "3.3 Document missing codes, universes, and derivations", - "chapter": "microdata-documentation/variables", - "time": "Two to four hours per survey", - "skills": "Subject-matter statistician" - }, - { - "title": "2.3 Recover question text and code lists from questionnaires and codebooks", - "chapter": "microdata-documentation/produce", - "time": "Half a day per survey; less with a pipeline", - "skills": "Data curator; developer for the extraction" - }, - { - "title": "2.4 Order the documentation backlog", - "chapter": "microdata-documentation/produce", - "time": "One day, then an hour per quarter", - "skills": "Head of the microdata unit" - }, - { - "title": "4.2 Map variables to concepts and classifications", - "chapter": "microdata-documentation/concepts", - "time": "One day per survey the first time", - "skills": "Subject-matter statistician; classification specialist" - }, { "title": "6.2 Map free text to the vocabulary", "chapter": "metadata-curation-with-llms/standardize", @@ -291,17 +229,13 @@ }, "AI-native": { "roles": [ - "Developer or data engineer", - "Data curator", - "Methodologist" + "Developer or data engineer" ], "effort_days": [ - 10, - 15 - ], - "per_period": [ - "Ongoing" + 5, + 10 ], + "per_period": [], "per_item": [], "recipes": [ { @@ -309,12 +243,6 @@ "chapter": "ai-ready-dissemination/find", "time": "One to two weeks including evaluation", "skills": "Developer with Python; a server with a few GB of memory" - }, - { - "title": "4.3 Reuse questions and concepts across surveys", - "chapter": "microdata-documentation/concepts", - "time": "Ongoing; one week to set up", - "skills": "Methodologist; data curator" } ] } @@ -325,8 +253,8 @@ "Developer or data engineer", "Methodologist", "Dissemination officer", - "Data curator", "Management", + "Data curator", "Subject-matter staff", "Catalog or data manager", "Disclosure control specialist" @@ -344,7 +272,6 @@ ], "per_item": [ "About 30 minutes per indicator", - "Half a day per survey", "seconds per record" ], "recipes": [ @@ -402,18 +329,6 @@ "time": "Half a day per batch", "skills": "Dissemination officer" }, - { - "title": "6.1 Draft the crosswalk between two rounds", - "chapter": "microdata-documentation/rounds", - "time": "Minutes to run; an hour to review", - "skills": "Data curator; Python" - }, - { - "title": "7.1 Document the design, the weights, and the response", - "chapter": "microdata-documentation/fitness", - "time": "Half a day per survey", - "skills": "Survey methodologist" - }, { "title": "5.1 Compare the composition with population benchmarks", "chapter": "ml-ready-datasets/represent", @@ -426,8 +341,8 @@ "roles": [ "Developer or data engineer", "Data curator", - "Methodologist", "Dissemination officer", + "Methodologist", "Catalog or data manager", "Subject-matter staff", "Staff of the unit", @@ -443,7 +358,6 @@ "Minutes per batch", "Minutes per change", "Minutes per period", - "One hour per release", "minutes per batch", "minutes per month", "minutes per run", @@ -451,8 +365,6 @@ ], "per_item": [ "Half a day per dataset", - "Half a day per survey", - "Half a day per variable", "One day per task", "One hour per component", "One to three minutes per issue", @@ -543,24 +455,6 @@ "time": "Minutes per batch", "skills": "Python" }, - { - "title": "6.2 Publish a correspondence table for a recoded variable", - "chapter": "microdata-documentation/rounds", - "time": "Half a day per variable", - "skills": "Subject-matter statistician; classification specialist" - }, - { - "title": "7.2 Publish a quality statement in machine-readable fields", - "chapter": "microdata-documentation/fitness", - "time": "Half a day per survey", - "skills": "Survey methodologist; data curator" - }, - { - "title": "7.3 Version each release and publish errata", - "chapter": "microdata-documentation/fitness", - "time": "One hour per release", - "skills": "Data curator" - }, { "title": "5.2 Report thin groups and the coverage of the label", "chapter": "ml-ready-datasets/represent", @@ -615,12 +509,11 @@ "roles": [ "Developer or data engineer", "Methodologist", - "Data curator", "Data scientist" ], "effort_days": [ - 15, - 25 + 10, + 15 ], "per_period": [ "repeat with each change" @@ -651,12 +544,6 @@ "time": "One day to set up", "skills": "Developer" }, - { - "title": "6.3 Harmonize variable names and definitions across rounds", - "chapter": "microdata-documentation/rounds", - "time": "One to two weeks for a survey program", - "skills": "Methodologist; data curator" - }, { "title": "5.3 Publish the model's error by group with the dataset", "chapter": "ml-ready-datasets/represent", @@ -686,9 +573,7 @@ "Catalog or data manager", "Management", "Subject-matter staff", - "Data curator", - "IT operations and security", - "Methodologist" + "IT operations and security" ], "effort_days": [ 4, @@ -699,7 +584,6 @@ "seconds per run" ], "per_item": [ - "Half a day per survey", "minutes per dataset" ], "recipes": [ @@ -721,12 +605,6 @@ "time": "One day for the first 30 questions", "skills": "Dissemination staff; speakers of each supported language" }, - { - "title": "1.1 Write the study record in the World Bank microdata schema", - "chapter": "microdata-documentation/find", - "time": "Half a day per survey", - "skills": "Data curator; the Metadata Editor or a text editor" - }, { "title": "3.1 Write the guidance resource", "chapter": "serving-statistics-to-agents/context", @@ -739,12 +617,6 @@ "time": "Half a day to set up; minutes per dataset", "skills": "Catalog administrator" }, - { - "title": "7.1 Document the design, the weights, and the response", - "chapter": "microdata-documentation/fitness", - "time": "Half a day per survey", - "skills": "Survey methodologist" - }, { "title": "8.2 Cite to the document, the page, and the table", "chapter": "data-from-documents/cite", @@ -769,16 +641,15 @@ "roles": [ "Developer or data engineer", "Dissemination officer", - "Data curator", + "Management", "Catalog or data manager", "IT operations and security", - "Subject-matter staff", - "Management", - "Methodologist" + "Data curator", + "Subject-matter staff" ], "effort_days": [ - 8, - 36 + 6, + 6 ], "per_period": [ "Half a day per period", @@ -786,7 +657,6 @@ "Minutes per record for review", "Minutes per run", "One hour per period plus follow-up", - "One hour per release", "Ongoing", "Part of each report", "Part of publication", @@ -795,10 +665,7 @@ "with each change" ], "per_item": [ - "Half a day per survey", - "Half a day per survey by hand", - "Seconds per record", - "Two hours per survey once the catalog exists" + "Seconds per record" ], "recipes": [ { @@ -819,24 +686,6 @@ "time": "Two hours", "skills": "Python" }, - { - "title": "1.2 Publish the record in a catalog with a persistent identifier", - "chapter": "microdata-documentation/find", - "time": "Two hours per survey once the catalog exists", - "skills": "Catalog administrator" - }, - { - "title": "5.1 Group variables into themes", - "chapter": "microdata-documentation/navigate", - "time": "Half a day per survey by hand; minutes with the pipeline, plus review", - "skills": "Data curator; Python for the automated version" - }, - { - "title": "5.2 Measure variable search with a known-variable question set", - "chapter": "microdata-documentation/navigate", - "time": "One day for the first 30 questions", - "skills": "Data curator; speakers of each supported language; Python" - }, { "title": "3.2 Serve code lists, the release calendar, and notes as resources", "chapter": "serving-statistics-to-agents/context", @@ -879,18 +728,6 @@ "time": "One hour per period plus follow-up", "skills": "Dissemination officer" }, - { - "title": "7.2 Publish a quality statement in machine-readable fields", - "chapter": "microdata-documentation/fitness", - "time": "Half a day per survey", - "skills": "Survey methodologist; data curator" - }, - { - "title": "7.3 Version each release and publish errata", - "chapter": "microdata-documentation/fitness", - "time": "One hour per release", - "skills": "Data curator" - }, { "title": "8.1 Make extracted tables findable", "chapter": "data-from-documents/cite", @@ -961,8 +798,8 @@ "IT operations and security" ], "effort_days": [ - 20, - 30 + 16, + 20 ], "per_period": [ "part of each release" @@ -987,12 +824,6 @@ "time": "One week to set up; part of each release", "skills": "Python; subject-matter reviewers" }, - { - "title": "5.3 Add semantic variable search across surveys", - "chapter": "microdata-documentation/navigate", - "time": "One to two weeks including evaluation", - "skills": "Developer with Python" - }, { "title": "9.3 Close the loop in the catalog", "chapter": "monitoring-data-use/act", @@ -1634,23 +1465,21 @@ "Foundational": { "roles": [ "Management", - "Disclosure control specialist", "Developer or data engineer", "Dissemination officer", "Catalog or data manager", - "Data curator" + "Disclosure control specialist" ], "effort_days": [ - 4, - 4 + 2, + 2 ], "per_period": [ "minutes after that", "part of each release" ], "per_item": [ - "One to three days per dataset", - "One to three days per file" + "One to three days per dataset" ], "recipes": [ { @@ -1682,18 +1511,6 @@ "chapter": "ml-ready-datasets/license", "time": "One to three days per dataset", "skills": "Disclosure control unit; dataset owner" - }, - { - "title": "8.1 Adopt an access policy with tiers", - "chapter": "microdata-documentation/access", - "time": "One day to draft; approval time varies", - "skills": "Management; legal; disclosure control specialist" - }, - { - "title": "8.2 Apply disclosure control and document it in the record", - "chapter": "microdata-documentation/access", - "time": "One to three days per file", - "skills": "Disclosure control specialist; data curator" } ] }, @@ -1701,17 +1518,14 @@ "roles": [ "Developer or data engineer", "Catalog or data manager", - "IT operations and security", - "Data curator" + "IT operations and security" ], "effort_days": [ 3, 4 ], "per_period": [], - "per_item": [ - "One hour per survey" - ], + "per_item": [], "recipes": [ { "title": "5.2 Return provenance with every API response", @@ -1736,24 +1550,16 @@ "chapter": "ml-ready-datasets/license", "time": "An hour", "skills": "Catalog administrator; developer" - }, - { - "title": "8.3 State the conditions in every record", - "chapter": "microdata-documentation/access", - "time": "One hour per survey", - "skills": "Data curator" } ] }, "AI-native": { "roles": [ - "Developer or data engineer", - "Catalog or data manager", - "IT operations and security" + "Developer or data engineer" ], "effort_days": [ 12, - 18 + 16 ], "per_period": [], "per_item": [], @@ -1775,12 +1581,6 @@ "chapter": "ai-ready-dissemination/retrieve", "time": "One day for a prototype; a week to harden", "skills": "Python developer" - }, - { - "title": "8.4 Define what automated and AI clients may access", - "chapter": "microdata-documentation/access", - "time": "One day", - "skills": "Catalog administrator; developer" } ] } From 3e649c72f963ffa15f4e3891f60a6960fafed6d4 Mon Sep 17 00:00:00 2001 From: avsolatorio Date: Thu, 22 Oct 2026 12:57:32 +0000 Subject: [PATCH 2/4] Docs site: chapter mentions of the hidden microdata cookbook name the chapter topic --- cookbook/language-models-in-production/design.mdx | 4 ++-- cookbook/metadata-curation-with-llms/standardize.mdx | 2 +- cookbook/ml-ready-datasets/license.mdx | 4 ++-- cookbook/ml-ready-datasets/version.mdx | 2 +- cookbook/monitoring-data-use/act.mdx | 2 +- cookbook/serving-statistics-to-agents/purpose.mdx | 2 +- cookbook/serving-statistics-to-agents/safety.mdx | 2 +- cookbook/synthetic-data-for-sharing/purpose.mdx | 2 +- cookbook/synthetic-data-for-sharing/risk.mdx | 2 +- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cookbook/language-models-in-production/design.mdx b/cookbook/language-models-in-production/design.mdx index 2ebab669..3f2e5836 100644 --- a/cookbook/language-models-in-production/design.mdx +++ b/cookbook/language-models-in-production/design.mdx @@ -123,7 +123,7 @@ the model judges each one. a different concept. Ask for a confidence and a reason. 3. The methodologist confirms or rejects each proposal. A confirmed match links the new variable to the concept - (microdata cookbook, chapter 4) + (concepts chapter of the microdata cookbook) and inherits its definition. 4. Track the share of proposals confirmed at each level of confidence. Stop showing proposals below the level at which few are confirmed. @@ -143,7 +143,7 @@ definition, and the comparability note records the added category. **What this does not do.** A match says that two questions measure the same concept. It does not make their data comparable; that depends on the survey mode, the population, and the categories -(microdata cookbook, chapter 6). +(comparability chapter of the microdata cookbook). diff --git a/cookbook/metadata-curation-with-llms/standardize.mdx b/cookbook/metadata-curation-with-llms/standardize.mdx index 245d911d..aa9d458f 100644 --- a/cookbook/metadata-curation-with-llms/standardize.mdx +++ b/cookbook/metadata-curation-with-llms/standardize.mdx @@ -180,7 +180,7 @@ filter for every user. - **Vocabulary format:** SKOS for concepts and labels, XKOS for classifications; a CSV with preferred label, URI, and alternates as the working form. - **Mapping:** exact and fuzzy string matching (this chapter); embeddings for semantic matching; a model for proposals with a confidence. - **Storage:** `keywords` and `topics` with vocabulary and URI in the World Bank schemas; the Metadata Editor's controlled lists. -- **International vocabularies:** the SDG indicator framework, the classifications of chapter 4 of the microdata cookbook, and thematic thesauri, as sources of URIs. +- **International vocabularies:** the SDG indicator framework, the classifications of the concepts chapter of the microdata cookbook, and thematic thesauri, as sources of URIs. ## World Bank examples diff --git a/cookbook/ml-ready-datasets/license.mdx b/cookbook/ml-ready-datasets/license.mdx index d08dcc5d..5262a440 100644 --- a/cookbook/ml-ready-datasets/license.mdx +++ b/cookbook/ml-ready-datasets/license.mdx @@ -156,7 +156,7 @@ treatment of the source records is stated in the card. 1. Establish the source. A dataset built from a public use file inherits its disclosure control - (microdata cookbook, chapter 8). + (access chapter of the microdata cookbook). A dataset built from the full file is a new product and gets a new review, with the key variables, the uniqueness measures, and the recoding the unit requires. @@ -210,7 +210,7 @@ text is reviewed again when a new version adds records. - [Understanding CC licenses and generative AI](https://creativecommons.org/2023/08/18/understanding-cc-licenses-and-generative-ai/) (Creative Commons, 2023): what the licences do and do not say about training. - [Responsible AI Licenses](https://www.licenses.ai/): the RAIL family, for comparison when the organization considers use restrictions. - [sdcMicro practice guide](https://sdcpractice.readthedocs.io/): the disclosure control workflow the unit applies to a new product. -- Microdata cookbook, chapter 8: the access policy with tiers and the rules for AI clients. +- Access chapter of the microdata cookbook: the access policy with tiers and the rules for AI clients. ## Common mistakes diff --git a/cookbook/ml-ready-datasets/version.mdx b/cookbook/ml-ready-datasets/version.mdx index 71501249..5bea3f21 100644 --- a/cookbook/ml-ready-datasets/version.mdx +++ b/cookbook/ml-ready-datasets/version.mdx @@ -231,7 +231,7 @@ terms ask for citation and the card makes it easy. - [DataCite metadata schema](https://schema.datacite.org/): the fields for a dataset version's DOI, including the relation to other versions. - [Monitoring cookbook, chapter 3](/cookbook/monitoring-data-use/citations): collecting and merging citation events for a DOI. - [Hugging Face dataset cards](https://huggingface.co/docs/hub/datasets-cards): where a dataset's card links a paper, and where model cards name the datasets a model was trained on. -- Microdata cookbook, chapter 7: versioning each release and publishing errata, the same practice for microdata. +- Fitness chapter of the microdata cookbook: versioning each release and publishing errata, the same practice for microdata. ## Common mistakes diff --git a/cookbook/monitoring-data-use/act.mdx b/cookbook/monitoring-data-use/act.mdx index eaa937c7..96e3928e 100644 --- a/cookbook/monitoring-data-use/act.mdx +++ b/cookbook/monitoring-data-use/act.mdx @@ -183,7 +183,7 @@ matches only; "contains" and fuzzy matches wait for review. - [Proof-Carrying Numbers](https://arxiv.org/abs/2509.06902): the program's method for checking a quoted figure against its source, applied to misquotes. - [NADA](https://nada.ihsn.org/): the catalog's citations feature, where "used in" lists are published per study. - [World Bank metadata schemas](https://github.com/worldbank/metadata-schemas): `data_sources` in document records and `aliases` in series records, the fields that close the loop. -- [Findability chapter of the dissemination cookbook](/cookbook/ai-ready-dissemination/find) and chapter 3 of the microdata cookbook: the completeness and dictionary checks that use counts prioritize. +- [Findability chapter of the dissemination cookbook](/cookbook/ai-ready-dissemination/find) and the dictionary chapter of the microdata cookbook: the completeness and dictionary checks that use counts prioritize. - [Evaluation chapter of the dissemination cookbook](/cookbook/ai-ready-dissemination/evaluate): the question sets that misquoted figures feed. ## Common mistakes diff --git a/cookbook/serving-statistics-to-agents/purpose.mdx b/cookbook/serving-statistics-to-agents/purpose.mdx index 2a565538..344cfb31 100644 --- a/cookbook/serving-statistics-to-agents/purpose.mdx +++ b/cookbook/serving-statistics-to-agents/purpose.mdx @@ -88,7 +88,7 @@ should have stayed behind a licence. scope at launch. Microdata are a special case: the interface may serve their metadata, so that an assistant can tell a user that a survey exists and how to apply for it, but it never serves the data - (chapter 8 of the microdata cookbook). + (the access chapter of the microdata cookbook). 3. State what the interface does not do. It does not estimate, forecast, or combine series into new figures. The assistant reports what the organization publishes, and the statement says so in those words. diff --git a/cookbook/serving-statistics-to-agents/safety.mdx b/cookbook/serving-statistics-to-agents/safety.mdx index 567dddab..3eac2cfd 100644 --- a/cookbook/serving-statistics-to-agents/safety.mdx +++ b/cookbook/serving-statistics-to-agents/safety.mdx @@ -72,7 +72,7 @@ answer. 1. Give the server a credential to the API and the catalog that can read public records and values and nothing else. With that credential, a licensed microdata file is unreachable, which is the access rule of - chapter 8 of the microdata cookbook. + the access chapter of the microdata cookbook. 2. Implement no tool that writes, deletes, submits, or triggers anything. Mark every tool read-only in its annotations and say so in the manifest. The annotation is a hint that clients may display; the diff --git a/cookbook/synthetic-data-for-sharing/purpose.mdx b/cookbook/synthetic-data-for-sharing/purpose.mdx index 98455be6..713578d3 100644 --- a/cookbook/synthetic-data-for-sharing/purpose.mdx +++ b/cookbook/synthetic-data-for-sharing/purpose.mdx @@ -149,7 +149,7 @@ that users do analyse. inference, never the identification of a respondent in the file. 2. Where the organization uses partial synthesis as disclosure control, document it in the public use file's disclosure control record - (microdata cookbook, chapter 8), + (access chapter of the microdata cookbook), with utility measured for the estimates users will make. 3. Keep the two products apart in the catalog, with different labels, records, and terms of use. diff --git a/cookbook/synthetic-data-for-sharing/risk.mdx b/cookbook/synthetic-data-for-sharing/risk.mdx index 529da4c2..2448b993 100644 --- a/cookbook/synthetic-data-for-sharing/risk.mdx +++ b/cookbook/synthetic-data-for-sharing/risk.mdx @@ -153,7 +153,7 @@ recorded review against the limits. in a small area with a rare combination that occurs once in the real file. 2. Record the limits in the survey's disclosure control documentation - (microdata cookbook, chapter 8). + (access chapter of the microdata cookbook). 3. Review each file's report against the limits. Record the reviewer, the date, and the decision in the synthesis record. From 97a2c5b3b06a9303b994dcd28746d1a2c7c139ce Mon Sep 17 00:00:00 2001 From: avsolatorio Date: Fri, 23 Oct 2026 18:11:43 +0000 Subject: [PATCH 3/4] Docs site: an unlisted catalogue page at /unlisted lists the hidden cookbooks with their chapters and the standalone unlisted pages; noindex, out of the sitemap and the search index --- website/docusaurus.config.js | 3 + website/src/content/cookbooks.js | 2 +- website/src/pages/unlisted.js | 104 ++++++++++++++++++++++++++ website/src/pages/unlisted.module.css | 45 +++++++++++ 4 files changed, 153 insertions(+), 1 deletion(-) create mode 100644 website/src/pages/unlisted.js create mode 100644 website/src/pages/unlisted.module.css diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index 8be2d0ce..d37ae761 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -104,6 +104,8 @@ const config = { '/ai4data/nso-roadmap/**', '/ai4data/modern-nso', '/ai4data/modern-nso/**', + '/ai4data/unlisted', + '/ai4data/unlisted/**', ...hiddenCookbooks.map((id) => `/ai4data/cookbook/${id}/**`), ...draftCookbooks.flatMap((id) => [`/ai4data/cookbook/${id}/`, `/ai4data/cookbook/${id}/**`]), ], @@ -282,6 +284,7 @@ const config = { ignoreFiles: [ 'modern-nso', 'nso-roadmap', + 'unlisted', ...hiddenCookbooks.map((id) => new RegExp(`^cookbook/${id.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(/|$)`)), ], }), diff --git a/website/src/content/cookbooks.js b/website/src/content/cookbooks.js index c11d8f70..e9a366a1 100644 --- a/website/src/content/cookbooks.js +++ b/website/src/content/cookbooks.js @@ -3,7 +3,7 @@ import {isHiddenCookbook, isHiddenPath, isIndexedCookbook} from './visibility'; // Cookbooks listed on the /cookbook landing page. Each cookbook is a folder // under /cookbook with its own sidebar in sidebars-cookbook.js. The ids in // website/cookbook-visibility.json are dropped from every list exported here. -const allCookbooks = [ +export const allCookbooks = [ { id: 'ai-ready-dissemination', version: '0.1', diff --git a/website/src/pages/unlisted.js b/website/src/pages/unlisted.js new file mode 100644 index 00000000..ed723530 --- /dev/null +++ b/website/src/pages/unlisted.js @@ -0,0 +1,104 @@ +import Head from '@docusaurus/Head'; +import Link from '@docusaurus/Link'; +import useGlobalData from '@docusaurus/useGlobalData'; +import Layout from '@theme/Layout'; +import Heading from '@theme/Heading'; +import {allCookbooks} from '@site/src/content/cookbooks'; +import {hiddenCookbooks} from '@site/src/content/visibility'; +import styles from './unlisted.module.css'; + +// A catalogue of the pages that exist on the site without being listed on it: the hidden +// cookbooks, read from website/cookbook-visibility.json, and the standalone pages below. +// This page is itself unlisted: no link leads here, it carries a noindex tag, and the site +// config keeps it out of the sitemap and the search index. + +const STANDALONE_PAGES = [ + {to: '/nso-roadmap', title: 'Roadmap for AI-readiness in a statistical office', note: 'Planning draft; linked only from the footer.'}, + {to: '/modern-nso', title: 'A modern statistical office', note: 'Composite portrait at three maturity levels.'}, +]; + +function chapterLabel(path, cookbookPath) { + const slug = path.replace(cookbookPath, '').replace(/\/$/, ''); + if (!slug) return 'Overview'; + return slug.charAt(0).toUpperCase() + slug.slice(1).replace(/-/g, ' '); +} + +export default function Unlisted() { + const globalData = useGlobalData(); + const docsPlugins = globalData['docusaurus-plugin-content-docs'] || {}; + const allDocs = Object.values(docsPlugins).flatMap((plugin) => (plugin.versions || []).flatMap((v) => v.docs || [])); + const hidden = hiddenCookbooks.map((id) => { + const card = allCookbooks.find((c) => c.id === id); + const prefix = `/cookbook/${id}/`; + const chapters = allDocs + .filter((d) => d.path.startsWith(prefix) && d.path !== prefix) + .sort((a, b) => a.path.localeCompare(b.path)); + return {id, card, prefix, chapters}; + }); + const otherUnlisted = allDocs.filter((d) => d.unlisted && !hiddenCookbooks.some((id) => d.path.startsWith(`/cookbook/${id}/`))); + + return ( + + + + +
+ Unlisted pages +

+ These pages are built and deployed with the site but are not listed on it: no card, no navigation entry, no search result, no sitemap entry, and a noindex tag. They open from + these links for review. This page is unlisted in the same way; keep its address. +

+ + Hidden cookbooks +

+ Listed in website/cookbook-visibility.json under hidden. Removing an id from that list publishes the cookbook; nothing else changes. +

+ {hidden.map(({id, card, prefix, chapters}) => ( +
+ + {card ? card.title : id} + + {card && ( +

+ {card.audience} · version {card.version}, {card.date} · {card.chapters} chapters +

+ )} +
    + {chapters.map((d) => ( +
  • + {chapterLabel(d.path, prefix)} + {d.path} +
    • + ))} +
+
+ ))} + + Standalone pages +
    + {STANDALONE_PAGES.map((p) => ( +
  • + {p.title} + {p.to} + {p.note} +
    • + ))} +
+ + {otherUnlisted.length > 0 && ( + <> + Other unlisted documents +
    + {otherUnlisted.map((d) => ( +
  • + {d.id} + {d.path} +
    • + ))} +
+ + )} +
+ + ); +} diff --git a/website/src/pages/unlisted.module.css b/website/src/pages/unlisted.module.css new file mode 100644 index 00000000..9fcc7176 --- /dev/null +++ b/website/src/pages/unlisted.module.css @@ -0,0 +1,45 @@ +.page { + max-width: 880px; + margin: 0 auto; + padding: 2.5rem 1rem 4rem; +} +.lede { + max-width: 70ch; + color: var(--ifm-color-emphasis-700); +} +.note { + font-size: 0.9rem; + color: var(--ifm-color-emphasis-700); +} +.cookbook { + margin: 1.25rem 0 1.75rem; + padding: 1rem 1.25rem; + border: 1px solid var(--ifm-color-emphasis-200); + border-radius: 10px; +} +.cookbook h3 { + margin-bottom: 0.25rem; +} +.meta { + font-size: 0.9rem; + color: var(--ifm-color-emphasis-700); + margin-bottom: 0.75rem; +} +.chapters { + list-style: none; + padding: 0; + margin: 0; + display: grid; + gap: 0.35rem; +} +.chapters li { + display: flex; + flex-wrap: wrap; + gap: 0.25rem 0.75rem; + align-items: baseline; +} +.path { + font-family: var(--ifm-font-family-monospace); + font-size: 0.8rem; + color: var(--ifm-color-emphasis-600); +} From 1d525aafb9459f67e4924127c48a68c2e7bf87a3 Mon Sep 17 00:00:00 2001 From: avsolatorio Date: Fri, 23 Oct 2026 18:12:29 +0000 Subject: [PATCH 4/4] Docs site: the unlisted catalogue matches cookbook paths with the site base path --- website/src/pages/unlisted.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/website/src/pages/unlisted.js b/website/src/pages/unlisted.js index ed723530..a7ffd9cc 100644 --- a/website/src/pages/unlisted.js +++ b/website/src/pages/unlisted.js @@ -1,5 +1,6 @@ import Head from '@docusaurus/Head'; import Link from '@docusaurus/Link'; +import useBaseUrl from '@docusaurus/useBaseUrl'; import useGlobalData from '@docusaurus/useGlobalData'; import Layout from '@theme/Layout'; import Heading from '@theme/Heading'; @@ -24,18 +25,19 @@ function chapterLabel(path, cookbookPath) { } export default function Unlisted() { + const base = useBaseUrl('/'); // the plugin's doc paths carry the site's base path const globalData = useGlobalData(); const docsPlugins = globalData['docusaurus-plugin-content-docs'] || {}; const allDocs = Object.values(docsPlugins).flatMap((plugin) => (plugin.versions || []).flatMap((v) => v.docs || [])); const hidden = hiddenCookbooks.map((id) => { const card = allCookbooks.find((c) => c.id === id); - const prefix = `/cookbook/${id}/`; + const prefix = `${base}cookbook/${id}/`; const chapters = allDocs .filter((d) => d.path.startsWith(prefix) && d.path !== prefix) .sort((a, b) => a.path.localeCompare(b.path)); return {id, card, prefix, chapters}; }); - const otherUnlisted = allDocs.filter((d) => d.unlisted && !hiddenCookbooks.some((id) => d.path.startsWith(`/cookbook/${id}/`))); + const otherUnlisted = allDocs.filter((d) => d.unlisted && !hiddenCookbooks.some((id) => d.path.startsWith(`${base}cookbook/${id}/`))); return (