feat(data-index): Pages rollout kit — 28-repo generator, consistency guards, 200-checker

bin/check-data-pages

@@ -0,0 +1,140 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Confirm each relaton-data-* GitHub Pages index site is live. For every repo in
+# data-index/configs.yml this checks two URLs and prints their HTTP status:
+#
+#   site      - https://relaton.github.io/relaton-data-<repo>/  (the Pages site)
+#   raw-index - <baseurl><source>  (the index the theme plugin fetches at build
+#               time; a 404 here is why a built site can render empty)
+#
+# Exits non-zero if any checked URL is not 200, so it doubles as a rollout gate.
+# This is the operator's "confirm each site returns 200" step: run it AFTER the
+# Cimas push + _config.yml commits + Pages enablement have landed.
+#
+# Usage:
+#   bin/check-data-pages                     # check every repo (site + raw-index)
+#   bin/check-data-pages iso ieee            # check only these repos
+#   bin/check-data-pages --raw-only          # skip the Pages site, check raw-index
+#   bin/check-data-pages --site-only         # skip raw-index, check the Pages site
+#   bin/check-data-pages --base https://data.relaton.org  # custom Pages host
+#   bin/check-data-pages --list              # list known repos and exit
+#   bin/check-data-pages --config PATH ...   # use an alternate configs.yml
+
+require "optparse"
+require "net/http"
+require "uri"
+
+require_relative "../lib/data_index_config"
+
+options = {
+  config: DataIndexConfig::DEFAULT_CONFIG_PATH,
+  base: DataIndexConfig::PAGES_HOST,
+  list: false,
+  site: true,
+  raw: true,
+  timeout: 15,
+}
+parser = OptionParser.new do |o|
+  o.banner = "Usage: bin/check-data-pages [repo...] [options]"
+  o.on("--base HOST", "Pages host (default #{DataIndexConfig::PAGES_HOST})") { |v| options[:base] = v }
+  o.on("--site-only", "Check only the Pages site URL") { options[:raw] = false }
+  o.on("--raw-only", "Check only the raw-index URL") { options[:site] = false }
+  o.on("--timeout SECONDS", Integer, "Per-request timeout (default 15)") { |v| options[:timeout] = v }
+  o.on("--config PATH", "Path to configs.yml") { |v| options[:config] = v }
+  o.on("--list", "List known repos and exit") { options[:list] = true }
+  o.on("-h", "--help", "Show this help") do
+    puts o
+    exit
+  end
+end
+parser.parse!(ARGV)
+
+if !options[:site] && !options[:raw]
+  warn "nothing to check: --site-only and --raw-only are mutually exclusive"
+  exit 2
+end
+
+config = DataIndexConfig.load(options[:config])
+known = config.repos.map { |e| e["repo"] }
+
+if options[:list]
+  puts known
+  exit
+end
+
+selected = ARGV.empty? ? known : ARGV.dup
+unknown = selected - known
+unless unknown.empty?
+  warn "unknown repo(s): #{unknown.join(', ')}"
+  warn "known repos: #{known.join(', ')}"
+  exit 1
+end
+
+# Follow up to a few redirects (custom Pages domains 301 to https/apex) and
+# return the final numeric status, or a short error token on a network failure.
+def http_status(url, timeout:, hops: 5)
+  uri = URI.parse(url)
+  return "BADURI" unless uri.is_a?(URI::HTTP)
+
+  http = Net::HTTP.new(uri.host, uri.port)
+  http.use_ssl = (uri.scheme == "https")
+  http.open_timeout = timeout
+  http.read_timeout = timeout
+  # Pass a block so Net::HTTP hands back the response without buffering the body
+  # (we only need the status + Location; index files can be large).
+  res = http.request_get(uri.request_uri) { |_response| }
+  code = res.code.to_i
+  if [301, 302, 307, 308].include?(code) && hops.positive? && res["location"]
+    return http_status(URI.join(url, res["location"]).to_s, timeout: timeout, hops: hops - 1)
+  end
+
+  code
+rescue StandardError => e
+  "ERR:#{e.class.to_s.split('::').last}"
+end
+
+checks = []
+checks << :site if options[:site]
+checks << :raw if options[:raw]
+
+# Precompute the URLs once (also lets us size each column to its widest URL so
+# the status/URL columns line up regardless of repo-name or URL length).
+site_urls = options[:site] ? selected.to_h { |r| [r, config.pages_url(r, base: options[:base])] } : {}
+raw_urls = options[:raw] ? selected.to_h { |r| [r, config.raw_index_url(r)] } : {}
+
+repo_w = ([4] + selected.map(&:length)).max # 4 = len("repo")
+site_w = (site_urls.values.map(&:length) + [0]).max
+
+header = format("%-#{repo_w}s", "repo")
+header += "  #{format('%-4s', 'site')} #{format("%-#{site_w}s", '')}" if options[:site]
+header += "  raw-index" if options[:raw]
+puts header
+puts "-" * header.length
+
+failures = 0
+selected.each do |repo|
+  row = format("%-#{repo_w}s", repo)
+  statuses = {}
+
+  if options[:site]
+    st = http_status(site_urls[repo], timeout: options[:timeout])
+    statuses[:site] = st
+    row += "  #{format('%-4s', st)} #{format("%-#{site_w}s", site_urls[repo])}"
+  end
+
+  if options[:raw]
+    st = http_status(raw_urls[repo], timeout: options[:timeout])
+    statuses[:raw] = st
+    row += "  #{format('%-4s', st)} #{raw_urls[repo]}"
+  end
+
+  puts row
+  failures += 1 unless checks.all? { |k| statuses[k] == 200 }
+end
+
+if failures.positive?
+  warn "\n#{failures} repo(s) not fully 200"
+  exit 1
+end
+puts "\nall #{selected.size} repo(s) return 200"

cimas-config/cimas.yml

@@ -340,7 +340,7 @@ repositories:
       .github/workflows/keep-alive.yml: gh-actions/master/keep-alive.yml
   relaton-data-w3c:
     remote: ssh://git@github.com/relaton/relaton-data-w3c
-    branch: main
+    branch: v2
     files:
       .github/workflows/crawler.yml: gh-actions/data/crawler.yml
       .github/workflows/deploy.yml: gh-actions/data/deploy.yml
@@ -364,7 +364,7 @@ repositories:
       .github/workflows/keep-alive.yml: gh-actions/master/keep-alive.yml
   relaton-data-oasis:
     remote: ssh://git@github.com/relaton/relaton-data-oasis
-    branch: main
+    branch: v2
     files:
       .github/workflows/crawler.yml: gh-actions/data/crawler.yml
       .github/workflows/deploy.yml: gh-actions/data/deploy.yml
@@ -409,7 +409,7 @@ repositories:
       .github/workflows/keep-alive.yml: gh-actions/master/keep-alive.yml
   relaton-data-ids:
     remote: ssh://git@github.com/relaton/relaton-data-ids
-    branch: main
+    branch: v2
     files:
       .github/workflows/crawler.yml: gh-actions/data/crawler.yml
       .github/workflows/deploy.yml: gh-actions/data/deploy.yml

data-index/README.adoc

@@ -16,6 +16,12 @@ merges the theme's base `_config.yml` with the repo's own (repo values win), run
   real default branch — drives `baseurl`), and `pubid_class` (blank for flat
   indexes). Edit this to change any repo's config.
 `../bin/gen-data-index-config`:: Renders a repo's `_config.yml` from `configs.yml`.
+`generated/`:: The committed snapshot of all 28 rendered `_config.yml` files (one
+  per target repo) — the exact bytes to commit into each data repo. Regenerate with
+  `../bin/gen-data-index-config --out data-index/generated`; kept honest by
+  `spec/generated_configs_spec.rb`.
+`../bin/check-data-pages`:: Confirms each repo's Pages site and raw index URL return
+  200 (the "confirm each site returns 200" gate). Run it after the rollout lands.
 
 == Why `_config.yml` lives in each data repo (not Cimas)
 
@@ -64,30 +70,86 @@ bundle, the tier-C flavors below render citable DocIDs rather than a fallback li
 | adobe, ccsds, cie, easc, etsi, gost, iec, ieee, iho, iso, itu-r, jcgm, jis, nist, oiml, plateau
 |===
 
+== Status (verified 2026-08-03)
+
+`configs.yml` covers *28 repos*: the 25 rollout targets plus the 3 already-live
+(*ids, oasis, w3c*), folded in so the generator is the single source for every site.
+The 3 live ones carry per-repo `favicon`/`description` overrides (and w3c a
+`pubid_require: relaton/w3c/pubid`); regenerating their `_config.yml` reproduces the
+live site (with `baseurl` corrected from `/main/` to their real default `/v2/`).
+
+Live (200): *relaton-data-ids, -oasis, -w3c* — already rolled out.
+
+Pending (Pages 404): *the other 25 repos in `configs.yml`*. Every one's published index
+(`<baseurl><source>`) already returns 200 — i.e. the data side is ready.
+
+Pages *enablement* state (verified via `gh api repos/relaton/relaton-data-<repo>/pages`):
+
+* *Already enabled* (`build_type: workflow`, source = default branch, no build
+  published yet so the site 404s) — the *20 `v2`-default repos*. These need only
+  steps 2–3 below; *skip step 1*.
+* *Not enabled* (Pages API 404) — the *5 `main`-default repos*: *adobe, easc, gost,
+  jcgm, oiml*. These need step 1, which requires *repo admin* (a plain `push`/`repo`
+  token gets HTTP 404 from the Create-Pages endpoint).
+
+Re-check current state any time with:
+
+[source,sh]
+----
+bin/check-data-pages              # site + raw-index for all 28
+bin/check-data-pages --site-only  # just the Pages sites (the 404s to fix)
+----
+
 == Rollout
 
-. Enable GitHub Pages (source: *GitHub Actions*) on the target repo.
-. Generate and commit the config onto the repo's default branch:
+Do these per target repo (or batch the Cimas step across all of them). Steps 1–3
+all happen *in the data repo* / on GitHub — this `support` repo only holds the
+source of truth and tooling.
+
+. *Enable GitHub Pages* (source: *GitHub Actions*) — *only for the 5 not-yet-enabled
+  `main`-default repos* (adobe, easc, gost, jcgm, oiml); the 20 `v2` repos already
+  have it (see Status). *Requires repo admin* — a `push`-only token gets HTTP 404:
 +
 [source,sh]
 ----
-# print one repo's config
-bin/gen-data-index-config iso
-
-# or write files for several repos into a dir
-bin/gen-data-index-config --out /tmp/cfg 3gpp ogc xsf
-# then copy /tmp/cfg/<repo>_config.yml -> <data repo>/_config.yml and commit
+gh api -X POST repos/relaton/relaton-data-<repo>/pages \
+  -f 'build_type=workflow'      # 409 if already enabled (safe to ignore)
+----
+. *Commit the `_config.yml`* onto the repo's default branch (from `configs.yml`'s
+  `branch:` — `v2` for most, `main` for adobe/easc/gost/jcgm/oiml). Use the
+  committed snapshot:
++
+[source,sh]
+----
+# snapshot already rendered in data-index/generated/
+cp data-index/generated/<repo>_config.yml <data repo>/_config.yml
+# (or regenerate on the fly: bin/gen-data-index-config <repo>)
 ----
-. Roll the Cimas plumbing (`deploy.yml`, `Gemfile.deploy`) to every data repo:
+. *Roll the Cimas plumbing* (`deploy.yml`, `Gemfile.deploy`) to every data repo —
+  from the support repo root, per `../README.adoc`:
 +
 [source,sh]
 ----
-# from the support repo root, per cimas-config usage in ../README.adoc
 cimas sync -f $CIMAS_CONFIG -r $CIMAS_WD -d $CIMAS_MASTER_DIR -g data
 cimas push -f $CIMAS_CONFIG -r $CIMAS_WD -b data-pages -m 'Pages deploy' -g data
+cimas open-prs -f $CIMAS_CONFIG -r $CIMAS_WD -b data-pages -m 'Pages deploy' -g data
 ----
-. Confirm the `Deploy` workflow goes green and the site lists DocID -> file with
-  working pagination.
-
-TIP: Start with Tier A (renders cleanly today); hold Tier B/C until the theme
-hand-off lands.
+. *Confirm* the `Deploy` workflow goes green and each site returns 200:
++
+[source,sh]
+----
+bin/check-data-pages              # exits non-zero until every repo is 200
+----
++
+The site should list DocID -> file with working pagination and a citable DocID
+per row.
+
+TIP: Start with Tier A (renders cleanly today); Tier B/C need the flavor pubid gem
+in the build bundle, which `cimas-config/gh-actions/data/Gemfile.deploy` already
+provides (it `eval_gemfile`s each repo's own Gemfile). See "Rendering readiness".
+
+NOTE: `configs.yml` and `../cimas-config/cimas.yml` must agree on every repo's
+`branch:` and both must map `deploy.yml` + `Gemfile.deploy`; a drift silently
+skips the publish (the deploy job only runs on the repo's default branch).
+`spec/cimas_data_pages_spec.rb` enforces this — run `bundle exec rspec` after
+editing either file.

data-index/configs.yml

@@ -14,6 +14,12 @@
 #                 MUST match the `pubid_class:` the flavor passes to
 #                 Relaton::Index.find_or_create in relaton's lib/relaton/<flavor>/data_fetcher.rb.
 #                 No leading "::" — the theme plugin resolves it with Object.const_get.
+#
+# Optional per-entry overrides (fall back to the defaults/template when absent) —
+# used by the already-live ids/oasis/w3c to keep their own branding:
+#   favicon        - override defaults.favicon
+#   description     - override the templated "Welcome to the <display> ..." line
+#   pubid_require   - override defaults.pubid_require (w3c uses relaton/w3c/pubid)
 
 defaults:
   paginate: 100
@@ -48,3 +54,37 @@ repos:
   - { repo: rfcs,         display: RFC,           source: index-v1.yaml, branch: v2,   pubid_class: }
   - { repo: rfcsubseries, display: RFC Subseries, source: index-v1.yaml, branch: v2,   pubid_class: }
   - { repo: xsf,          display: XSF,           source: index-v1.yaml, branch: v2,   pubid_class: }
+
+  # Already-live repos (their sites returned 200 before this generator existed).
+  # Folded in so the generator covers all 28; they carry per-repo favicon/
+  # description overrides (and w3c a non-default pubid_require). branch: v2 is
+  # each repo's real default (index-v1.yaml is served from v2).
+  - repo: ids
+    display: Internet-Drafts
+    source: index-v1.yaml
+    branch: v2
+    pubid_class:
+    favicon: 'https://www.ietf.org/static/img/ietf-logo.dc9646b3c716.svg'
+    description: 'Bibliographic data information for Internet-Drafts in Relaton format'
+  - repo: oasis
+    display: OASIS
+    source: index-v1.yaml
+    branch: v2
+    pubid_class:
+    favicon: 'https://www.oasis-open.org/wp-content/uploads/2020/11/cropped-2-layers@3x-32x32.png'
+    description: >-
+      Welcome to the OASIS standards index site! This is where you can find the
+      information you need about the various standards developed and maintained by
+      the OASIS open community. Explore the OASIS standards and learn how they can
+      help you solve your business and technical challenges!
+  - repo: w3c
+    display: W3C
+    source: index-v1.yaml
+    branch: v2
+    pubid_class: Relaton::W3c::PubId
+    pubid_require: relaton/w3c/pubid
+    favicon: 'https://www.w3.org/assets/logos/w3c/w3c-no-bars.svg'
+    description: >-
+      Welcome to the World Wide Web Consortium standards index site! This is where
+      you can find the information you need about the various standards developed and
+      maintained by the W3C open community.

data-index/generated/3gpp_config.yml

@@ -0,0 +1,9 @@
+title: 3GPP Index
+description: >-
+  Welcome to the 3GPP standards index site!
+paginate: 100
+jekyll-index:
+  favicon: 'https://www.relaton.org/favicon.ico'
+  source: 'index-v1.yaml'
+  baseurl: 'https://raw.githubusercontent.com/relaton/relaton-data-3gpp/v2/'
+  add_type_to_reference: true

data-index/generated/README.adoc

@@ -0,0 +1,10 @@
+= Generated relaton-data-* `_config.yml` snapshot
+
+*Do not edit by hand.* These files are generated from `../configs.yml` by
+`../../bin/gen-data-index-config --out data-index/generated` and are kept in sync
+by `spec/generated_configs_spec.rb`.
+
+Each `<repo>_config.yml` is the exact `_config.yml` to commit into
+`relaton/relaton-data-<repo>` (on that repo's default branch) as part of the Pages
+rollout — see `../README.adoc`. To change a repo's config, edit `../configs.yml`
+and regenerate; never edit these files directly.

data-index/generated/adobe_config.yml

@@ -0,0 +1,11 @@
+title: Adobe Index
+description: >-
+  Welcome to the Adobe standards index site!
+paginate: 100
+jekyll-index:
+  favicon: 'https://www.relaton.org/favicon.ico'
+  source: 'index-v2.yaml'
+  baseurl: 'https://raw.githubusercontent.com/relaton/relaton-data-adobe/main/'
+  add_type_to_reference: true
+  pubid_class: 'Pubid::Adobe::Identifier'
+  pubid_require: 'pubid'

data-index/generated/bipm_config.yml

@@ -0,0 +1,9 @@
+title: BIPM Index
+description: >-
+  Welcome to the BIPM standards index site!
+paginate: 100
+jekyll-index:
+  favicon: 'https://www.relaton.org/favicon.ico'
+  source: 'index-v1.yaml'
+  baseurl: 'https://raw.githubusercontent.com/relaton/relaton-data-bipm/v2/'
+  add_type_to_reference: true

data-index/generated/calconnect_config.yml

@@ -0,0 +1,9 @@
+title: CalConnect Index
+description: >-
+  Welcome to the CalConnect standards index site!
+paginate: 100
+jekyll-index:
+  favicon: 'https://www.relaton.org/favicon.ico'
+  source: 'index-v1.yaml'
+  baseurl: 'https://raw.githubusercontent.com/relaton/relaton-data-calconnect/v2/'
+  add_type_to_reference: true

data-index/generated/ccsds_config.yml

@@ -0,0 +1,11 @@
+title: CCSDS Index
+description: >-
+  Welcome to the CCSDS standards index site!
+paginate: 100
+jekyll-index:
+  favicon: 'https://www.relaton.org/favicon.ico'
+  source: 'index-v2.yaml'
+  baseurl: 'https://raw.githubusercontent.com/relaton/relaton-data-ccsds/v2/'
+  add_type_to_reference: true
+  pubid_class: 'Pubid::Ccsds::Identifier'
+  pubid_require: 'pubid'