ci(data-deploy): publish on the crawler, serialize Pages deploys

.github/workflows/data-deploy.yml

@@ -14,6 +14,12 @@ name: Deploy
 #     deploy:
 #       uses: relaton/support/.github/workflows/data-deploy.yml@main
 #
+# Callers should publish on the crawler finishing rather than on a cron guessed
+# to land after it — see `cimas-config/gh-actions/data/deploy.yml`, the template
+# Cimas syncs into every relaton-data-* repo. A caller needs only the trigger;
+# both the "skip a failed crawler run" guard and the Pages concurrency group are
+# declared here, so no per-repo copy can drift or forget them.
+#
 # Optionally, a caller passes the per-flavor branding its `_config.yml` used to
 # carry — `title`, `favicon` and `description` (see relaton/relaton#96). Left
 # unset, the page carries no favicon and no description, exactly as before.
@@ -79,8 +85,45 @@ on:
 permissions:
   contents: read
 
+# Serialize Pages deployments per calling repo. Runs overlap easily — the caller
+# template fires on `workflow_run` *and* a fallback cron, and a `source: git`
+# build takes ~10 minutes — and two concurrent actions/deploy-pages runs make the
+# loser fail with a concurrent-deployment error. Mirrors GitHub's own Pages
+# starter workflow.
+#
+# Declared at the top level of this *called* workflow, which is where GitHub
+# documents concurrency for a reusable workflow to live: `jobs.<id>.concurrency`
+# on the calling job does not behave as expected. One group per caller repo
+# (`github.repository` evaluates in the caller's context), so no data repo ever
+# queues behind a sibling. `cancel-in-progress: false` is deliberate — queue
+# deployments rather than kill a Pages deploy mid-flight.
+#
+# Only runs that can actually publish share the queue. Under GitHub's default
+# `queue: single` a *pending* run is cancelled the moment a third joins its
+# group, so anything parked in the deployment queue that will never deploy can
+# drop a queued run carrying freshly crawled data — the staleness this
+# workflow's triggers exist to avoid. Pull requests, tag pushes and pushes to a
+# non-default branch all build for ~10 minutes and then skip `deploy`, so they
+# are keyed by ref into groups of their own.
+#
+# The discriminator is the deploy job's own gate, so the two cannot drift.
+concurrency:
+  group: >-
+    pages-${{ github.repository }}-${{
+    github.ref == format('refs/heads/{0}', github.event.repository.default_branch)
+    && 'deploy' || github.ref }}
+  cancel-in-progress: false
+
 jobs:
   build_index_page:
+    # A `workflow_run` caller fires on `completed`, not `success` (there is no
+    # `success` activity type), so filter the conclusion here: rebuild for every
+    # other trigger, but skip a run whose crawler actually failed. This lives in
+    # the shared workflow rather than in each caller's `deploy:` job so no caller
+    # can forget it — a called workflow inherits the caller run's `github`
+    # context, event payload included. `deploy` is `needs: build_index_page`, so
+    # skipping the build skips the publish with it.
+    if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4 # the data repo (has ./data)

cimas-config/gh-actions/data/deploy.yml

@@ -5,13 +5,48 @@ on:
     # Include v2: many relaton-data-* repos moved their default branch to v2.
     # The reusable data-deploy.yml still gates publication on the repo's actual
     # default branch, so listing all three here is safe for main/master repos.
+    #
+    # NOTE: this trigger does NOT fire on crawler commits. crawler.yml pushes
+    # its data with the default actions/checkout GITHUB_TOKEN, and GitHub
+    # deliberately raises no workflow events for GITHUB_TOKEN pushes (it would
+    # recurse). Fresh data reaches Pages via `workflow_run` below; this trigger
+    # only ever covers human pushes.
     branches: [ master, main, v2 ]
     tags: [ v* ]
   pull_request:
+  # Publish because the crawl finished, not an hour after it was meant to.
+  # Scheduled dispatch drifts badly under load: 11 of relaton-data-iana's last
+  # 12 Crawler runs *started* at or after 14:58 UTC despite a 14:00 cron,
+  # several past 16:00 — so the old 15:00 deploy usually indexed the previous
+  # day's data, and on a bad day was a full cycle behind.
+  #
+  # `workflows:` matches crawler.yml's `name:`; Cimas syncs both templates, so
+  # they stay in step (pinned by spec/data_deploy_caller_template_spec.rb).
+  # `branches:` filters the *triggering* run's branch, so the same repo-agnostic
+  # set as `push` above needs no per-repo edit. The run itself is against the
+  # repo's default branch, which keeps data-deploy.yml's publish gate and the
+  # raw-YAML --base-url links correct.
+  workflow_run:
+    workflows: [ Crawler ]
+    types: [ completed ]
+    branches: [ master, main, v2 ]
+  # Fallback only, and it is NOT about days the crawler commits nothing —
+  # `types: [completed]` fires on every crawler run regardless of whether the
+  # "Push data" step committed. What this covers is a crawler that never ran or
+  # was disabled, and picking up upstream relaton/relaton changes that alter the
+  # generated index. Cost: a second full build per day (~10 min under
+  # `source: git`, which compiles the frontend); drop to weekly if that bites.
+  #
+  # Set clear of the crawler's observed run window rather than one hour after
+  # its nominal cron — see the note on workflow_run above.
   schedule:
-  - cron: '0 15 * * *'
+  - cron: '0 18 * * *'
   workflow_dispatch:
 
 jobs:
   deploy:
-    uses: relaton/support/.github/workflows/data-deploy.yml@main
+    # No `if:` and no `concurrency:` here on purpose. data-deploy.yml declares
+    # both centrally — it skips a `workflow_run` whose crawler failed, and
+    # serializes Pages deployments per repo — so every caller gets them without
+    # a copy of its own to drift.
+    uses: relaton/support/.github/workflows/data-deploy.yml@main

data-index/README.adoc

@@ -31,6 +31,48 @@ the byte-identical plumbing (`deploy.yml`, `Gemfile.deploy`). This generator is
 single place the per-repo configs are produced; the output is committed into each
 data repo, where a maintainer may further tune it without a central clobber.
 
+== When the site rebuilds
+
+The Cimas template `cimas-config/gh-actions/data/deploy.yml` publishes on
+`workflow_run`: the Deploy run starts because the repo's `Crawler` workflow
+completed, not on a cron guessed to land after it. That ordering used to be
+time-based (crawler at 14:00, deploy at 15:00) and did not hold — GitHub's
+scheduled dispatch drifts heavily under load, so the deploy usually indexed the
+previous day's data. The `push` trigger cannot cover the gap either: the crawler
+commits with the default `GITHUB_TOKEN`, and GitHub raises no workflow events
+for those pushes.
+
+The `0 18 * * *` cron is a fallback, placed clear of the crawler's observed run
+window rather than one hour after its nominal cron. Note what it does *not*
+cover: `types: [completed]` fires on every crawler run whether or not anything
+was committed, so the cron is not there for quiet days. It covers a crawler that
+never ran or was disabled, and upstream `relaton/relaton` changes that alter the
+generated index — at the price of a second full build per day.
+
+Two things a caller deliberately does *not* declare, because `data-deploy.yml`
+declares them centrally for the whole fleet: the `if:` that skips a `workflow_run`
+whose crawler failed, and the Pages `concurrency` group. That group is keyed on
+the deploy job's own default-branch gate, so only runs that can actually publish
+share a queue — GitHub keeps just one *pending* run per group and cancels it when
+the next arrives, so a tag push or PR build sharing the queue could otherwise
+drop a run carrying fresh data.
+
+[CAUTION]
+====
+Cimas replaces `deploy.yml` wholesale in all 29 mapped repos. Four of them carry
+a hand-added `with:` block that nothing here regenerates — `configs.yml` renders
+`_config.yml` only:
+
+* `relaton-data-bipm`, `relaton-data-3gpp` — `source: git`
+* `relaton-data-w3c`, `relaton-data-iana` — `source: git` plus `favicon:` and
+  `description:`
+
+Syncing over them drops those inputs back to `source: gem`, which fails with
+`Could not find command "index"` until relaton-cli ships `index`, and loses the
+branding. Re-apply the `with:` blocks after a sync, or hold those four back and
+edit them by hand.
+====
+
 == The pubid rule
 
 The theme plugin renders a structured index `:id` (a Hash) by instantiating a

spec/data_deploy_caller_template_spec.rb

@@ -0,0 +1,87 @@
+# Guards the Cimas caller template that Cimas syncs into every relaton-data-*
+# repo as `.github/workflows/deploy.yml`.
+#
+# The problem it encodes: the template's `push` trigger cannot fire on crawled
+# data. crawler.yml's "Push data" step commits with the default
+# actions/checkout GITHUB_TOKEN, and GitHub deliberately raises no workflow
+# events for GITHUB_TOKEN pushes. That left the daily cron as the only path from
+# crawl to Pages — and it assumed a fixed one-hour gap that scheduled dispatch
+# does not honour (11 of relaton-data-iana's last 12 Crawler runs *started* at
+# or after 14:58 UTC despite a 14:00 cron, several past 16:00), so the 15:00
+# deploy usually indexed the previous day's data.
+#
+# Ordering is now deterministic via `workflow_run`. The couplings below are all
+# silent when broken — nothing goes red in CI, the fleet just quietly stops
+# republishing — so they are pinned here.
+RSpec.describe "cimas-config/gh-actions/data/deploy.yml" do
+  repo_root = File.expand_path("..", __dir__)
+  template = ->(name) { YAML.safe_load_file(File.join(repo_root, "cimas-config/gh-actions/data", name)) }
+
+  deploy = template.call("deploy.yml")
+  crawler = template.call("crawler.yml")
+  # Psych reads the unquoted `on:` key as YAML 1.1 boolean true.
+  triggers = deploy.fetch(true)
+  workflow_run = triggers["workflow_run"]
+
+  # "0 14 * * *" -> 14
+  cron_hour = ->(schedule) { Integer(schedule.fetch(0).fetch("cron").split.fetch(1)) }
+
+  describe "the workflow_run trigger" do
+    it "is declared, so a crawler commit reaches Pages the same day" do
+      expect(workflow_run).to be_a(Hash)
+    end
+
+    it "names the crawler template's own `name:`" do
+      # THE load-bearing assertion. `workflows:` matches the triggering
+      # workflow's `name:` field, and both files are synced together by Cimas —
+      # so renaming crawler.yml's `name:` silently stops the deploy trigger
+      # firing in every data repo, with nothing red to notice.
+      expect(workflow_run.fetch("workflows")).to eq([crawler.fetch("name")])
+    end
+
+    it "listens for completion rather than success" do
+      # `success` is not a valid workflow_run activity type; the conclusion is
+      # filtered centrally instead, by data-deploy.yml's build_index_page `if:`.
+      expect(workflow_run.fetch("types")).to eq(["completed"])
+    end
+
+    it "accepts the same default branches as the push trigger" do
+      # `workflow_run`'s branch filter matches the *triggering* run's branch.
+      # Listing the same set as `push` keeps the template repo-agnostic (many
+      # relaton-data-* repos moved their default branch to v2), so this needs no
+      # per-repo edit. Publication is still gated on the repo's real default
+      # branch inside data-deploy.yml.
+      expect(workflow_run.fetch("branches")).to eq(triggers.fetch("push").fetch("branches"))
+    end
+  end
+
+  it "schedules its fallback cron clear of the crawler's observed window" do
+    # A fallback for days the crawler commits nothing, not the primary path.
+    # The old "one hour after the crawl" gap was inside the drift, so require a
+    # real margin over the crawler template's own cron.
+    expect(cron_hour.call(triggers.fetch("schedule")))
+      .to be >= cron_hour.call(crawler.fetch(true).fetch("schedule")) + 3
+  end
+
+  describe "what it deliberately leaves to the shared workflow" do
+    deploy_job = deploy.fetch("jobs").fetch("deploy")
+
+    it "calls the shared data-deploy.yml" do
+      expect(deploy_job.fetch("uses")).to eq("relaton/support/.github/workflows/data-deploy.yml@main")
+    end
+
+    it "declares no concurrency of its own" do
+      # data-deploy.yml owns it (see spec/data_deploy_workflow_spec.rb). A
+      # duplicate group here would only add a second queue in front of the same
+      # deployment, and per GitHub's docs a caller-side group that collides with
+      # the called workflow's can cancel the caller.
+      expect(deploy).not_to have_key("concurrency")
+      expect(deploy_job).not_to have_key("concurrency")
+    end
+
+    it "carries no failed-crawler `if:` guard" do
+      # Also central, so a repo whose deploy.yml drifts still gets it.
+      expect(deploy_job).not_to have_key("if")
+    end
+  end
+end

spec/data_deploy_workflow_spec.rb

@@ -13,6 +13,10 @@
   inputs = workflow.fetch(true).fetch("workflow_call").fetch("inputs")
   build_job = workflow.fetch("jobs").fetch("build_index_page")
   index_steps = build_job.fetch("steps").select { |s| s["run"]&.match?(/\brelaton index\b/) }
+  # "Publish only from the repo's real default branch" — the one condition that
+  # decides whether a run reaches actions/deploy-pages, reused by the
+  # concurrency group below so the two cannot drift apart.
+  deploy_gate = "github.ref == format('refs/heads/{0}', github.event.repository.default_branch)"
 
   it "keeps every input optional, so existing callers need no changes" do
     expect(inputs.values).to all(include("required" => false))
@@ -61,4 +65,60 @@
       expect(index_steps.map { |s| s.fetch("run") }).to all(include("#{flag} "))
     end
   end
+
+  it "publishes only from the repository's real default branch" do
+    # Hardcoding master/main once excluded the relaton-data-* repos that moved
+    # their default branch to v2 — fresh data built, never published.
+    expect(workflow.fetch("jobs").fetch("deploy").fetch("if")).to eq(deploy_gate)
+  end
+
+  describe "the concurrency group" do
+    concurrency = workflow["concurrency"]
+
+    it "is declared, so two Pages deployments cannot collide" do
+      # Both runs reach actions/deploy-pages@v4 and the loser fails with a
+      # concurrent-deployment error. Overlap is easy: the caller template fires
+      # on `workflow_run` *and* a fallback cron, and a `source: git` build runs
+      # for ~10 minutes. Declared here rather than per caller because GitHub
+      # documents the *called* workflow's top level as where concurrency for a
+      # reusable workflow belongs — `jobs.<id>.concurrency` on the calling job
+      # "will not behave as expected".
+      expect(concurrency).to be_a(Hash)
+    end
+
+    it "scopes the group to the calling repository" do
+      # `github.repository` evaluates in the caller's context, so each
+      # relaton-data-* repo queues against itself and never against a sibling.
+      expect(concurrency.fetch("group")).to start_with("pages-${{ github.repository }}")
+    end
+
+    it "queues only the runs that can actually publish" do
+      # Under GitHub's default `queue: single` a pending run is cancelled the
+      # moment a third joins its group. Pull requests, tag pushes and pushes to
+      # a non-default branch all build and then skip `deploy`, so parking them
+      # in the deployment queue would let them drop a pending run carrying
+      # freshly crawled data. Keying on the deploy job's *own* gate is what
+      # keeps the split honest — assert they are literally the same expression,
+      # so narrowing one and forgetting the other fails here.
+      expect(concurrency.fetch("group")).to include(deploy_gate)
+    end
+
+    it "queues deployments instead of cancelling one mid-flight" do
+      # Deliberate, and the opposite of GitHub's default: a cancelled
+      # actions/deploy-pages run can leave the Pages deployment half-applied.
+      expect(concurrency.fetch("cancel-in-progress")).to be(false)
+    end
+  end
+
+  it "skips a build whose triggering workflow run failed" do
+    # A `workflow_run` caller fires on `completed`, not `success`. The guard
+    # lives here, not in each caller's `deploy:` job, so no caller can forget
+    # it — a called workflow inherits the caller run's `github` context,
+    # event payload included. `deploy` is `needs: build_index_page`, so
+    # skipping the build skips the publish too.
+    expect(build_job.fetch("if")).to eq(
+      "github.event_name != 'workflow_run' || " \
+      "github.event.workflow_run.conclusion == 'success'",
+    )
+  end
 end