diff --git a/.github/workflows/data-deploy.yml b/.github/workflows/data-deploy.yml index afc6c42..d381386 100644 --- a/.github/workflows/data-deploy.yml +++ b/.github/workflows/data-deploy.yml @@ -14,6 +14,12 @@ name: Deploy # deploy: # uses: relaton/support/.github/workflows/data-deploy.yml@main # +# Callers should publish on the crawler finishing rather than on a cron guessed +# to land after it — see `cimas-config/gh-actions/data/deploy.yml`, the template +# Cimas syncs into every relaton-data-* repo. A caller needs only the trigger; +# both the "skip a failed crawler run" guard and the Pages concurrency group are +# declared here, so no per-repo copy can drift or forget them. +# # Optionally, a caller passes the per-flavor branding its `_config.yml` used to # carry — `title`, `favicon` and `description` (see relaton/relaton#96). Left # unset, the page carries no favicon and no description, exactly as before. @@ -79,8 +85,45 @@ on: permissions: contents: read +# Serialize Pages deployments per calling repo. Runs overlap easily — the caller +# template fires on `workflow_run` *and* a fallback cron, and a `source: git` +# build takes ~10 minutes — and two concurrent actions/deploy-pages runs make the +# loser fail with a concurrent-deployment error. Mirrors GitHub's own Pages +# starter workflow. +# +# Declared at the top level of this *called* workflow, which is where GitHub +# documents concurrency for a reusable workflow to live: `jobs..concurrency` +# on the calling job does not behave as expected. One group per caller repo +# (`github.repository` evaluates in the caller's context), so no data repo ever +# queues behind a sibling. `cancel-in-progress: false` is deliberate — queue +# deployments rather than kill a Pages deploy mid-flight. +# +# Only runs that can actually publish share the queue. Under GitHub's default +# `queue: single` a *pending* run is cancelled the moment a third joins its +# group, so anything parked in the deployment queue that will never deploy can +# drop a queued run carrying freshly crawled data — the staleness this +# workflow's triggers exist to avoid. Pull requests, tag pushes and pushes to a +# non-default branch all build for ~10 minutes and then skip `deploy`, so they +# are keyed by ref into groups of their own. +# +# The discriminator is the deploy job's own gate, so the two cannot drift. +concurrency: + group: >- + pages-${{ github.repository }}-${{ + github.ref == format('refs/heads/{0}', github.event.repository.default_branch) + && 'deploy' || github.ref }} + cancel-in-progress: false + jobs: build_index_page: + # A `workflow_run` caller fires on `completed`, not `success` (there is no + # `success` activity type), so filter the conclusion here: rebuild for every + # other trigger, but skip a run whose crawler actually failed. This lives in + # the shared workflow rather than in each caller's `deploy:` job so no caller + # can forget it — a called workflow inherits the caller run's `github` + # context, event payload included. `deploy` is `needs: build_index_page`, so + # skipping the build skips the publish with it. + if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 # the data repo (has ./data) diff --git a/cimas-config/gh-actions/data/deploy.yml b/cimas-config/gh-actions/data/deploy.yml index ca7e6ad..8b81c34 100644 --- a/cimas-config/gh-actions/data/deploy.yml +++ b/cimas-config/gh-actions/data/deploy.yml @@ -5,13 +5,48 @@ on: # Include v2: many relaton-data-* repos moved their default branch to v2. # The reusable data-deploy.yml still gates publication on the repo's actual # default branch, so listing all three here is safe for main/master repos. + # + # NOTE: this trigger does NOT fire on crawler commits. crawler.yml pushes + # its data with the default actions/checkout GITHUB_TOKEN, and GitHub + # deliberately raises no workflow events for GITHUB_TOKEN pushes (it would + # recurse). Fresh data reaches Pages via `workflow_run` below; this trigger + # only ever covers human pushes. branches: [ master, main, v2 ] tags: [ v* ] pull_request: + # Publish because the crawl finished, not an hour after it was meant to. + # Scheduled dispatch drifts badly under load: 11 of relaton-data-iana's last + # 12 Crawler runs *started* at or after 14:58 UTC despite a 14:00 cron, + # several past 16:00 — so the old 15:00 deploy usually indexed the previous + # day's data, and on a bad day was a full cycle behind. + # + # `workflows:` matches crawler.yml's `name:`; Cimas syncs both templates, so + # they stay in step (pinned by spec/data_deploy_caller_template_spec.rb). + # `branches:` filters the *triggering* run's branch, so the same repo-agnostic + # set as `push` above needs no per-repo edit. The run itself is against the + # repo's default branch, which keeps data-deploy.yml's publish gate and the + # raw-YAML --base-url links correct. + workflow_run: + workflows: [ Crawler ] + types: [ completed ] + branches: [ master, main, v2 ] + # Fallback only, and it is NOT about days the crawler commits nothing — + # `types: [completed]` fires on every crawler run regardless of whether the + # "Push data" step committed. What this covers is a crawler that never ran or + # was disabled, and picking up upstream relaton/relaton changes that alter the + # generated index. Cost: a second full build per day (~10 min under + # `source: git`, which compiles the frontend); drop to weekly if that bites. + # + # Set clear of the crawler's observed run window rather than one hour after + # its nominal cron — see the note on workflow_run above. schedule: - - cron: '0 15 * * *' + - cron: '0 18 * * *' workflow_dispatch: jobs: deploy: - uses: relaton/support/.github/workflows/data-deploy.yml@main \ No newline at end of file + # No `if:` and no `concurrency:` here on purpose. data-deploy.yml declares + # both centrally — it skips a `workflow_run` whose crawler failed, and + # serializes Pages deployments per repo — so every caller gets them without + # a copy of its own to drift. + uses: relaton/support/.github/workflows/data-deploy.yml@main diff --git a/data-index/README.adoc b/data-index/README.adoc index 630b05f..b1aa87a 100644 --- a/data-index/README.adoc +++ b/data-index/README.adoc @@ -31,6 +31,48 @@ the byte-identical plumbing (`deploy.yml`, `Gemfile.deploy`). This generator is single place the per-repo configs are produced; the output is committed into each data repo, where a maintainer may further tune it without a central clobber. +== When the site rebuilds + +The Cimas template `cimas-config/gh-actions/data/deploy.yml` publishes on +`workflow_run`: the Deploy run starts because the repo's `Crawler` workflow +completed, not on a cron guessed to land after it. That ordering used to be +time-based (crawler at 14:00, deploy at 15:00) and did not hold — GitHub's +scheduled dispatch drifts heavily under load, so the deploy usually indexed the +previous day's data. The `push` trigger cannot cover the gap either: the crawler +commits with the default `GITHUB_TOKEN`, and GitHub raises no workflow events +for those pushes. + +The `0 18 * * *` cron is a fallback, placed clear of the crawler's observed run +window rather than one hour after its nominal cron. Note what it does *not* +cover: `types: [completed]` fires on every crawler run whether or not anything +was committed, so the cron is not there for quiet days. It covers a crawler that +never ran or was disabled, and upstream `relaton/relaton` changes that alter the +generated index — at the price of a second full build per day. + +Two things a caller deliberately does *not* declare, because `data-deploy.yml` +declares them centrally for the whole fleet: the `if:` that skips a `workflow_run` +whose crawler failed, and the Pages `concurrency` group. That group is keyed on +the deploy job's own default-branch gate, so only runs that can actually publish +share a queue — GitHub keeps just one *pending* run per group and cancels it when +the next arrives, so a tag push or PR build sharing the queue could otherwise +drop a run carrying fresh data. + +[CAUTION] +==== +Cimas replaces `deploy.yml` wholesale in all 29 mapped repos. Four of them carry +a hand-added `with:` block that nothing here regenerates — `configs.yml` renders +`_config.yml` only: + +* `relaton-data-bipm`, `relaton-data-3gpp` — `source: git` +* `relaton-data-w3c`, `relaton-data-iana` — `source: git` plus `favicon:` and + `description:` + +Syncing over them drops those inputs back to `source: gem`, which fails with +`Could not find command "index"` until relaton-cli ships `index`, and loses the +branding. Re-apply the `with:` blocks after a sync, or hold those four back and +edit them by hand. +==== + == The pubid rule The theme plugin renders a structured index `:id` (a Hash) by instantiating a diff --git a/spec/data_deploy_caller_template_spec.rb b/spec/data_deploy_caller_template_spec.rb new file mode 100644 index 0000000..8ae763c --- /dev/null +++ b/spec/data_deploy_caller_template_spec.rb @@ -0,0 +1,87 @@ +# Guards the Cimas caller template that Cimas syncs into every relaton-data-* +# repo as `.github/workflows/deploy.yml`. +# +# The problem it encodes: the template's `push` trigger cannot fire on crawled +# data. crawler.yml's "Push data" step commits with the default +# actions/checkout GITHUB_TOKEN, and GitHub deliberately raises no workflow +# events for GITHUB_TOKEN pushes. That left the daily cron as the only path from +# crawl to Pages — and it assumed a fixed one-hour gap that scheduled dispatch +# does not honour (11 of relaton-data-iana's last 12 Crawler runs *started* at +# or after 14:58 UTC despite a 14:00 cron, several past 16:00), so the 15:00 +# deploy usually indexed the previous day's data. +# +# Ordering is now deterministic via `workflow_run`. The couplings below are all +# silent when broken — nothing goes red in CI, the fleet just quietly stops +# republishing — so they are pinned here. +RSpec.describe "cimas-config/gh-actions/data/deploy.yml" do + repo_root = File.expand_path("..", __dir__) + template = ->(name) { YAML.safe_load_file(File.join(repo_root, "cimas-config/gh-actions/data", name)) } + + deploy = template.call("deploy.yml") + crawler = template.call("crawler.yml") + # Psych reads the unquoted `on:` key as YAML 1.1 boolean true. + triggers = deploy.fetch(true) + workflow_run = triggers["workflow_run"] + + # "0 14 * * *" -> 14 + cron_hour = ->(schedule) { Integer(schedule.fetch(0).fetch("cron").split.fetch(1)) } + + describe "the workflow_run trigger" do + it "is declared, so a crawler commit reaches Pages the same day" do + expect(workflow_run).to be_a(Hash) + end + + it "names the crawler template's own `name:`" do + # THE load-bearing assertion. `workflows:` matches the triggering + # workflow's `name:` field, and both files are synced together by Cimas — + # so renaming crawler.yml's `name:` silently stops the deploy trigger + # firing in every data repo, with nothing red to notice. + expect(workflow_run.fetch("workflows")).to eq([crawler.fetch("name")]) + end + + it "listens for completion rather than success" do + # `success` is not a valid workflow_run activity type; the conclusion is + # filtered centrally instead, by data-deploy.yml's build_index_page `if:`. + expect(workflow_run.fetch("types")).to eq(["completed"]) + end + + it "accepts the same default branches as the push trigger" do + # `workflow_run`'s branch filter matches the *triggering* run's branch. + # Listing the same set as `push` keeps the template repo-agnostic (many + # relaton-data-* repos moved their default branch to v2), so this needs no + # per-repo edit. Publication is still gated on the repo's real default + # branch inside data-deploy.yml. + expect(workflow_run.fetch("branches")).to eq(triggers.fetch("push").fetch("branches")) + end + end + + it "schedules its fallback cron clear of the crawler's observed window" do + # A fallback for days the crawler commits nothing, not the primary path. + # The old "one hour after the crawl" gap was inside the drift, so require a + # real margin over the crawler template's own cron. + expect(cron_hour.call(triggers.fetch("schedule"))) + .to be >= cron_hour.call(crawler.fetch(true).fetch("schedule")) + 3 + end + + describe "what it deliberately leaves to the shared workflow" do + deploy_job = deploy.fetch("jobs").fetch("deploy") + + it "calls the shared data-deploy.yml" do + expect(deploy_job.fetch("uses")).to eq("relaton/support/.github/workflows/data-deploy.yml@main") + end + + it "declares no concurrency of its own" do + # data-deploy.yml owns it (see spec/data_deploy_workflow_spec.rb). A + # duplicate group here would only add a second queue in front of the same + # deployment, and per GitHub's docs a caller-side group that collides with + # the called workflow's can cancel the caller. + expect(deploy).not_to have_key("concurrency") + expect(deploy_job).not_to have_key("concurrency") + end + + it "carries no failed-crawler `if:` guard" do + # Also central, so a repo whose deploy.yml drifts still gets it. + expect(deploy_job).not_to have_key("if") + end + end +end diff --git a/spec/data_deploy_workflow_spec.rb b/spec/data_deploy_workflow_spec.rb index 2063515..bf2827b 100644 --- a/spec/data_deploy_workflow_spec.rb +++ b/spec/data_deploy_workflow_spec.rb @@ -13,6 +13,10 @@ inputs = workflow.fetch(true).fetch("workflow_call").fetch("inputs") build_job = workflow.fetch("jobs").fetch("build_index_page") index_steps = build_job.fetch("steps").select { |s| s["run"]&.match?(/\brelaton index\b/) } + # "Publish only from the repo's real default branch" — the one condition that + # decides whether a run reaches actions/deploy-pages, reused by the + # concurrency group below so the two cannot drift apart. + deploy_gate = "github.ref == format('refs/heads/{0}', github.event.repository.default_branch)" it "keeps every input optional, so existing callers need no changes" do expect(inputs.values).to all(include("required" => false)) @@ -61,4 +65,60 @@ expect(index_steps.map { |s| s.fetch("run") }).to all(include("#{flag} ")) end end + + it "publishes only from the repository's real default branch" do + # Hardcoding master/main once excluded the relaton-data-* repos that moved + # their default branch to v2 — fresh data built, never published. + expect(workflow.fetch("jobs").fetch("deploy").fetch("if")).to eq(deploy_gate) + end + + describe "the concurrency group" do + concurrency = workflow["concurrency"] + + it "is declared, so two Pages deployments cannot collide" do + # Both runs reach actions/deploy-pages@v4 and the loser fails with a + # concurrent-deployment error. Overlap is easy: the caller template fires + # on `workflow_run` *and* a fallback cron, and a `source: git` build runs + # for ~10 minutes. Declared here rather than per caller because GitHub + # documents the *called* workflow's top level as where concurrency for a + # reusable workflow belongs — `jobs..concurrency` on the calling job + # "will not behave as expected". + expect(concurrency).to be_a(Hash) + end + + it "scopes the group to the calling repository" do + # `github.repository` evaluates in the caller's context, so each + # relaton-data-* repo queues against itself and never against a sibling. + expect(concurrency.fetch("group")).to start_with("pages-${{ github.repository }}") + end + + it "queues only the runs that can actually publish" do + # Under GitHub's default `queue: single` a pending run is cancelled the + # moment a third joins its group. Pull requests, tag pushes and pushes to + # a non-default branch all build and then skip `deploy`, so parking them + # in the deployment queue would let them drop a pending run carrying + # freshly crawled data. Keying on the deploy job's *own* gate is what + # keeps the split honest — assert they are literally the same expression, + # so narrowing one and forgetting the other fails here. + expect(concurrency.fetch("group")).to include(deploy_gate) + end + + it "queues deployments instead of cancelling one mid-flight" do + # Deliberate, and the opposite of GitHub's default: a cancelled + # actions/deploy-pages run can leave the Pages deployment half-applied. + expect(concurrency.fetch("cancel-in-progress")).to be(false) + end + end + + it "skips a build whose triggering workflow run failed" do + # A `workflow_run` caller fires on `completed`, not `success`. The guard + # lives here, not in each caller's `deploy:` job, so no caller can forget + # it — a called workflow inherits the caller run's `github` context, + # event payload included. `deploy` is `needs: build_index_page`, so + # skipping the build skips the publish too. + expect(build_job.fetch("if")).to eq( + "github.event_name != 'workflow_run' || " \ + "github.event.workflow_run.conclusion == 'success'", + ) + end end