diff --git a/.github/workflows/actions.lock b/.github/workflows/actions.lock index db8f004b..8bc97290 100644 --- a/.github/workflows/actions.lock +++ b/.github/workflows/actions.lock @@ -49,12 +49,12 @@ workflows: - 'actions/upload-artifact@v7.0.2' - 'step-security/harden-runner@v2.22.3' '.github/workflows/governance.yml': - - 'metadatastician/standards@333b06a01db1874ffedb2de1bfabb195640927c7' + - 'metadatastician/standards@1725bebd3773ea25627479f813283abf1c497a85' '.github/workflows/hypatia-remediation-sweep.yml': - 'actions/upload-artifact@v7.0.2' - 'step-security/harden-runner@v2.22.3' '.github/workflows/hypatia-scan.yml': - - 'metadatastician/standards@333b06a01db1874ffedb2de1bfabb195640927c7' + - 'metadatastician/standards@1725bebd3773ea25627479f813283abf1c497a85' '.github/workflows/inbox-steward-intake.yml': - 'actions/checkout@v7.0.1' - 'step-security/harden-runner@v2.22.3' @@ -68,7 +68,7 @@ workflows: - 'actions/upload-artifact@v7.0.2' - 'erlef/setup-beam@v1.24.1' '.github/workflows/mirror.yml': - - 'metadatastician/standards@333b06a01db1874ffedb2de1bfabb195640927c7' + - 'metadatastician/standards@1725bebd3773ea25627479f813283abf1c497a85' '.github/workflows/pages.yml': - 'actions/checkout@v7.0.1' - 'actions/deploy-pages@v5.0.1' @@ -108,9 +108,10 @@ workflows: - 'dtolnay/rust-toolchain@v1' - 'swatinem/rust-cache@v2.9.3' '.github/workflows/scorecard.yml': - - 'metadatastician/standards@333b06a01db1874ffedb2de1bfabb195640927c7' + - 'metadatastician/standards@1725bebd3773ea25627479f813283abf1c497a85' '.github/workflows/secret-scanner.yml': - - 'metadatastician/standards@333b06a01db1874ffedb2de1bfabb195640927c7' + - 'metadatastician/standards@1725bebd3773ea25627479f813283abf1c497a85' + '.github/workflows/standards-integration.yml': [] '.github/workflows/security-policy.yml': - 'actions/cache@v6.1.0' - 'actions/checkout@v7.0.1' @@ -199,6 +200,11 @@ dependencies: commit: 'sha1-368f82528645a54fb793d4d04e342629a3f51346' owner_id: 44036562 repo_id: 438112499 + 'actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333': + ref: 'v8.0.2' + commit: 'sha1-9000827ccba6bdab643e8b6fd33ac0654aef8333' + owner_id: 44036562 + repo_id: 192626254 'actions/download-artifact@v8.0.2': ref: 'v8.0.2' commit: 'sha1-9000827ccba6bdab643e8b6fd33ac0654aef8333' @@ -367,14 +373,15 @@ dependencies: commit: 'sha1-e384daf6167a7a7567642d6a62060106244d2413' owner_id: 6759885 repo_id: 1352485172 - 'metadatastician/standards@333b06a01db1874ffedb2de1bfabb195640927c7': - ref: '333b06a01db1874ffedb2de1bfabb195640927c7' - commit: 'sha1-333b06a01db1874ffedb2de1bfabb195640927c7' + 'metadatastician/standards@1725bebd3773ea25627479f813283abf1c497a85': + ref: '1725bebd3773ea25627479f813283abf1c497a85' + commit: 'sha1-1725bebd3773ea25627479f813283abf1c497a85' owner_id: 297579217 repo_id: 1116521501 uses: - 'actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9' - 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1' + - 'actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333' - 'actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9' - 'denoland/setup-deno@22d081ff2d3a40755e97629de92e3bcbfa7cf2ed' - 'dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8' diff --git a/.github/workflows/governance.yml b/.github/workflows/governance.yml index 370e8467..16897142 100644 --- a/.github/workflows/governance.yml +++ b/.github/workflows/governance.yml @@ -16,7 +16,17 @@ permissions: jobs: governance: - uses: metadatastician/standards/.github/workflows/governance-reusable.yml@333b06a01db1874ffedb2de1bfabb195640927c7 + # Standards pin (immutable, do not float): 1725beb… is the first + # standards revision whose actions allowlist contains + # `metadatastician/standards/.github/workflows/*@*` and whose mirror + # reusable gates SSH-host verification on the key being configured. + # The previous pin (333b06a…) lacked that pattern, so Allowlist + # Preflight failed on the five canon reusables themselves — a + # pinned-policy mismatch (category 3), not five genuine findings. + # Bump all five standards callers + actions.lock together; see + # docs/consumers/standards-governance.adoc and + # test/standards-integration-test.sh. + uses: metadatastician/standards/.github/workflows/governance-reusable.yml@1725bebd3773ea25627479f813283abf1c497a85 # Scanner pin: hypatia scans itself with its own main branch. On a PR that # is the base commit; on push, schedule or dispatch it is the commit being # run. It is never the PR head, so a PR cannot weaken the rules that gate diff --git a/.github/workflows/hypatia-scan.yml b/.github/workflows/hypatia-scan.yml index 1a0dfb9a..ec4aa4e2 100644 --- a/.github/workflows/hypatia-scan.yml +++ b/.github/workflows/hypatia-scan.yml @@ -21,7 +21,7 @@ permissions: jobs: hypatia: - uses: metadatastician/standards/.github/workflows/hypatia-scan-reusable.yml@333b06a01db1874ffedb2de1bfabb195640927c7 + uses: metadatastician/standards/.github/workflows/hypatia-scan-reusable.yml@1725bebd3773ea25627479f813283abf1c497a85 # Scanner pin: hypatia's own main branch, never the PR head. See governance.yml. with: hypatia-ref: ${{ github.event.pull_request.base.sha || github.sha }} diff --git a/.github/workflows/mirror.yml b/.github/workflows/mirror.yml index f1e0d60b..5d62db1b 100644 --- a/.github/workflows/mirror.yml +++ b/.github/workflows/mirror.yml @@ -41,7 +41,7 @@ permissions: jobs: mirror: - uses: metadatastician/standards/.github/workflows/mirror-reusable.yml@333b06a01db1874ffedb2de1bfabb195640927c7 + uses: metadatastician/standards/.github/workflows/mirror-reusable.yml@1725bebd3773ea25627479f813283abf1c497a85 # Only the seven forge keys the callee declares (standards#1108 §6b). secrets: GITLAB_SSH_KEY: ${{ secrets.GITLAB_SSH_KEY }} diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index e5c4b41e..a9e4726a 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -16,7 +16,7 @@ permissions: jobs: scorecard: - uses: metadatastician/standards/.github/workflows/scorecard-reusable.yml@333b06a01db1874ffedb2de1bfabb195640927c7 + uses: metadatastician/standards/.github/workflows/scorecard-reusable.yml@1725bebd3773ea25627479f813283abf1c497a85 permissions: actions: read contents: read diff --git a/.github/workflows/secret-scanner.yml b/.github/workflows/secret-scanner.yml index 7ad62ab4..ef8f5b7b 100644 --- a/.github/workflows/secret-scanner.yml +++ b/.github/workflows/secret-scanner.yml @@ -22,4 +22,4 @@ jobs: contents: read pull-requests: write actions: read - uses: metadatastician/standards/.github/workflows/secret-scanner-reusable.yml@333b06a01db1874ffedb2de1bfabb195640927c7 + uses: metadatastician/standards/.github/workflows/secret-scanner-reusable.yml@1725bebd3773ea25627479f813283abf1c497a85 diff --git a/.github/workflows/standards-integration.yml b/.github/workflows/standards-integration.yml new file mode 100644 index 00000000..80cb7d1c --- /dev/null +++ b/.github/workflows/standards-integration.yml @@ -0,0 +1,69 @@ +# This workflow is managed by gh actions-lock. +# SPDX-License-Identifier: MPL-2.0 +# Standards-integration contract gate, with zero third-party actions +# (same pattern as cicd-squabbler.yml): reusable/action resolution is +# itself one of the failure modes under test, so this gate must be able +# to run — and to fail honestly — when that resolution is broken. +# Exit semantics: 0 = contract holds; 1 = config failure (category 3); +# 3 = required tool unavailable. 3 is a failure of this gate, never a +# pass: skipped or unavailable work is not successful analysis. +name: Standards Integration + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +concurrency: + group: standards-integration-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + contract: + name: Standards integration contract + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Check out exact event commit without external action + env: + REPO: ${{ github.repository }} + SHA: ${{ github.event.pull_request.head.sha || github.sha }} + TOKEN: ${{ github.token }} + run: | + set -euo pipefail + AUTH="AUTHORIZATION: basic $(printf 'x-access-token:%s' "$TOKEN" | base64 -w0)" + git init -q . + git remote add origin "https://github.com/$REPO.git" + git -c http.extraheader="$AUTH" fetch -q --depth 1 origin "$SHA" + git checkout -q FETCH_HEAD + + - name: Run integration contract tests (positive, negative, unavailable-tool) + env: + GH_TOKEN: ${{ github.token }} + run: | + set -uo pipefail + rc=0 + bash test/standards-integration-test.sh || rc=$? + if [ "$rc" -eq 3 ]; then + echo "::error::Standards-integration tooling unavailable (exit 3) — not a pass" + exit 1 + fi + if [ "$rc" -ne 0 ]; then + echo "::error::Standards-integration contract failed (exit $rc, category=config)" + exit 1 + fi + rc=0 + bash scripts/standards-integration/check-standards-integration.sh --root . --verify-remote || rc=$? + if [ "$rc" -eq 3 ]; then + echo "::error::Published pin could not be verified (exit 3) — not a pass" + exit 1 + fi + if [ "$rc" -ne 0 ]; then + echo "::error::Standards integration contract failed remote verification (exit $rc)" + exit 1 + fi + echo "outcome=pass category=findings-channel-ready (contract + published pin verified; not a scan result)" >> "$GITHUB_STEP_SUMMARY" diff --git a/README.adoc b/README.adoc index e1f90987..fbd00f90 100644 --- a/README.adoc +++ b/README.adoc @@ -185,6 +185,7 @@ by any dispatch path, and the CI blockage currently stopping the Rust lane. * link:docs/DEBT-REGISTER.adoc[Debt register] — what is broken and what is not gated * link:docs/governance/ACTIONS-PINNING.adoc[Actions pinning decision] — how `uses:` refs are pinned here (actions.lock, not inline SHAs) * link:docs/consumers/rsr-oracle.adoc[Consuming the RSR oracle] — the SHA-pinned reusable workflow and the explicit-SSOT contract +* link:docs/consumers/standards-governance.adoc[Consuming the standards governance integration] — the immutable standards pin, failure-category taxonomy, runtime prerequisites and bump handoff for the five shared reusables * https://github.com/hyperpolymath/hypatia/wiki[Wiki] — getting started, architecture, operations, FAQ * https://github.com/hyperpolymath/standards/blob/main/PALIMPSEST.adoc[Palimpsest Philosophy] — philosophical underpinnings (lives in hyperpolymath/standards) diff --git a/docs/consumers/standards-governance.adoc b/docs/consumers/standards-governance.adoc new file mode 100644 index 00000000..ced0f63c --- /dev/null +++ b/docs/consumers/standards-governance.adoc @@ -0,0 +1,264 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 +// Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) += Consuming the standards shared-governance integration +:toc: left +:toclevels: 3 + +This is the consumer contract for the five reusable workflows this +repository takes from `metadatastician/standards`, and the evidence that +fixed the pin. It exists because estate CI sampling kept finding +Hypatia-related workflows red with zero jobs — and a zero-job run is +not a Hypatia finding, not a pass, and not one defect. + +== The four failure categories (classify before you fix) + +[cols="1,2,2,2",options="header"] +|=== +| # | Category | Signature | Correct response + +| 1 +| Admission / source resolution +| `startup_failure`, zero jobs, ~0s. GitHub never started a job: a + `uses:` ref was not allowed, not found, or the actions lockfile + disagreed with the workflow. +| Fix the pin, the allowlist, or `actions.lock`. Never touch scanner + rules; no analysis happened. + +| 2 +| Runner setup / prerequisites +| Jobs start; a setup step fails before the tool runs (invalid + expression, missing secret/var consumed before its skip guard, + lockfile or dependency fetch failure). +| Name the missing prerequisite, gate setup on it, and report the + forge/tool as *skipped — not configured*, never as analysed. + +| 3 +| Tool / configuration +| The tool starts but its policy/config is stale or mismatched (e.g. an + allowlist fetched at an old pinned revision that does not cover the + reusables actually called). +| Bump the pinned policy source atomically (all callers + lock). + Do not weaken the check to match stale policy. + +| 4 +| Actual analysis / security findings +| The scanner binary demonstrably ran, produced a validated findings + artefact (non-empty, schema-checked JSON/SARIF), and reported items. +| Triage the findings. These are the only reds that may be called + findings, and the only greens that may be called clean. +|=== + +Historical population observations (many estate `startup_failure`s) are +evidence for category 1 *as a class*; they never prove a specific +Hypatia defect. Start from current runs, per repository. + +== Evidence on this repository, 2026-10-31 + +[cols="1,1,3",options="header"] +|=== +| Run / workflow | Category | Evidence + +| Governance `41196057086` +| 1 +| `startup_failure`, 0s, zero jobs on an intermediate branch revision + while the Governance pin was being changed. No job, no analysis. + +| Governance `41196138008`, `41196238803` — *Allowlist Preflight* +| 3 +| Jobs ran; only Allowlist Preflight failed. Callers pinned standards + `333b06a…`, whose `actions-allowlist.json` (both + `config/settings/` and the rhodium copy) contains *no* + `metadatastician/standards/…` pattern, so the five canon reusables + the caller itself uses were reported as allowlist gaps. Standards + `1725beb…` adds exactly + `metadatastician/standards/.github/workflows/*@*`. A pinned-policy + mismatch, not five genuine findings — and not a reason to disable + the gate: at the new pin the gate still rejects uncovered actions. + +| Mirror to Git Forges `41196238787` +| 2 +| At `333b06a…`, `Verify SSH host` ran *before* the + not-configured skip, so forges whose `*_SSH_KEY` secret is unset + failed on SSH host verification and only then printed "Skipped (… + not configured)". At `1725beb…` verification is gated on the key + being present; unconfigured forges print an explicit skip notice. + Owner-side remainder: register the deploy keys, or leave + `vars._MIRROR_ENABLED` unset — the run then says *skipped*, + which is legible and is not a mirror. + +| Hypatia Security Scan `41196238773` (and predecessors) +| 4 — channel working +| One job, *Hypatia Neurosymbolic Analysis*, succeeded. That is the + only shape of green that means "the scanner ran". Even then, green + means the reusable's own gating rules passed — read its verdict + grid / findings artefact before calling a repository clean. +|=== + +Remaining gaps after the bump, individually explained (Governance run +`41198944467` on PR #992, reproduced offline with the pinned gate): +the preflight now reports exactly two gaps — +`google/clusterfuzzlite/actions/build_fuzzers@v1` and +`google/clusterfuzzlite/actions/run_fuzzers@v1`. Root cause is again +category 3, upstream: the checker at `1725beb…` matches the complete +action path, the tree canon +(`rhodium-standard-repositories/actions-allowlist/allowed-actions.json`) +carries only `google/clusterfuzzlite@*`, while the live-policy list +(`config/settings/actions-allowlist.json`) already grants the two +narrow subpath patterns — the two allowlist copies have drifted. +Hypatia's `clusterfuzzlite.yml` uses the official actions, gated on +fuzz targets existing; disabling fuzzing to satisfy the gate would be +the wrong fix. Filed as `metadatastician/standards#1380`; when a +standards revision carrying the two narrow patterns is published, +bump the pin per the checklist below and re-run. Until then +Governance is *not* genuinely clean and must not be reported as such. + +Note also PR #989's body claimed the Governance pin had moved to +`1725beb…`; the merged tree still pinned `333b06a…` in all five +callers and in `actions.lock`. This change makes the tree match the +claim and adds a contract test so the claim is checked, not trusted. + +== The published integration (what consumers copy) + +`metadatastician/standards` at commit +`1725bebd3773ea25627479f813283abf1c497a85` (standards `main` at the +time of writing; the commit, not the branch, is the contract). Its +reusables carry their own contract tests upstream +(`scripts/tests/governance-reusable-contract-test.sh`, +`scripts/tests/hypatia-scanner-refs-test.sh`), resolve Hypatia to a +vetted default SHA (`adb2f41…`) unless a caller canaries `HEAD`, and +stage their policy helpers from `job.workflow_sha` — the exact commit +the caller pinned. + +Hypatia-specific example (other consumers should omit `hypatia-ref` and use +the reusable's vetted default Hypatia revision; the expression below resolves +in this repository, not in arbitrary consumer repositories). All five +Hypatia callers move together. Caller-specific permissions are checked in +`.github/workflows/{governance,hypatia-scan,mirror,scorecard,secret-scanner}.yml`; +the examples below are not a universal permission block. + +[source,yaml] +---- +jobs: + governance: + uses: metadatastician/standards/.github/workflows/governance-reusable.yml@1725bebd3773ea25627479f813283abf1c497a85 + with: + # Scanner source: base commit on PRs, event commit otherwise. + # Never the PR head — a PR must not weaken the rules gating it. + hypatia-ref: ${{ github.event.pull_request.base.sha || github.sha }} + permissions: + actions: read + contents: read + security-events: write + hypatia: + uses: metadatastician/standards/.github/workflows/hypatia-scan-reusable.yml@1725bebd3773ea25627479f813283abf1c497a85 + with: + hypatia-ref: ${{ github.event.pull_request.base.sha || github.sha }} + secrets: # only the declared secret; never `secrets: inherit` + HYPATIA_SCAN_PAT: ${{ secrets.HYPATIA_SCAN_PAT }} +---- + +These two snippets do not imply shared permissions across the five callers. +The checked-in permission contracts differ intentionally: + +[cols="1,2",options="header"] +|=== +| Caller | Reusable-job permissions + +| `governance.yml` +| `actions: read`, `contents: read`, `security-events: write` + +| `hypatia-scan.yml` +| `actions: read`, `contents: read`, `security-events: write` + +| `mirror.yml` +| `actions: read`, `contents: read`, `security-events: write` (as declared on the reusable job) + +| `scorecard.yml` +| `actions: read`, `contents: read`, `security-events: write`, `id-token: write` + +| `secret-scanner.yml` +| `security-events: write`, `contents: read`, `pull-requests: write`, `actions: read` +|=== + +Use the actual caller workflow as the authority when adopting only one lane; +do not copy a permission set from another caller. `hypatia-scan.yml` passes +only the declared optional `HYPATIA_SCAN_PAT`; `mirror.yml` passes only its +seven per-forge keys. The other three declare no secrets. + +Then regenerate `.github/workflows/actions.lock` with +`gh actions-lock` in the same PR (never hand-edit it) and run: + +[source,bash] +---- +bash test/standards-integration-test.sh # offline contract +bash scripts/standards-integration/check-standards-integration.sh \ + --root . --verify-remote # + published-commit proof +---- + +== Runtime prerequisites (explicit) + +* Runner: `ubuntu-latest` only. The reusables install their own + toolchain (Elixir 1.19.4 / OTP 28.3 via `erlef/setup-beam`) — no + consumer-side Elixir, Rust or Zig setup is required for the scan or + governance lanes. +* Repository Actions policy must allow + `metadatastician/standards/.github/workflows/*` (selected-actions + allowlist). Without it every reusable is a category-1 + `startup_failure`; that is an owner/administration setting, not a + workflow bug. +* `HYPATIA_SCAN_PAT` (fine-grained, read-only): optional. Without it + the scan falls back to `GITHUB_TOKEN` (Dependabot alerts then cannot + be read) and the live Actions-policy job reports itself *not + evaluated*. Absent credential = advisory rows say so; nothing may + report that surface as passed. +* Mirror lane only: per-forge `vars._MIRROR_ENABLED=true`, + `secrets._SSH_KEY`, and (for self-hosted Gitea) `vars.GITEA_HOST` + plus the pinned SSH fingerprint vars. Any missing piece = that forge + is *skipped*, stated in the run — not mirrored, not failed open. +* Permissions: exactly those in the snippet. `secrets: inherit` is + refused by the contract checker. + +== Severity / outcome semantics + +[cols="1,1,2",options="header"] +|=== +| Outcome | Meaning | May be reported as + +| `outcome=pass` from the contract checker (exit 0) +| Pins, lock and scanner-source contract hold. +| "Integration channel ready." Never "scan clean" — no scan ran. + +| `outcome=fail, category=config` (exit 1) +| Floating/divergent pin, lock drift, PR-head scanner, `secrets: inherit`. +| A category-3 configuration failure. Fix the integration; do not + triage as code findings and do not disable Hypatia to silence it. + +| `outcome=unavailable, category=tool-unavailable` (exit 3) +| A required tool (`gh` for remote proof, or core text tools) is absent. +| "Not verified." In CI this gate fails closed; locally, re-run with + the tool installed. Absent tools, skipped jobs and `startup_failure` + are never successful analysis. + +| Scanner verdict (from the reusable's grid/findings artefact) +| The only source of genuine findings (category 4) and of "clean". +| Clean only when the scan job ran, the findings artefact validated, + and the gating rows are green. Notify the repository owner + (`@hyperpolymath`) at that point — and only at that point. +|=== + +== Bumping the pin (handoff checklist) + +. Pick a standards commit that is published on `metadatastician/standards` + and whose upstream contract tests pass; record why in the PR. +. Change all five callers (`governance`, `hypatia-scan`, `mirror`, + `scorecard`, `secret-scanner`) to that one 40-hex SHA. +. Regenerate `actions.lock` with `gh actions-lock` in the same PR. +. Run `bash test/standards-integration-test.sh` and the + `--verify-remote` check; paste both outputs in the PR. +. Confirm on the PR: Governance jobs actually start (not + `startup_failure`), Allowlist Preflight passes or every remaining + gap is individually explained, Hypatia Security Scan's analysis job + ran, and Mirror prints explicit per-forge skip/mirror lines. +. Only when those checks are genuinely clean, notify `@hyperpolymath` + in the PR. A merged PR, a skipped job or a zero-job run is not + notification-worthy evidence. diff --git a/scripts/standards-integration/check-standards-integration.sh b/scripts/standards-integration/check-standards-integration.sh new file mode 100755 index 00000000..cc445b4d --- /dev/null +++ b/scripts/standards-integration/check-standards-integration.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# check-standards-integration.sh — verify this repo's shared-governance +# integration with metadatastician/standards is published, immutable and +# internally consistent. Offline by default; --verify-remote additionally +# asks GitHub (via gh) that the pinned standards commit exists. +# +# Outcome semantics (never collapse these into "green"): +# exit 0 outcome=pass category=findings-channel-ready +# The integration contract holds. This says NOTHING about +# whether a Hypatia scan found anything; it only proves the +# channel through which findings would arrive is sound. +# exit 1 outcome=fail category=config +# A real contract violation (floating/divergent pin, lock +# drift, dangling transitive lock record, floating scanner +# ref, secrets: inherit). Treat as a category-3 +# tool/configuration failure, not a code finding. +# exit 3 outcome=unavailable category=tool-unavailable +# A required tool (or the remote check) could not run. This +# is NOT a pass and NOT a finding; report it as unavailable. +# Usage: check-standards-integration.sh [--root DIR] [--verify-remote] +set -uo pipefail + +ROOT="." +VERIFY_REMOTE=0 +while [ $# -gt 0 ]; do + case "$1" in + --root) ROOT="${2:?--root needs a directory}"; shift 2 ;; + --verify-remote) VERIFY_REMOTE=1; shift ;; + *) echo "outcome=unavailable category=tool-unavailable severity=error detail=unknown argument: $1" >&2; exit 3 ;; + esac +done + +# Report a configuration failure with the supplied detail on stdout and exit 1. +fail() { echo "outcome=fail category=config severity=high detail=$1"; exit 1; } + +# Report an unavailable check with the supplied detail on stdout and exit 3. +unavailable() { echo "outcome=unavailable category=tool-unavailable severity=error detail=$1"; exit 3; } + +# Every external command invoked below is preflighted, so missing +# tooling is outcome=unavailable (exit 3), never misreported as a +# repository configuration defect (review feedback, PR #992). +for tool in grep sed awk sort head; do + command -v "$tool" >/dev/null 2>&1 || unavailable "required tool '$tool' not on PATH" +done + +WF="$ROOT/.github/workflows" +LOCK="$WF/actions.lock" +CALLERS=(governance.yml hypatia-scan.yml mirror.yml scorecard.yml secret-scanner.yml) + +# Print matching standards `uses:` lines from the supplied workflow file, +# stripping inline comments first. No matches produce no output; pipeline +# failures are suppressed in the exit status, though diagnostics may reach stderr. +standards_uses() { + sed -E 's/[[:space:]]+#.*$//' "$1" \ + | grep -E '^[[:space:]]*uses:[[:space:]]*metadatastician/standards/\.github/workflows/' || true +} + +for f in "${CALLERS[@]}"; do + [ -f "$WF/$f" ] || fail "missing standards caller workflow: $f" +done +[ -f "$LOCK" ] || fail "missing .github/workflows/actions.lock" + +# --- Admission/source resolution: every standards reusable must resolve -- +# to exactly one immutable 40-hex commit across all callers. A branch, +# tag-only or divergent pin is what produced zero-job startup_failure and +# stale-policy failures estate-wide; it is rejected here, before CI. +# Inline comments are stripped and EVERY matching uses: entry is +# validated and retained — a second reusable job in one caller cannot +# hide behind the first one's pin. +pins=() +for f in "${CALLERS[@]}"; do + uses_lines=$(standards_uses "$WF/$f") + [ -n "$uses_lines" ] || fail "$f declares no metadatastician/standards reusable" + while IFS= read -r line; do + ref=$(printf '%s' "$line" | sed -E 's/.*uses:[[:space:]]*//; s/[[:space:]].*$//') + if ! printf '%s' "$ref" | grep -qE '^metadatastician/standards/\.github/workflows/[A-Za-z0-9._-]+\.yml@[0-9a-f]{40}$'; then + fail "$f has a non-immutable standards ref (must be a full 40-hex SHA): $ref" + fi + pins+=("${ref##*@}") + done </dev/null 2>&1 \ + || unavailable "gh not on PATH; remote pin verification not run (offline checks above are not a substitute)" + resolved=$(gh api "repos/metadatastician/standards/commits/$PIN" --jq .sha 2>/dev/null) \ + || unavailable "gh could not resolve metadatastician/standards@$PIN (network/auth); pin not remotely verified" + [ "$resolved" = "$PIN" ] || fail "remote standards commit mismatch: pinned $PIN, resolved $resolved" + echo "remote=verified repo=metadatastician/standards commit=$PIN" +fi + +echo "outcome=pass category=findings-channel-ready severity=none standards_pin=$PIN callers=5 detail=immutable pin, lock closure and scanner-source contract hold; this is not a scan result" +exit 0 diff --git a/test/standards-integration-test.sh b/test/standards-integration-test.sh new file mode 100755 index 00000000..4d914573 --- /dev/null +++ b/test/standards-integration-test.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# Repo-specific tests for the metadatastician/standards shared-governance +# integration. Positive (real tree passes), negative (floating pin, lock +# drift, divergent pins, PR-head scanner ref are rejected) and +# unavailable-tool (missing gh for remote verification is exit 3 / +# outcome=unavailable — never reported as a pass). +set -euo pipefail + +ROOT=$(cd "$(dirname "$0")/.." && pwd) +CHECK="$ROOT/scripts/standards-integration/check-standards-integration.sh" +TMP=$(mktemp -d) +trap 'rm -rf "$TMP"' EXIT +chmod +x "$CHECK" + +expect_outcome() { # name expected_exit expected_outcome root extra... + local name="$1" want_rc="$2" want_outcome="$3"; shift 3 + local out rc=0 + out=$("$CHECK" "$@" 2>&1) || rc=$? + if [ "$rc" != "$want_rc" ] || ! printf '%s' "$out" | grep -q "outcome=$want_outcome"; then + echo "FAIL($name): want rc=$want_rc outcome=$want_outcome, got rc=$rc: $out" >&2 + exit 1 + fi + echo "ok($name): rc=$rc $out" +} + +fixture() { # fixture : minimal copy of the integration surface + local dir="$1" + mkdir -p "$dir/.github/workflows" + for f in governance.yml hypatia-scan.yml mirror.yml scorecard.yml secret-scanner.yml actions.lock; do + cp "$ROOT/.github/workflows/$f" "$dir/.github/workflows/$f" + done +} + +# Positive: the real tree satisfies the contract. +expect_outcome positive 0 pass --root "$ROOT" + +# Negative 1: floating @main standards ref (the historical HEAD-clone class). +fixture "$TMP/floating" +sed -i -E 's|(metadatastician/standards/\.github/workflows/[^@]+)@[0-9a-f]{40}|\1@main|' "$TMP/floating/.github/workflows/governance.yml" +expect_outcome floating-pin 1 fail --root "$TMP/floating" + +# Negative 2: lock drift (workflow pin and actions.lock disagree). +fixture "$TMP/lockdrift" +sed -i 's|metadatastician/standards@[0-9a-f]\{40\}|metadatastician/standards@0000000000000000000000000000000000000000|g' "$TMP/lockdrift/.github/workflows/actions.lock" +expect_outcome lock-drift 1 fail --root "$TMP/lockdrift" + +# Negative 3: divergent pins across callers. +fixture "$TMP/divergent" +sed -i 's|@[0-9a-f]\{40\}|@1111111111111111111111111111111111111111|' "$TMP/divergent/.github/workflows/scorecard.yml" +expect_outcome divergent-pins 1 fail --root "$TMP/divergent" + +# Negative 4: scanner would run the PR head's own rules. +fixture "$TMP/prhead" +sed -i 's@hypatia-ref:.*@hypatia-ref: ${{ github.event.pull_request.head.sha }}@' "$TMP/prhead/.github/workflows/hypatia-scan.yml" +expect_outcome pr-head-scanner 1 fail --root "$TMP/prhead" + +# Negative 5: a 40-hex string in a comment must not disguise a floating +# scanner value (github.sha on pull_request is the PR merge commit). +fixture "$TMP/comment-sha" +sed -i 's@hypatia-ref:.*@hypatia-ref: ${{ github.sha }} # 0000000000000000000000000000000000000000@' "$TMP/comment-sha/.github/workflows/hypatia-scan.yml" +expect_outcome comment-disguised-scanner 1 fail --root "$TMP/comment-sha" + +# Negative 6: a second standards reusable job in one caller, pinned to a +# different revision, must not hide behind the first job's pin. +fixture "$TMP/second-job" +cat >> "$TMP/second-job/.github/workflows/governance.yml" <<'YAML' + second: + uses: metadatastician/standards/.github/workflows/governance-reusable.yml@1111111111111111111111111111111111111111 +YAML +expect_outcome second-job-divergent 1 fail --root "$TMP/second-job" + +# Negative 7: dangling transitive lock record — a nested uses: with no +# exact top-level dependency entry (the PR #992 download-artifact case). +fixture "$TMP/dangling" +sed -i "s| - 'actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9'| - 'actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9'\n - 'acme/missing@2222222222222222222222222222222222222222'|" \ + "$TMP/dangling/.github/workflows/actions.lock" +expect_outcome dangling-transitive 1 fail --root "$TMP/dangling" + +# Positive 2: an inline comment after a valid pin is not part of the ref. +fixture "$TMP/inline-comment" +sed -i -E 's|(uses:.*@[0-9a-f]{40})$|\1 # pinned standards revision|' "$TMP/inline-comment/.github/workflows/scorecard.yml" +expect_outcome inline-comment-pin 0 pass --root "$TMP/inline-comment" + +# Unavailable tool: remote verification requested but gh absent from PATH. +# Build a PATH with only the checker's offline tools — no gh. The checker +# must say unavailable (exit 3), never pass. +mkdir -p "$TMP/nogh" +for t in bash grep sed awk sort head env; do + src=$(command -v "$t" || true); [ -n "$src" ] && ln -s "$src" "$TMP/nogh/$t" +done +rc=0 +out=$(PATH="$TMP/nogh" "$TMP/nogh/bash" "$CHECK" --root "$ROOT" --verify-remote 2>&1) || rc=$? +if [ "$rc" != "3" ] || ! printf '%s' "$out" | grep -q 'outcome=unavailable'; then + echo "FAIL(unavailable-tool): want rc=3 outcome=unavailable, got rc=$rc: $out" >&2 + exit 1 +fi +if printf '%s' "$out" | grep -q 'outcome=pass'; then + echo "FAIL(unavailable-tool): unavailable was reported as pass" >&2 + exit 1 +fi +echo "ok(unavailable-tool): rc=$rc $out" + +# Unavailable tool 2: an offline core tool (sort) missing must also be +# exit 3 / unavailable, not misreported as divergent pins (exit 1). +mkdir -p "$TMP/nosort" +for t in bash grep sed awk head env; do + src=$(command -v "$t" || true); [ -n "$src" ] && ln -s "$src" "$TMP/nosort/$t" +done +rc=0 +out=$(PATH="$TMP/nosort" "$TMP/nosort/bash" "$CHECK" --root "$ROOT" 2>&1) || rc=$? +if [ "$rc" != "3" ] || ! printf '%s' "$out" | grep -q 'outcome=unavailable'; then + echo "FAIL(unavailable-sort): want rc=3 outcome=unavailable, got rc=$rc: $out" >&2 + exit 1 +fi +echo "ok(unavailable-sort): rc=$rc $out" + +echo "PASS: standards-integration positive, negative and unavailable-tool cases"