diff --git a/.github/workflows/release-image.yml b/.github/workflows/release-image.yml index 5c08830..cc16a48 100644 --- a/.github/workflows/release-image.yml +++ b/.github/workflows/release-image.yml @@ -30,6 +30,7 @@ jobs: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} SYNOLOGY_RUNNER_BASE_DIR: /volume1/docker/github-runner-fleet COMPOSE_PROJECT_NAME: github-runner-fleet + RELEASE_LOG_DIR: /tmp/runner-release timeout-minutes: 45 steps: - uses: actions/checkout@v7 @@ -121,7 +122,7 @@ jobs: - name: guard main branch before any publish run: | - if [[ "${GITHUB_REF_NAME}" != "main" ]]; then + if [[ "${GITHUB_REF}" != "refs/heads/main" ]]; then echo "release-image may only publish from main" >&2 exit 1 fi @@ -130,38 +131,7 @@ jobs: id: release_state env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - set -euo pipefail - final_ref="${{ steps.release_meta.outputs.image_ref }}" - staging_ref="${{ steps.release_meta.outputs.image_repo }}:candidate-${{ steps.release_meta.outputs.image_tag }}-${GITHUB_SHA}" - final_digest="" - release_exists=false - - if final_digest="$(docker buildx imagetools inspect "${final_ref}" --format '{{.Manifest.Digest}}' 2>/tmp/final-image-inspect.log)"; then - test -n "${final_digest}" - publish_required=false - working_ref="${final_ref}" - echo "found existing immutable candidate ${final_ref}@${final_digest}; verification-only recovery mode" - else - publish_required=true - working_ref="${staging_ref}" - echo "final tag is absent; build will publish unique staging reference ${staging_ref}" - fi - - if gh release view "${{ steps.release_meta.outputs.release_tag }}" >/tmp/github-release.txt 2>/tmp/github-release-error.log; then - release_exists=true - fi - if [[ "${release_exists}" == "true" && -z "${final_digest}" ]]; then - echo "GitHub Release exists but ${final_ref} is missing; refusing registry mutation" >&2 - exit 1 - fi - - { - echo "publish_required=${publish_required}" - echo "working_ref=${working_ref}" - echo "final_digest=${final_digest}" - echo "release_exists=${release_exists}" - } >> "${GITHUB_OUTPUT}" + run: bash scripts/release/preflight.sh "${{ steps.release_meta.outputs.image_ref }}" "${{ steps.release_meta.outputs.release_tag }}" - name: Build and push unique staging reference if: ${{ steps.release_state.outputs.publish_required == 'true' }} @@ -190,23 +160,24 @@ jobs: - name: Sign image digest if: ${{ steps.release_state.outputs.publish_required == 'true' }} - run: cosign sign --yes "${{ steps.release_state.outputs.working_ref }}@${{ steps.image_digest.outputs.digest }}" + run: bash scripts/release/retry.sh cosign-sign-index cosign sign --yes "${{ steps.release_state.outputs.working_ref }}@${{ steps.image_digest.outputs.digest }}" - name: Sign per-platform image digests if: ${{ steps.release_state.outputs.publish_required == 'true' }} run: | + set -euo pipefail image_repo="${{ steps.release_meta.outputs.image_repo }}" subject="${{ steps.release_state.outputs.working_ref }}@${{ steps.image_digest.outputs.digest }}" docker buildx imagetools inspect "${subject}" --raw \ | jq -r '.manifests[].digest' \ | while IFS= read -r digest; do - cosign sign --yes "${image_repo}@${digest}" + bash scripts/release/retry.sh "cosign-sign-${digest#sha256:}" cosign sign --yes "${image_repo}@${digest}" done - name: Attach SBOM attestation if: ${{ steps.release_state.outputs.publish_required == 'true' }} run: | - cosign attest --yes \ + bash scripts/release/retry.sh cosign-sbom-attest cosign attest --yes \ --predicate /tmp/github-runner-fleet.spdx.json \ --type spdxjson \ "${{ steps.release_state.outputs.working_ref }}@${{ steps.image_digest.outputs.digest }}" @@ -226,34 +197,33 @@ jobs: timeout 5m cosign verify \ --certificate-identity-regexp "https://github.com/${{ github.repository }}/.github/workflows/release-image.yml@refs/heads/main" \ --certificate-oidc-issuer https://token.actions.githubusercontent.com \ - "${subject}" 2>&1 | tee /tmp/cosign-signature-verify.log + "${subject}" 2>&1 | tee "${RELEASE_LOG_DIR}/cosign-signature-verify.log" timeout 5m cosign verify-attestation \ --type spdxjson \ --certificate-identity-regexp "https://github.com/${{ github.repository }}/.github/workflows/release-image.yml@refs/heads/main" \ --certificate-oidc-issuer https://token.actions.githubusercontent.com \ - "${subject}" 2>&1 | tee /tmp/cosign-sbom-verify.log + "${subject}" 2>&1 | tee "${RELEASE_LOG_DIR}/cosign-sbom-verify.log" timeout 5m cosign verify-attestation \ --type slsaprovenance \ --certificate-identity-regexp "https://github.com/${{ github.repository }}/.github/workflows/release-image.yml@refs/heads/main" \ --certificate-oidc-issuer https://token.actions.githubusercontent.com \ - "${subject}" 2>&1 | tee /tmp/cosign-provenance-verify.log + "${subject}" 2>&1 | tee "${RELEASE_LOG_DIR}/cosign-provenance-verify.log" - - name: Preserve verification diagnostics - if: ${{ failure() }} - uses: actions/upload-artifact@v7 - with: - name: release-verification-diagnostics-${{ github.run_attempt }} - path: | - /tmp/cosign-*-verify.log - /tmp/final-image-inspect.log - /tmp/github-release*.log - /tmp/imagetools.txt - if-no-files-found: warn + - name: Validate immutable image runtimes before promotion + run: | + set -euo pipefail + bash scripts/release/validate-runtime.sh \ + "${{ steps.release_state.outputs.working_ref }}@${{ steps.image_digest.outputs.digest }}" \ + 2>&1 | tee "${RELEASE_LOG_DIR}/runtime-validation.log" - name: Promote verified digest to final tag if: ${{ steps.release_state.outputs.publish_required == 'true' }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RELEASE_REQUIRE_ABSENT: "true" run: | set -euo pipefail + bash scripts/release/preflight.sh "${{ steps.release_meta.outputs.image_ref }}" "${{ steps.release_meta.outputs.release_tag }}" subject="${{ steps.release_state.outputs.working_ref }}@${{ steps.image_digest.outputs.digest }}" docker buildx imagetools create \ --tag "${{ steps.release_meta.outputs.image_ref }}" \ @@ -274,24 +244,41 @@ jobs: sleep 10 done - - run: | - docker run --rm --platform linux/amd64 --entrypoint /bin/sh "${{ steps.release_meta.outputs.image_ref }}" -lc \ - 'test "$(cd /actions-runner && ./bin/Runner.Listener --version)" = "$(cat /.runner-version)" && command -v pgrep && pgrep --version | head -n 1 && docker --version && node --version && python3 --version && terraform version | head -n 1' - - - run: | - docker run --rm --platform linux/arm64 --entrypoint /bin/sh "${{ steps.release_meta.outputs.image_ref }}" -lc \ - 'test "$(cd /actions-runner && ./bin/Runner.Listener --version)" = "$(cat /.runner-version)" && command -v pgrep && pgrep --version | head -n 1 && docker --version && node --version && python3 --version && terraform version | head -n 1' - - if: ${{ inputs.publish_project_release }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - if gh release view "${{ steps.release_meta.outputs.release_tag }}" >/dev/null 2>&1; then + set -euo pipefail + if [[ "${{ steps.release_state.outputs.release_exists }}" == "true" ]]; then echo "release ${{ steps.release_meta.outputs.release_tag }} already exists; verification completed without mutation" exit 0 fi + if [[ "${{ steps.release_state.outputs.publish_required }}" != "true" && "${{ steps.release_state.outputs.source_verified }}" != "true" ]]; then + echo "refusing to create a GitHub Release for an existing image at this dispatch SHA" >&2 + exit 1 + fi + + if [[ "${{ steps.release_state.outputs.tag_exists }}" == "true" ]]; then + tag_identity="$(gh api "repos/${GITHUB_REPOSITORY}/git/ref/tags/${{ steps.release_meta.outputs.release_tag }}" --jq '.object | "\(.type) \(.sha)"')" + test "${tag_identity}" = "commit ${GITHUB_SHA}" + else + # Create atomically; never silently reuse an unrelated source tag. + gh api "repos/${GITHUB_REPOSITORY}/git/refs" --method POST \ + -f "ref=refs/tags/${{ steps.release_meta.outputs.release_tag }}" \ + -f "sha=${GITHUB_SHA}" >/dev/null + fi gh release create "${{ steps.release_meta.outputs.release_tag }}" \ - --target "${GITHUB_SHA}" \ + --verify-tag \ --title "${{ steps.release_meta.outputs.release_tag }}" \ --generate-notes + + - name: Preserve verification diagnostics + if: ${{ failure() }} + uses: actions/upload-artifact@v7 + with: + name: release-verification-diagnostics-${{ github.run_attempt }} + path: | + ${{ env.RELEASE_LOG_DIR }}/*.log + /tmp/imagetools.txt + if-no-files-found: warn diff --git a/.runner-version b/.runner-version index 8fd64a6..d74e351 100644 --- a/.runner-version +++ b/.runner-version @@ -1 +1 @@ -2.336.0 +2.337.0 diff --git a/README.md b/README.md index dd1fa3c..6d350a1 100644 --- a/README.md +++ b/README.md @@ -289,15 +289,17 @@ The release workflow: - enforces that `package.json` version matches `config/pools.yaml` image tag - validates `config/pools.yaml` - runs the local `pnpm smoke-test` contract on `linux/amd64` -- publishes the configured tag from `config/pools.yaml` -- verifies the pushed tag with `docker buildx imagetools inspect` +- requires confirmed absence of the final image, Git tag, and GitHub Release before a new publication +- pushes a unique `candidate----` reference +- resolves the candidate's immutable digest with `docker buildx imagetools inspect` - confirms both `linux/amd64` and `linux/arm64` are present - signs the pushed digest with keyless cosign and attaches SPDX SBOM plus SLSA provenance attestations -- verifies the signature and attestations before release creation +- retries transient signing/attestation failures with bounded attempts and preserves failure diagnostics +- verifies signatures, attestations, and both platform runtimes against the immutable candidate digest before promotion +- rechecks final-tag absence and promotes the verified digest to the configured version tag - retries `pnpm validate-image` until the GitHub Packages API sees the new tag -- runs post-publish toolchain checks for both `linux/amd64` and `linux/arm64` -- automatically creates the matching GitHub release tag `v` after successful publishes from `main` -- can still be dispatched manually from `main`; set `publish_project_release=true` to create the matching GitHub Release during a manual run +- creates the matching source Git tag `v` and GitHub Release after successful validation +- runs only on explicit dispatch from `refs/heads/main`; `publish_project_release=true` is the default Only point [config/pools.yaml](config/pools.yaml) at a tag that this workflow has already published and verified. @@ -310,7 +312,7 @@ cosign verify \ ghcr.io/omt-global/github-runner-fleet: ``` -To keep the repository release and GHCR image tag aligned, merge the version bump to `main`. The release workflow first checks whether the matching repo release already exists; if it does, the automatic run fails before publishing an image so an existing GHCR version tag is not replaced. For a new version, the workflow publishes and verifies the image, then creates the matching repo tag and GitHub Release. +To keep the repository release and GHCR image tag aligned, merge the version bump to `main`, then dispatch `Release Image`. Routine main pushes do not publish. Completed releases can be verified again without mutation. Recovery can create missing release metadata only when verified provenance binds the existing image to the exact dispatch commit and any existing Git tag matches that SHA. Follow the [release recovery procedure](docs/release-recovery.md) to complete a same-source retry or supersede an older candidate with a new version. ## Runtime Contract diff --git a/docs/release-recovery.md b/docs/release-recovery.md index 638ee04..1e6ee52 100644 --- a/docs/release-recovery.md +++ b/docs/release-recovery.md @@ -2,12 +2,16 @@ `Release Image` is manual and transactional. It no longer publishes on every push to `main`. -For a new version, first update the canonical version and image tag together, merge that change, and dispatch the workflow from `main`. The workflow publishes a unique `candidate--` reference, signs and verifies its immutable digest, and only then promotes that digest to the final version tag and creates the matching GitHub Release. +For a new version, first update the canonical version and image tag together, merge that change, and dispatch the workflow from `refs/heads/main`. The workflow publishes a unique `candidate----` reference. A retry never reuses an earlier candidate tag. It signs and verifies the immutable digest and checks that exact image's runtime on both `linux/amd64` and `linux/arm64` before promoting the digest to the final version tag and creating the matching GitHub Release. -For a partially published version such as `0.2.1`, dispatch the workflow without changing the version. If the final image tag already exists, the workflow enters verification-only recovery mode: it does not rebuild, replace, re-sign, or re-attest the digest. If the existing signature, SBOM attestation, and SLSA provenance verify, the workflow may safely create the missing GitHub Release. If verification fails, supersede the candidate with a new version; never replace the existing tag. +When the final image already exists, the workflow enters verification-only recovery mode: it does not rebuild, replace, re-sign, or re-attest the digest. It verifies the existing signature, SBOM attestation, SLSA provenance, and platform runtimes. A completed Git tag and GitHub Release are left unchanged. + +If a final image exists without its GitHub Release, automated recovery first verifies GitHub's source provenance for that exact digest. The provenance must match this repository, `release-image.yml`, `refs/heads/main`, and the current dispatch commit SHA. Any existing Git tag must also be a lightweight tag at that exact SHA. Only then may the workflow finish creating the missing release metadata. This permits a same-source retry after an image-only dispatch (`publish_project_release=false`) or a failed release creation without assigning an unrelated commit to the image. If source verification fails or `main` has advanced, supersede the candidate with a new version; never replace the existing tag. ## `0.2.1` and `0.2.2` recovery decision The repository cannot prove `0.2.1` completed the immutable release contract, so it is frozen as an unreleased candidate and must not be replaced. Version `0.2.2` was never published as a verified project release, so it must not be treated as a completed release either. Version `0.2.3` supersedes these candidates and is the next permitted release target. -If a GitHub Release exists but its image tag is missing, the workflow fails before any registry mutation. Verification commands are bounded to five minutes and their logs are retained as a diagnostic artifact on failure. +If a GitHub Release or Git tag exists but its image tag is missing, the workflow fails before any registry mutation. Authentication failures, registry outages, and unexpected lookup errors also stop publication; only confirmed manifest absence and GitHub HTTP 404 responses permit a new release. The workflow checks absence again immediately before promotion, and creates the source Git tag with a create-only API call so it cannot silently reuse an unrelated tag. + +Signing and SBOM attestation retry transient transport failures up to three times, with a two-minute limit per attempt. This covers the July 28, 2026 candidate's failed Rekor request without disabling transparency-log verification. Permanent signing errors stop immediately. Verification commands remain bounded to five minutes. Lookup, signing, attestation, verification, and runtime logs are retained as a diagnostic artifact on failure. diff --git a/package.json b/package.json index 447c2d2..298b27a 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "node": ">=20.0.0" }, "scripts": { - "build": "tsc -p tsconfig.json", + "build": "tsc -p tsconfig.json && node scripts/copy-build-metadata.mjs", "audit-log": "tsx src/cli.ts audit-log", "check-runner-version": "tsx src/cli.ts check-runner-version", "config-diff": "tsx src/cli.ts config-diff", diff --git a/scripts/copy-build-metadata.mjs b/scripts/copy-build-metadata.mjs new file mode 100644 index 0000000..c08b37f --- /dev/null +++ b/scripts/copy-build-metadata.mjs @@ -0,0 +1,6 @@ +import { copyFileSync } from "node:fs"; + +copyFileSync( + new URL("../.runner-version", import.meta.url), + new URL("../dist/.runner-version", import.meta.url) +); diff --git a/scripts/release/preflight.sh b/scripts/release/preflight.sh new file mode 100644 index 0000000..864cfef --- /dev/null +++ b/scripts/release/preflight.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +set -euo pipefail + +final_ref="${1:?usage: preflight.sh }" +release_tag="${2:?release tag is required}" +: "${GITHUB_REPOSITORY:?}" +: "${GITHUB_SHA:?}" +: "${GITHUB_RUN_ID:?}" +: "${GITHUB_RUN_ATTEMPT:?}" +: "${GITHUB_OUTPUT:?}" +log_dir="${RELEASE_LOG_DIR:-/tmp/runner-release}" +mkdir -p "${log_dir}" + +if [[ ! "${GITHUB_SHA}" =~ ^[0-9a-f]{40}$ || ! "${GITHUB_RUN_ID}" =~ ^[0-9]+$ || ! "${GITHUB_RUN_ATTEMPT}" =~ ^[0-9]+$ ]]; then + echo "invalid immutable release run identity" >&2 + exit 1 +fi +if [[ "${final_ref}" == *@* || "${final_ref}" != *:* || "${release_tag}" != "v${final_ref##*:}" ]]; then + echo "release tag must match the versioned image reference" >&2 + exit 1 +fi + +final_digest="" +if final_digest="$(docker buildx imagetools inspect "${final_ref}" --format '{{.Manifest.Digest}}' 2>"${log_dir}/final-image-inspect.log")"; then + if [[ ! "${final_digest}" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "registry returned an invalid digest; refusing registry mutation" >&2 + exit 1 + fi +elif [[ "$(cat "${log_dir}/final-image-inspect.log")" == "ERROR: ${final_ref}: not found" ]]; then + # This is Buildx's exact manifest-not-found response. Authentication failures, + # unavailable registries, and unfamiliar errors must never mean absence. + final_digest="" +else + cat "${log_dir}/final-image-inspect.log" >&2 + echo "cannot establish final image state; refusing registry mutation" >&2 + exit 1 +fi + +lookup_exists=false +lookup_github() { + local api_path="$1" + local log_name="$2" + local status_line + lookup_exists=false + if gh api --include "${api_path}" >"${log_dir}/${log_name}-response.log" 2>"${log_dir}/${log_name}-error.log"; then + lookup_exists=true + else + IFS= read -r status_line < "${log_dir}/${log_name}-response.log" || true + if [[ ! "${status_line}" =~ ^HTTP/[0-9.]+[[:space:]]404[[:space:]] ]]; then + cat "${log_dir}/${log_name}-error.log" >&2 + echo "cannot establish ${log_name} state; refusing registry mutation" >&2 + exit 1 + fi + fi +} + +lookup_github "repos/${GITHUB_REPOSITORY}/releases/tags/${release_tag}" github-release +release_exists="${lookup_exists}" +lookup_github "repos/${GITHUB_REPOSITORY}/git/ref/tags/${release_tag}" github-tag +tag_exists="${lookup_exists}" + +if [[ -z "${final_digest}" && ( "${release_exists}" == true || "${tag_exists}" == true ) ]]; then + echo "GitHub Release or Git tag exists but ${final_ref} is missing; refusing registry mutation" >&2 + exit 1 +fi +if [[ "${RELEASE_REQUIRE_ABSENT:-false}" == true && -n "${final_digest}" ]]; then + echo "final image appeared after preflight; refusing to replace it during promotion" >&2 + exit 1 +fi + +source_verified=false +if [[ -n "${final_digest}" && "${release_exists}" != true ]]; then + # Recovery may publish release metadata only when authenticated provenance + # binds this existing digest to the exact source commit of this dispatch. + if ! timeout 5m gh attestation verify "oci://${final_ref%:*}@${final_digest}" \ + --repo "${GITHUB_REPOSITORY}" \ + --signer-workflow "${GITHUB_REPOSITORY}/.github/workflows/release-image.yml" \ + --source-ref refs/heads/main \ + --source-digest "${GITHUB_SHA}" \ + >"${log_dir}/source-provenance-verify.log" 2>&1; then + cat "${log_dir}/source-provenance-verify.log" >&2 + echo "refusing to attach this dispatch SHA to an older image without matching verified source provenance" >&2 + echo "dispatch the original verified source commit or supersede the candidate with a new version; never replace the existing tag" >&2 + exit 1 + fi + if [[ "${tag_exists}" == true ]]; then + if ! tag_identity="$(gh api "repos/${GITHUB_REPOSITORY}/git/ref/tags/${release_tag}" --jq '.object | "\(.type) \(.sha)"' 2>"${log_dir}/github-tag-identity-error.log")" || [[ "${tag_identity}" != "commit ${GITHUB_SHA}" ]]; then + echo "existing release tag is not a lightweight tag at the verified source SHA; refusing release recovery" >&2 + exit 1 + fi + fi + source_verified=true +fi +if [[ -n "${final_digest}" && "${release_exists}" == true && "${tag_exists}" != true ]]; then + echo "GitHub Release has no matching Git tag; refusing inconsistent release recovery" >&2 + exit 1 +fi + +if [[ -n "${final_digest}" ]]; then + publish_required=false + working_ref="${final_ref}" + echo "found existing image ${final_ref}@${final_digest}; verification-only recovery mode" +else + publish_required=true + working_ref="${final_ref%:*}:candidate-${final_ref##*:}-${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + echo "confirmed final image and release are absent; staging reference ${working_ref}" +fi + +{ + echo "publish_required=${publish_required}" + echo "working_ref=${working_ref}" + echo "final_digest=${final_digest}" + echo "release_exists=${release_exists}" + echo "tag_exists=${tag_exists}" + echo "source_verified=${source_verified}" +} >> "${GITHUB_OUTPUT}" diff --git a/scripts/release/retry.sh b/scripts/release/retry.sh new file mode 100644 index 0000000..ced6613 --- /dev/null +++ b/scripts/release/retry.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euo pipefail + +log_name="${1:?usage: retry.sh [arguments...]}" +shift +if [[ ! "${log_name}" =~ ^[a-zA-Z0-9_-]+$ || $# -eq 0 ]]; then + echo "a safe log name and command are required" >&2 + exit 1 +fi +log_dir="${RELEASE_LOG_DIR:-/tmp/runner-release}" +mkdir -p "${log_dir}" + +for attempt in 1 2 3; do + attempt_log="${log_dir}/${log_name}-attempt-${attempt}.log" + echo "${log_name}: attempt ${attempt}/3 (two-minute limit)" + if timeout 2m "$@" >"${attempt_log}" 2>&1; then + cat "${attempt_log}" + exit 0 + else + status=$? + fi + cat "${attempt_log}" >&2 + echo "${log_name}: attempt ${attempt} failed with exit ${status}" >&2 + + # Retry the observed Rekor transport failure and other transient network/server + # failures. Invalid signatures, identities, and predicates fail immediately. + if [[ "${status}" -ne 124 ]] && ! grep -Eiq 'giving up after [0-9]+ attempt|connection reset|TLS handshake timeout|context deadline exceeded|i/o timeout|temporary failure|connection refused|service unavailable|unexpected status.*(429|50[0-9])' "${attempt_log}"; then + exit "${status}" + fi + if [[ "${attempt}" -eq 3 ]]; then + echo "${log_name}: retry limit reached; see preserved attempt logs" >&2 + exit "${status}" + fi + sleep "$((attempt * 5))" +done diff --git a/scripts/release/validate-runtime.sh b/scripts/release/validate-runtime.sh new file mode 100644 index 0000000..eea4be2 --- /dev/null +++ b/scripts/release/validate-runtime.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +subject="${1:?usage: validate-runtime.sh }" +if [[ ! "${subject}" =~ @sha256:[0-9a-f]{64}$ ]]; then + echo "runtime validation requires an immutable image digest" >&2 + exit 1 +fi + +for platform in linux/amd64 linux/arm64; do + echo "validating release runtime ${subject} (${platform})" + docker run --rm --platform "${platform}" --entrypoint /bin/bash "${subject}" -lc ' + set -euo pipefail + test "$(cd /actions-runner && ./bin/Runner.Listener --version)" = "$(cat /.runner-version)" + command -v pgrep + pgrep --version + docker --version + node --version + python3 --version + terraform version + ' +done diff --git a/test/audit.test.ts b/test/audit.test.ts index 9e07b55..e1630c4 100644 --- a/test/audit.test.ts +++ b/test/audit.test.ts @@ -146,10 +146,9 @@ describe("audit log", () => { const directory = createTempDir(); const filePath = path.join(directory, "audit.jsonl"); fs.writeFileSync(filePath, `${JSON.stringify({ old: "x".repeat(8_800) })}\n`, "utf8"); - await Promise.all(Array.from({ length: 8 }, (_value, index) => - spawnAuditWriter(filePath, index) - )); + await runAuditWriters(filePath, 8); + expect(fs.existsSync(`${filePath}.1`)).toBe(true); const records = [filePath, `${filePath}.1`] .filter((entry) => fs.existsSync(entry)) .flatMap((entry) => readJsonLines(entry)); @@ -277,24 +276,50 @@ function createTempDir(): string { return directory; } -function spawnAuditWriter(filePath: string, index: number): Promise { - return new Promise((resolve, reject) => { +async function runAuditWriters(filePath: string, count: number): Promise { + const writers = Array.from({ length: count }, (_value, index) => { const child = spawn(process.execPath, [ - "--import", "tsx", "src/cli.ts", "audit-log", - "--file", filePath, "--max-size-bytes", "10000" - ], { cwd: path.resolve("."), stdio: ["pipe", "ignore", "pipe"] }); + "--import", "tsx", "test/fixtures/audit-writer.ts", filePath, String(index) + ], { cwd: path.resolve("."), stdio: ["ignore", "ignore", "pipe", "ipc"] }); let error = ""; - child.stderr.on("data", (chunk) => { error += String(chunk); }); - child.on("error", reject); - child.on("exit", (code) => code === 0 ? resolve() : reject(new Error(`audit writer exited ${code}: ${error}`))); - child.stdin.end(JSON.stringify({ - event: "runner_job_start", - runner_name: `process-${index}`, - pool: "synology-private", - plane: "synology", - org: "omt-global" - })); + child.stderr!.on("data", (chunk) => { error += String(chunk); }); + child.on("error", (cause) => { error += cause.message; }); + const ready = new Promise((resolve, reject) => { + child.once("message", (message) => { + if (message === "ready") resolve(); + else reject(new Error(`unexpected audit writer message: ${String(message)}`)); + }); + child.once("close", () => reject(new Error(`audit writer ${index} closed before ready: ${error}`))); + }); + const closed = new Promise((resolve) => child.once("close", resolve)); + return { child, ready, closed, failure: () => new Error(`audit writer ${index} failed: ${error}`) }; }); + + let timer: ReturnType | undefined; + try { + await Promise.race([ + (async () => { + // Finish imports in every process before releasing the concurrent writes. + await Promise.all(writers.map((writer) => writer.ready)); + await Promise.all(writers.map(({ child }) => new Promise((resolve, reject) => { + child.send("write", (error) => error ? reject(error) : resolve()); + }))); + await Promise.all(writers.map(async (writer) => { + if (await writer.closed !== 0) throw writer.failure(); + })); + })(), + new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new Error("audit writers did not finish within 25000ms")), 25_000); + }) + ]); + } finally { + clearTimeout(timer); + for (const { child } of writers) { + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + } + // Reap failed or timed-out writers before afterEach removes their directory. + await Promise.all(writers.map((writer) => writer.closed)); + } } function readJsonLines(filePath: string): unknown[] { diff --git a/test/build-artifact.test.ts b/test/build-artifact.test.ts new file mode 100644 index 0000000..b30ae9a --- /dev/null +++ b/test/build-artifact.test.ts @@ -0,0 +1,50 @@ +import { execFileSync, execSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { expect, test } from "vitest"; +import { readCanonicalRunnerVersion } from "../src/lib/runner-version.js"; + +test("packages canonical runner metadata for the built reader", () => { + const repository = path.resolve("."); + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "runner-build-test-")); + try { + // Exercise the real build command with one source module, not the full app. + for (const relativePath of [ + "package.json", "tsconfig.json", ".runner-version", + "src/lib/runner-version.ts", "scripts/copy-build-metadata.mjs" + ]) { + const destination = path.join(directory, relativePath); + fs.mkdirSync(path.dirname(destination), { recursive: true }); + fs.copyFileSync(path.join(repository, relativePath), destination); + } + fs.symlinkSync( + path.join(repository, "node_modules"), + path.join(directory, "node_modules"), + process.platform === "win32" ? "junction" : "dir" + ); + const { scripts } = JSON.parse(fs.readFileSync(path.join(directory, "package.json"), "utf8")); + execSync(scripts.build, { + cwd: directory, + env: { + ...process.env, + PATH: [path.join(repository, "node_modules/.bin"), process.env.PATH].join(path.delimiter) + }, + timeout: 20_000, + stdio: "pipe" + }); + + expect(fs.readFileSync(path.join(directory, "dist/.runner-version"), "utf8")) + .toBe(fs.readFileSync(path.join(repository, ".runner-version"), "utf8")); + fs.rmSync(path.join(directory, ".runner-version")); + const builtModule = pathToFileURL(path.join(directory, "dist/src/lib/runner-version.js")); + const actual = execFileSync(process.execPath, [ + "--input-type=module", "--eval", + `import { readCanonicalRunnerVersion } from ${JSON.stringify(builtModule.href)}; console.log(readCanonicalRunnerVersion());` + ], { cwd: os.tmpdir(), encoding: "utf8", timeout: 5_000 }); + expect(actual.trim()).toBe(readCanonicalRunnerVersion()); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } +}); diff --git a/test/env.test.ts b/test/env.test.ts index 2f0b2e9..4f329de 100644 --- a/test/env.test.ts +++ b/test/env.test.ts @@ -55,7 +55,7 @@ describe("loadDeploymentEnv", () => { expect(env.githubApiUrl).toBe("https://api.github.com"); expect(env.composeProjectName).toBe("github-runner-fleet"); - expect(env.runnerVersion).toBe("2.336.0"); + expect(env.runnerVersion).toBe("2.337.0"); expect(env.githubPat).toBeUndefined(); expect(env.raw).toMatchObject({ GITHUB_API_URL: "https://api.github.com", @@ -82,7 +82,7 @@ describe("loadDeploymentEnv", () => { LUME_RUNNER_BASE_DIR: `${os.homedir()}/Library/Application Support/github-runner-fleet/lume`, LUME_RUNNER_ENV_FILE: `${os.homedir()}/Library/Application Support/github-runner-fleet/lume/runner.env`, COMPOSE_PROJECT_NAME: "github-runner-fleet", - RUNNER_VERSION: "2.336.0" + RUNNER_VERSION: "2.337.0" }); }); diff --git a/test/fixtures/audit-writer.ts b/test/fixtures/audit-writer.ts new file mode 100644 index 0000000..05eef67 --- /dev/null +++ b/test/fixtures/audit-writer.ts @@ -0,0 +1,17 @@ +import { writeAuditRecord } from "../../src/lib/audit.js"; + +const [filePath, index] = process.argv.slice(2); + +process.once("message", (message) => { + if (message !== "write") throw new Error("expected audit write command"); + writeAuditRecord({ + event: "runner_job_start", + runner_name: `process-${index}`, + pool: "synology-private", + plane: "synology", + org: "omt-global" + }, { filePath, maxSizeBytes: 10_000 }); + process.disconnect(); +}); + +process.send!("ready"); diff --git a/test/release-scripts.test.ts b/test/release-scripts.test.ts new file mode 100644 index 0000000..8ba5c1d --- /dev/null +++ b/test/release-scripts.test.ts @@ -0,0 +1,366 @@ +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import YAML from "yaml"; +import { afterEach, describe, expect, test } from "vitest"; + +const temporaryDirectories: string[] = []; +const imageRef = "ghcr.io/test/fleet:0.2.3"; +const digest = `sha256:${"a".repeat(64)}`; +const sourceSha = "b".repeat(40); + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + fs.rmSync(directory, { recursive: true, force: true }); + } +}); + +function fixture(overrides: NodeJS.ProcessEnv = {}) { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "release-script-test-")); + temporaryDirectories.push(directory); + const bin = path.join(directory, "bin"); + fs.mkdirSync(bin); + const callsFile = path.join(directory, "calls.jsonl"); + const outputFile = path.join(directory, "outputs"); + const logDirectory = path.join(directory, "logs"); + fs.writeFileSync(outputFile, ""); + const stub = `#!${process.execPath} +const fs = require("node:fs"); +const path = require("node:path"); +const command = path.basename(process.argv[1]); +const args = process.argv.slice(2); +fs.appendFileSync(process.env.TEST_CALLS, JSON.stringify({command, args}) + "\\n"); +const env = process.env; +if (command === "docker") { + if (args[0] === "run") { + process.exit(args.includes(env.TEST_RUNTIME_FAIL_PLATFORM) ? 23 : 0); + } + const state = env.TEST_IMAGE_STATE || "missing"; + if (state === "present" || state === "invalid") { + console.log(state === "present" ? env.TEST_DIGEST : "not-a-digest"); + process.exit(0); + } + console.error(state === "missing" ? "ERROR: " + args[3] + ": not found" : state); + process.exit(1); +} +if (command === "gh") { + if (args[0] === "attestation") { + console.log(env.TEST_PROVENANCE_VALID === "true" ? "verified source provenance" : "source provenance mismatch"); + process.exit(env.TEST_PROVENANCE_VALID === "true" ? 0 : 1); + } + if (args.includes("--jq")) { + console.log(env.TEST_TAG_IDENTITY || "commit " + env.GITHUB_SHA); + process.exit(0); + } + if (args.includes("POST")) process.exit(env.TEST_TAG_CREATE_CONFLICT === "true" ? 1 : 0); + if (args[0] === "release") process.exit(0); + const state = args[2].includes("/releases/") ? env.TEST_RELEASE_STATE : env.TEST_TAG_STATE; + if (state === "present") { + console.log("HTTP/2.0 200 OK\\r\\n\\r\\n{}"); + process.exit(0); + } + if (state !== "network") console.log("HTTP/2.0 " + (state || "404") + " Not Found\\r\\n\\r\\n{}"); + console.error(state === "network" ? "error connecting to api.github.com" : "GitHub lookup failed"); + process.exit(1); +} +if (command === "cosign") { + const countFile = env.TEST_ATTEMPTS; + const count = fs.existsSync(countFile) ? Number(fs.readFileSync(countFile, "utf8")) + 1 : 1; + fs.writeFileSync(countFile, String(count)); + if (count <= Number(env.TEST_SIGN_FAILURES || "0")) { + console.error(env.TEST_SIGN_ERROR || 'Post "https://rekor.sigstore.dev/api/v1/log/entries" giving up after 2 attempt(s)'); + process.exit(2); + } + console.log("signed immutable image"); +} +`; + for (const name of ["docker", "gh", "cosign"]) { + fs.writeFileSync(path.join(bin, name), stub, { mode: 0o755 }); + } + fs.writeFileSync(path.join(bin, "timeout"), `#!/bin/bash +printf '%s\\n' "$1" >> "$TEST_TIMEOUTS" +if [[ "$TEST_TIMEOUT" == true ]]; then exit 124; fi +shift +exec "$@" +`, { mode: 0o755 }); + fs.writeFileSync(path.join(bin, "sleep"), "#!/bin/bash\nexit 0\n", { mode: 0o755 }); + const env = { + ...process.env, + PATH: `${bin}:${process.env.PATH}`, + GITHUB_REPOSITORY: "test/fleet", + GITHUB_SHA: sourceSha, + GITHUB_RUN_ID: "1234", + GITHUB_RUN_ATTEMPT: "1", + GITHUB_OUTPUT: outputFile, + RELEASE_LOG_DIR: logDirectory, + TEST_DIGEST: digest, + TEST_CALLS: callsFile, + TEST_ATTEMPTS: path.join(directory, "attempts"), + TEST_TIMEOUTS: path.join(directory, "timeouts"), + ...overrides + }; + return { + directory, + logDirectory, + run: (script: string, args: string[] = []) => spawnSync("bash", [path.resolve(script), ...args], { + env, encoding: "utf8", timeout: 10_000 + }), + runShell: (script: string) => spawnSync("bash", ["-c", script], { + env, encoding: "utf8", timeout: 10_000 + }), + output: () => fs.readFileSync(outputFile, "utf8"), + calls: (): Array<{ command: string; args: string[] }> => fs.existsSync(callsFile) + ? fs.readFileSync(callsFile, "utf8").trim().split("\n").map((line) => JSON.parse(line)) + : [] + }; +} + +function preflight(context: ReturnType) { + return context.run("scripts/release/preflight.sh", [imageRef, "v0.2.3"]); +} + +describe("release preflight", () => { + test("permits a new candidate only after confirmed registry and GitHub absence", () => { + const context = fixture(); + expect(preflight(context).status).toBe(0); + expect(context.output()).toContain("publish_required=true"); + expect(context.output()).toContain(`working_ref=ghcr.io/test/fleet:candidate-0.2.3-${sourceSha}-1234-1`); + expect(context.calls().filter((call) => call.command === "gh").map((call) => call.args)).toEqual([ + ["api", "--include", "repos/test/fleet/releases/tags/v0.2.3"], + ["api", "--include", "repos/test/fleet/git/ref/tags/v0.2.3"] + ]); + }); + + test("uses distinct candidate references for retries and separate dispatches", () => { + const contexts = [fixture(), fixture({ GITHUB_RUN_ATTEMPT: "2" }), fixture({ GITHUB_RUN_ID: "5678" })]; + const references = contexts.map((context) => { + expect(preflight(context).status).toBe(0); + return context.output().split("\n").find((line) => line.startsWith("working_ref=")); + }); + expect(new Set(references).size).toBe(3); + }); + + test.each([ + "ERROR: failed to do request: connection refused", + "ERROR: unexpected status from HEAD request: 401 Unauthorized", + "ERROR: no builder found: not found", + `ERROR: ${imageRef}: not found\nERROR: registry unavailable`, + "invalid" + ])("fails closed on an uncertain registry result: %s", (state) => { + const context = fixture({ TEST_IMAGE_STATE: state }); + expect(preflight(context).status).not.toBe(0); + expect(context.output()).toBe(""); + expect(context.calls().every((call) => call.command === "docker")).toBe(true); + expect(fs.existsSync(path.join(context.logDirectory, "final-image-inspect.log"))).toBe(true); + }); + + test.each(["403", "500", "network"])("fails closed on a GitHub release lookup failure: %s", (state) => { + const context = fixture({ TEST_RELEASE_STATE: state }); + const result = preflight(context); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("cannot establish github-release state"); + expect(context.output()).toBe(""); + }); + + test("fails closed when source Git tag lookup fails", () => { + const context = fixture({ TEST_TAG_STATE: "403" }); + expect(preflight(context).stderr).toContain("cannot establish github-tag state"); + expect(context.output()).toBe(""); + }); + + test.each([ + { TEST_RELEASE_STATE: "present" }, + { TEST_TAG_STATE: "present" } + ])("rejects existing release or source tag without its image: %j", (state) => { + const context = fixture(state); + const result = preflight(context); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("GitHub Release or Git tag exists"); + expect(context.output()).toBe(""); + }); + + test.each([{}, { TEST_TAG_STATE: "present" }])("rejects orphan final-image recovery without verified source identity: %j", (state) => { + const context = fixture({ TEST_IMAGE_STATE: "present", ...state }); + const result = preflight(context); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("refusing to attach this dispatch SHA to an older image"); + expect(context.output()).toBe(""); + }); + + test.each(["404", "present"])("allows same-source image recovery with Git tag state %s", (tagState) => { + const context = fixture({ TEST_IMAGE_STATE: "present", TEST_TAG_STATE: tagState, TEST_PROVENANCE_VALID: "true" }); + expect(preflight(context).status).toBe(0); + expect(context.output()).toContain("publish_required=false"); + expect(context.output()).toContain("source_verified=true"); + expect(context.calls().find((call) => call.args[0] === "attestation")?.args).toEqual([ + "attestation", "verify", `oci://ghcr.io/test/fleet@${digest}`, + "--repo", "test/fleet", "--signer-workflow", "test/fleet/.github/workflows/release-image.yml", + "--source-ref", "refs/heads/main", "--source-digest", sourceSha + ]); + expect(fs.readFileSync(path.join(context.directory, "timeouts"), "utf8")).toBe("5m\n"); + }); + + test.each([`commit ${"c".repeat(40)}`, `tag ${sourceSha}`])("rejects a recovery tag with identity %s", (identity) => { + const context = fixture({ TEST_IMAGE_STATE: "present", TEST_TAG_STATE: "present", TEST_PROVENANCE_VALID: "true", TEST_TAG_IDENTITY: identity }); + expect(preflight(context).stderr).toContain("not a lightweight tag at the verified source SHA"); + expect(context.output()).toBe(""); + }); + + test("fails closed when source provenance verification times out", () => { + const context = fixture({ TEST_IMAGE_STATE: "present", TEST_TIMEOUT: "true" }); + expect(preflight(context).status).not.toBe(0); + expect(context.output()).toBe(""); + expect(fs.readFileSync(path.join(context.directory, "timeouts"), "utf8")).toBe("5m\n"); + }); + + test("rejects a GitHub Release whose source tag is missing", () => { + const context = fixture({ TEST_IMAGE_STATE: "present", TEST_RELEASE_STATE: "present" }); + expect(preflight(context).stderr).toContain("GitHub Release has no matching Git tag"); + expect(context.output()).toBe(""); + }); + + test("verifies an already completed release without rebuilding", () => { + const context = fixture({ TEST_IMAGE_STATE: "present", TEST_RELEASE_STATE: "present", TEST_TAG_STATE: "present" }); + expect(preflight(context).status).toBe(0); + expect(context.output()).toContain("publish_required=false"); + expect(context.output()).toContain(`final_digest=${digest}`); + expect(context.output()).toContain(`working_ref=${imageRef}`); + expect(context.calls().some((call) => call.args[0] === "attestation")).toBe(false); + }); + + test("stops promotion if a final image appeared after initial preflight", () => { + const context = fixture({ TEST_IMAGE_STATE: "present", TEST_RELEASE_STATE: "present", TEST_TAG_STATE: "present", RELEASE_REQUIRE_ABSENT: "true" }); + expect(preflight(context).stderr).toContain("final image appeared after preflight"); + expect(context.output()).toBe(""); + }); +}); + +describe("bounded signing retries", () => { + test("recovers from transient Rekor failures and preserves every attempt", () => { + const context = fixture({ TEST_SIGN_FAILURES: "2" }); + const result = context.run("scripts/release/retry.sh", ["sbom", "cosign", "attest", "--yes", `image@${digest}`]); + expect(result.status).toBe(0); + expect(context.calls()).toHaveLength(3); + expect(fs.readFileSync(path.join(context.logDirectory, "sbom-attempt-1.log"), "utf8")).toContain("rekor.sigstore.dev"); + expect(fs.readFileSync(path.join(context.logDirectory, "sbom-attempt-3.log"), "utf8")).toContain("signed immutable image"); + expect(fs.readFileSync(path.join(context.directory, "timeouts"), "utf8")).toBe("2m\n2m\n2m\n"); + }); + + test("returns persistent failure after three attempts", () => { + const context = fixture({ TEST_SIGN_FAILURES: "10" }); + const result = context.run("scripts/release/retry.sh", ["sbom", "cosign", "attest"]); + expect(result.status).toBe(2); + expect(result.stderr).toContain("retry limit reached"); + expect(context.calls()).toHaveLength(3); + }); + + test("does not retry permanent signing failures", () => { + const context = fixture({ TEST_SIGN_FAILURES: "10", TEST_SIGN_ERROR: "invalid predicate format" }); + expect(context.run("scripts/release/retry.sh", ["sbom", "cosign", "attest"]).status).toBe(2); + expect(context.calls()).toHaveLength(1); + }); + + test("bounds hung attempts and returns the timeout status", () => { + const context = fixture({ TEST_TIMEOUT: "true" }); + expect(context.run("scripts/release/retry.sh", ["sbom", "cosign", "attest"]).status).toBe(124); + expect(fs.readdirSync(context.logDirectory)).toHaveLength(3); + expect(fs.readFileSync(path.join(context.directory, "timeouts"), "utf8")).toBe("2m\n2m\n2m\n"); + }); +}); + +describe("immutable runtime validation", () => { + test("executes both platform checks against the same immutable digest", () => { + const context = fixture(); + const subject = `ghcr.io/test/fleet:candidate-123@${digest}`; + expect(context.run("scripts/release/validate-runtime.sh", [subject]).status).toBe(0); + const calls = context.calls(); + expect(calls).toHaveLength(2); + expect(calls.map((call) => call.args[3])).toEqual(["linux/amd64", "linux/arm64"]); + for (const call of calls) { + expect(call.args[6]).toBe(subject); + expect(call.args[8]).toContain("Runner.Listener --version"); + expect(call.args[8]).toContain("set -euo pipefail"); + } + }); + + test("refuses mutable tag validation", () => { + const context = fixture(); + expect(context.run("scripts/release/validate-runtime.sh", [imageRef]).status).not.toBe(0); + expect(context.calls()).toHaveLength(0); + }); + + test.each(["linux/amd64", "linux/arm64"])("propagates a runtime failure on %s", (platform) => { + const context = fixture({ TEST_RUNTIME_FAIL_PLATFORM: platform }); + expect(context.run("scripts/release/validate-runtime.sh", [`image@${digest}`]).status).toBe(23); + expect(context.calls()).toHaveLength(platform === "linux/amd64" ? 1 : 2); + }); +}); + +const workflow = YAML.parse(fs.readFileSync(path.resolve(".github/workflows/release-image.yml"), "utf8")); +const releaseStep = workflow.jobs.publish_and_verify.steps.find((step: { if?: string }) => step.if === "${{ inputs.publish_project_release }}"); + +function releaseScript(publishRequired: boolean, releaseExists: boolean, sourceVerified = false, tagExists = false): string { + return (releaseStep.run as string) + .replaceAll("${{ steps.release_state.outputs.publish_required }}", String(publishRequired)) + .replaceAll("${{ steps.release_state.outputs.release_exists }}", String(releaseExists)) + .replaceAll("${{ steps.release_state.outputs.source_verified }}", String(sourceVerified)) + .replaceAll("${{ steps.release_state.outputs.tag_exists }}", String(tagExists)) + .replaceAll("${{ steps.release_meta.outputs.release_tag }}", "v0.2.3"); +} + +describe("source-bound project release creation", () => { + test("creates the source ref at the dispatch SHA before publishing its release", () => { + const context = fixture(); + expect(context.runShell(releaseScript(true, false)).status).toBe(0); + expect(context.calls().map((call) => call.args)).toEqual([ + ["api", "repos/test/fleet/git/refs", "--method", "POST", "-f", "ref=refs/tags/v0.2.3", "-f", `sha=${sourceSha}`], + ["release", "create", "v0.2.3", "--verify-tag", "--title", "v0.2.3", "--generate-notes"] + ]); + }); + + test("never reuses a source tag that appeared after preflight", () => { + const context = fixture({ TEST_TAG_CREATE_CONFLICT: "true" }); + expect(context.runShell(releaseScript(true, false)).status).not.toBe(0); + expect(context.calls()).toHaveLength(1); + }); + + test("does not recreate a completed GitHub Release", () => { + const context = fixture(); + expect(context.runShell(releaseScript(false, true)).status).toBe(0); + expect(context.calls()).toHaveLength(0); + }); + + test("does not create a release for an existing image at a new dispatch SHA", () => { + const context = fixture(); + expect(context.runShell(releaseScript(false, false)).status).not.toBe(0); + expect(context.calls()).toHaveLength(0); + }); + + test("creates missing release metadata after verifying same-source image provenance", () => { + const context = fixture(); + expect(context.runShell(releaseScript(false, false, true)).status).toBe(0); + expect(context.calls()).toHaveLength(2); + expect(context.calls()[0]?.args).toContain(`sha=${sourceSha}`); + }); + + test("rechecks an existing same-source tag before finishing a partial release", () => { + const context = fixture(); + expect(context.runShell(releaseScript(false, false, true, true)).status).toBe(0); + expect(context.calls()).toHaveLength(2); + expect(context.calls()[0]?.args).toContain("repos/test/fleet/git/ref/tags/v0.2.3"); + expect(context.calls()[1]?.args.slice(0, 2)).toEqual(["release", "create"]); + }); + + test("refuses recovery if an existing tag moved after source verification", () => { + const context = fixture({ TEST_TAG_IDENTITY: `commit ${"c".repeat(40)}` }); + expect(context.runShell(releaseScript(false, false, true, true)).status).not.toBe(0); + expect(context.calls()).toHaveLength(1); + }); + + test.each(["refs/heads/other", "refs/tags/main"])("rejects a release dispatch from %s", (ref) => { + const context = fixture({ GITHUB_REF: ref }); + const guard = workflow.jobs.publish_and_verify.steps.find((step: { name?: string }) => step.name === "guard main branch before any publish"); + expect(context.runShell(guard.run).status).not.toBe(0); + }); +}); diff --git a/test/release-workflow.test.ts b/test/release-workflow.test.ts index 49d3be6..f292d9e 100644 --- a/test/release-workflow.test.ts +++ b/test/release-workflow.test.ts @@ -76,14 +76,6 @@ describe("release workflow", () => { step.run === "pnpm check-runner-version -- --fail-after-days 21" ) ).toBe(true); - expect( - steps.filter( - (step) => - typeof step.run === "string" && - step.run.includes("Runner.Listener --version") && - step.run.includes("cat /.runner-version") - ) - ).toHaveLength(2); expect( steps.some( (step) => @@ -165,8 +157,8 @@ describe("release workflow", () => { (step) => step.name === "guard main branch before any publish" && typeof step.run === "string" && - step.run.includes('GITHUB_REF_NAME') && - step.run.includes("main") && + step.run.includes('${GITHUB_REF}') && + step.run.includes("refs/heads/main") && step.run.includes("release-image may only publish from main") ) ).toBe(true); @@ -174,9 +166,7 @@ describe("release workflow", () => { (step) => step.name === "Preflight immutable release state" && typeof step.run === "string" && - step.run.includes("candidate-") && - step.run.includes("verification-only recovery mode") && - step.run.includes("refusing registry mutation") + step.run.includes("bash scripts/release/preflight.sh") ); const imagePublishIndex = steps.findIndex( (step) => @@ -192,15 +182,30 @@ describe("release workflow", () => { const promoteIndex = steps.findIndex( (step) => step.name === "Promote verified digest to final tag" ); + const runtimeIndex = steps.findIndex( + (step) => step.name === "Validate immutable image runtimes before promotion" + ); expect(String(steps[verifyIndex]?.run)).toContain("timeout 5m cosign verify"); expect(String(steps[verifyIndex]?.run)).toContain( "timeout 5m cosign verify-attestation" ); expect(perPlatformSignIndex).toBeLessThan(promoteIndex); - expect(promoteIndex).toBeGreaterThan(verifyIndex); + expect(runtimeIndex).toBeGreaterThan(verifyIndex); + expect(promoteIndex).toBeGreaterThan(runtimeIndex); + expect(String(steps[runtimeIndex]?.run)).toContain( + 'steps.release_state.outputs.working_ref }}@${{ steps.image_digest.outputs.digest' + ); + expect(String(steps[runtimeIndex]?.run)).toContain( + "bash scripts/release/validate-runtime.sh" + ); + expect(String(steps[runtimeIndex]?.run)).not.toContain("outputs.image_ref"); expect(String(steps[promoteIndex]?.run)).toContain( "docker buildx imagetools create" ); + expect(steps[promoteIndex]?.env).toMatchObject({ + RELEASE_REQUIRE_ABSENT: "true" + }); + expect(String(steps[promoteIndex]?.run)).toContain("bash scripts/release/preflight.sh"); expect( steps.some( (step) => @@ -209,21 +214,20 @@ describe("release workflow", () => { step.uses === "actions/upload-artifact@v7" ) ).toBe(true); - expect( - steps.filter( - (step) => - typeof step.run === "string" && - step.run.includes("command -v pgrep") && - step.run.includes("docker --version") && - step.run.includes("terraform version") - ) - ).toHaveLength(2); + expect(steps.findIndex((step) => step.name === "Preserve verification diagnostics")) + .toBeGreaterThan(promoteIndex); + for (const name of ["Sign image digest", "Sign per-platform image digests", "Attach SBOM attestation"]) { + expect(String(steps.find((step) => step.name === name)?.run)) + .toContain("bash scripts/release/retry.sh"); + } expect( steps.some( (step) => step.if === "${{ inputs.publish_project_release }}" && typeof step.run === "string" && step.run.includes("gh release create") && + step.run.includes('--verify-tag') && + step.run.includes('"sha=${GITHUB_SHA}"') && step.run.includes("--generate-notes") && step.run.includes("verification completed without mutation") ) diff --git a/test/runner-version.test.ts b/test/runner-version.test.ts index df7554c..5a7a354 100644 --- a/test/runner-version.test.ts +++ b/test/runner-version.test.ts @@ -41,7 +41,7 @@ describe("runner version helpers", () => { }); test("reads the canonical disabled-update runner version", () => { - expect(readCanonicalRunnerVersion()).toBe("2.336.0"); + expect(readCanonicalRunnerVersion()).toBe("2.337.0"); }); test("computes whole days since the latest runner release", () => {