From 114d015c512766f9c9e8e65a055b958329116cc0 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 12:03:41 +0000 Subject: [PATCH 01/47] fix(actions-lock): accept lockfile v0.0.3, which shares the v0.0.2 grammar gh-actions-lock v0.2.0 writes lockfile v0.0.3, and the estate gate now pins v0.2.0 (standards#1250). ActionsLock accepted only v0.0.2, so any relocked repository raised a high-severity invalid_actions_lock from workflow_audit and workflow_hardening (palimpsest-license alerts 347 and 349). Hypatia's own lock has been v0.0.3 since #926, so its own "parses the repository's authoritative generated lockfile" test failed. v0.0.3 has the v0.0.2 line grammar: same sections, same record fields, same `'': []` form (measured on this repo's lock and on rsr-template-repo's). Both versions are now parsed by the one strict grammar; every other version string is still refused, and a second version line is still a duplicate. Tests: one case for v0.0.3 (accepted; malformed v0.0.3 still fails closed), one for unsupported versions and a duplicate line. Run standalone against the module: 14/14 pass; the unpatched module fails 3 of them, including the existing own-lock test. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LET2eHeDbN3b4WuxYjMUX9 --- lib/rules/actions_lock.ex | 27 ++++++++++++++++++--------- test/rules/actions_lock_test.exs | 27 +++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/lib/rules/actions_lock.ex b/lib/rules/actions_lock.ex index fce24544..79fe2998 100644 --- a/lib/rules/actions_lock.ex +++ b/lib/rules/actions_lock.ex @@ -3,7 +3,8 @@ defmodule Hypatia.Rules.ActionsLock do @moduledoc """ - Strict local validation for machine-generated `gh actions-lock` v0.0.2 files. + Strict local validation for machine-generated `gh actions-lock` v0.0.2 and + v0.0.3 files. This module validates lockfile *shape and association*. It does not replace `gh actions-lock --verify`, which remains authoritative for resolving refs @@ -13,11 +14,19 @@ defmodule Hypatia.Rules.ActionsLock do The parser is deliberately line-oriented. `actions.lock` is generated by a single tool with a versioned stable format, and Hypatia must not add a general - YAML dependency merely to consume that format. Unknown or malformed v0.0.2 - lines fail closed rather than being guessed at. + YAML dependency merely to consume that format. Unknown or malformed lines fail + closed rather than being guessed at. + + v0.0.3 is what gh-actions-lock v0.2.0 writes, and the version the estate gate + pins (standards#1250). Its line grammar is the v0.0.2 grammar: the same + `workflows:` and `dependencies:` sections, the same record fields (`ref`, + `commit`, `owner_id`, `repo_id`, `uses`) and the same `'': []` form for a + workflow with nothing to lock (measured on rsr-template-repo and this repo's + own lock, 2026-10-19). Both versions are therefore parsed by one grammar; any + other version string is still refused. """ - @supported_version "v0.0.2" + @supported_versions ["v0.0.2", "v0.0.3"] @action_ref ~r/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]+@[^\s':]+$/ @dependency_key ~r/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+@[^\s':]+$/ @sha_ref ~r/^[0-9A-Fa-f]{40}$/ @@ -38,7 +47,7 @@ defmodule Hypatia.Rules.ActionsLock do } @doc """ - Parse and validate a complete v0.0.2 lockfile. + Parse and validate a complete v0.0.2 or v0.0.3 lockfile. Returns `{:error, reason}` for missing, unsupported, malformed, incomplete, duplicate, or contradictory input. One bad dependency invalidates the whole @@ -136,17 +145,17 @@ defmodule Hypatia.Rules.ActionsLock do cond do match = Regex.run(~r/^version: '([^']+)'$/, line) -> case match do - [_, @supported_version] when is_nil(state.version) -> - {:ok, %{state | version: @supported_version}} + [_, version] when version in @supported_versions and is_nil(state.version) -> + {:ok, %{state | version: version}} - [_, @supported_version] -> + [_, version] when version in @supported_versions -> line_error(line_number, :duplicate_version) [_, version] -> line_error(line_number, {:unsupported_version, version}) end - line == "workflows:" and state.version == @supported_version -> + line == "workflows:" and state.version in @supported_versions -> {:ok, %{state | section: :workflows}} true -> diff --git a/test/rules/actions_lock_test.exs b/test/rules/actions_lock_test.exs index e7e9c9eb..7c1c5432 100644 --- a/test/rules/actions_lock_test.exs +++ b/test/rules/actions_lock_test.exs @@ -83,6 +83,33 @@ defmodule Hypatia.Rules.ActionsLockTest do assert {:error, {:line, _, :malformed_dependency_commit}} = ActionsLock.parse(non_hex) end + test "accepts lockfile v0.0.3, which gh-actions-lock v0.2.0 writes, with the same grammar" do + v003 = String.replace(@valid_lock, "version: 'v0.0.2'", "version: 'v0.0.3'") + + assert {:ok, lock} = ActionsLock.parse(v003) + assert lock.version == "v0.0.3" + assert ActionsLock.pinned?(lock, "ci.yml", "actions/checkout@v7.0.1") + refute ActionsLock.pinned?(lock, "other.yml", "actions/checkout@v7.0.1") + + # The grammar is shared, not loosened: a malformed v0.0.3 lock still fails closed. + malformed = String.replace(v003, ~r/sha1-[0-9a-f]{40}/, "sha1-short") + assert {:error, {:line, _, :malformed_dependency_commit}} = ActionsLock.parse(malformed) + end + + test "rejects every version outside the supported set, and a second version line" do + for version <- ["v0.0.1", "v0.0.4", "v0.1.0", "0.0.3"] do + assert {:error, {:line, 2, {:unsupported_version, ^version}}} = + @valid_lock + |> String.replace("version: 'v0.0.2'", "version: '#{version}'") + |> ActionsLock.parse() + end + + twice = + String.replace(@valid_lock, "version: 'v0.0.2'", "version: 'v0.0.2'\nversion: 'v0.0.3'") + + assert {:error, {:line, 3, :duplicate_version}} = ActionsLock.parse(twice) + end + test "rejects unsupported versions and missing dependency entries" do assert {:error, {:line, 2, {:unsupported_version, "v9"}}} = @valid_lock From e68277c8b6d87547c0768e1e3f2c3453e0031246 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 12:07:22 +0000 Subject: [PATCH 02/47] ci: repin standards callers to 75bd599, whose own lock GitHub accepts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since GitHub began enforcing actions.lock at job setup (~2026-10-15), every job in a workflow that calls a standards reusable dies at `Set up job` with "Unable to resolve action actions/checkout@3d3c42e…: invalid lockfile pin". GitHub checks a reusable workflow's actions against the CALLEE's lock at the called SHA. Standards' lock at the pins used here (da2c748, 571cc73, 2479cf7) gives leaf actions their own SHA as ref; standards' lock-selfcheck rates all three POISON. All five callers now pin 75bd599 (standards main; every workflow green there; lock-selfcheck SELF-CONSISTENT). The interfaces are compatible: no new required inputs or secrets. This repo's lock records these callers as `[]` and has no self-SHA leaf records, so it needs no change (relock-standards-pin.sh: nothing to relock); the governance lock gate at 75bd599 (gh-actions-lock v0.2.0) exits 0 on this tree. Same root cause and fix as hyperpolymath/palimpsest-license#187. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LET2eHeDbN3b4WuxYjMUX9 --- .github/workflows/governance.yml | 2 +- .github/workflows/hypatia-scan.yml | 2 +- .github/workflows/mirror.yml | 2 +- .github/workflows/scorecard.yml | 2 +- .github/workflows/secret-scanner.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/governance.yml b/.github/workflows/governance.yml index 0bf27a56..b770367f 100644 --- a/.github/workflows/governance.yml +++ b/.github/workflows/governance.yml @@ -16,7 +16,7 @@ permissions: jobs: governance: - uses: hyperpolymath/standards/.github/workflows/governance-reusable.yml@da2c748aad55c1a1dcba00b60fe4a35017bc6540 + uses: hyperpolymath/standards/.github/workflows/governance-reusable.yml@75bd5994470661a099df562d6d5ad2fe744dbd66 permissions: actions: read contents: read diff --git a/.github/workflows/hypatia-scan.yml b/.github/workflows/hypatia-scan.yml index 61c81175..62c93431 100644 --- a/.github/workflows/hypatia-scan.yml +++ b/.github/workflows/hypatia-scan.yml @@ -21,5 +21,5 @@ permissions: jobs: hypatia: - uses: hyperpolymath/standards/.github/workflows/hypatia-scan-reusable.yml@da2c748aad55c1a1dcba00b60fe4a35017bc6540 + uses: hyperpolymath/standards/.github/workflows/hypatia-scan-reusable.yml@75bd5994470661a099df562d6d5ad2fe744dbd66 secrets: inherit diff --git a/.github/workflows/mirror.yml b/.github/workflows/mirror.yml index 0bd744f2..aaae2b43 100644 --- a/.github/workflows/mirror.yml +++ b/.github/workflows/mirror.yml @@ -41,7 +41,7 @@ permissions: jobs: mirror: - uses: hyperpolymath/standards/.github/workflows/mirror-reusable.yml@2479cf769ed5f0481ccf64860a2ab954514c2b59 + uses: hyperpolymath/standards/.github/workflows/mirror-reusable.yml@75bd5994470661a099df562d6d5ad2fe744dbd66 secrets: inherit permissions: actions: read diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 79a29fee..24486170 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -16,7 +16,7 @@ permissions: jobs: scorecard: - uses: hyperpolymath/standards/.github/workflows/scorecard-reusable.yml@da2c748aad55c1a1dcba00b60fe4a35017bc6540 + uses: hyperpolymath/standards/.github/workflows/scorecard-reusable.yml@75bd5994470661a099df562d6d5ad2fe744dbd66 permissions: actions: read contents: read diff --git a/.github/workflows/secret-scanner.yml b/.github/workflows/secret-scanner.yml index 0dbf55c7..542c81eb 100644 --- a/.github/workflows/secret-scanner.yml +++ b/.github/workflows/secret-scanner.yml @@ -22,5 +22,5 @@ jobs: contents: read pull-requests: write actions: read - uses: hyperpolymath/standards/.github/workflows/secret-scanner-reusable.yml@571cc734cd69fb846032ec77a662aa8ee4fc32cd + uses: hyperpolymath/standards/.github/workflows/secret-scanner-reusable.yml@75bd5994470661a099df562d6d5ad2fe744dbd66 secrets: inherit From dd11703e5ca72a1a590fffba969ef43b49980ca3 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 12:10:26 +0000 Subject: [PATCH 03/47] fix(ci): key the Idris 2 cache on the Chez Scheme version `ubuntu-latest` moved to the 26.04 image. The cached ~/.idris2 holds an idris2.so compiled by the previous image's Chez, and a fasl-object only loads under the Chez that wrote it, so every cache hit now dies with Exception: incompatible fasl-object version 0.0.0-pre-release.73 found in /home/runner/.idris2/bin/idris2_app/idris2.so and abi-codegen-drift (a required check) is red on every PR. The cache key did not name Chez, so the stale build kept being restored. The installed chezscheme package version is now part of the key in both workflows that share the entry (abi-codegen-drift.yml, verify-proofs.yml), so an image or package change rebuilds Idris 2 once instead of loading a stale build. verify-proofs.yml has the same latent failure; it simply has not run on the new image yet. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_0165rGu3tsKmJtevmrdMVyfJ --- .github/workflows/abi-codegen-drift.yml | 19 ++++++++++++++++++- .github/workflows/verify-proofs.yml | 19 ++++++++++++++++++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/.github/workflows/abi-codegen-drift.yml b/.github/workflows/abi-codegen-drift.yml index 0a436791..152de519 100644 --- a/.github/workflows/abi-codegen-drift.yml +++ b/.github/workflows/abi-codegen-drift.yml @@ -49,6 +49,23 @@ jobs: sudo apt-get update sudo apt-get install -y chezscheme + # The cached ~/.idris2 holds a Chez-compiled idris2.so. A fasl-object + # is only loadable by the Chez that wrote it, and `ubuntu-latest` moved + # to a newer image (26.04) with a newer Chez, so the old cache restored + # an idris2.so the new Chez refuses: + # Exception: incompatible fasl-object version 0.0.0-pre-release.73 + # The installed chezscheme package version is therefore part of the key, + # so an image or package bump rebuilds Idris 2 instead of loading a stale + # build. Keep this step and the key in step with abi-codegen-drift.yml / + # verify-proofs.yml, which share one cache entry on purpose. + - name: Chez Scheme version (Idris 2 cache key) + id: chez + run: | + set -euo pipefail + v="$(dpkg-query -W -f='${Version}' chezscheme | tr -c 'A-Za-z0-9.' '-')" + [ -n "$v" ] + echo "version=$v" >> "$GITHUB_OUTPUT" + - name: Cache Idris 2 install id: cache-idris uses: actions/cache@v6.1.0 @@ -60,7 +77,7 @@ jobs: # bootstrap. Keep the suffix in step with it. `-3` = the # single-prefix layout (#820); `-2` and earlier entries are the # split-prefix caches and must not be restored. - key: idris2-${{ env.IDRIS2_VERSION }}-${{ runner.os }}-3 + key: idris2-${{ env.IDRIS2_VERSION }}-${{ runner.os }}-chez${{ steps.chez.outputs.version }}-3 - name: Build Idris 2 from source if: steps.cache-idris.outputs.cache-hit != 'true' diff --git a/.github/workflows/verify-proofs.yml b/.github/workflows/verify-proofs.yml index b61a34c6..e2131ad0 100644 --- a/.github/workflows/verify-proofs.yml +++ b/.github/workflows/verify-proofs.yml @@ -89,6 +89,23 @@ jobs: sudo apt-get update sudo apt-get install -y chezscheme + # The cached ~/.idris2 holds a Chez-compiled idris2.so. A fasl-object + # is only loadable by the Chez that wrote it, and `ubuntu-latest` moved + # to a newer image (26.04) with a newer Chez, so the old cache restored + # an idris2.so the new Chez refuses: + # Exception: incompatible fasl-object version 0.0.0-pre-release.73 + # The installed chezscheme package version is therefore part of the key, + # so an image or package bump rebuilds Idris 2 instead of loading a stale + # build. Keep this step and the key in step with abi-codegen-drift.yml / + # verify-proofs.yml, which share one cache entry on purpose. + - name: Chez Scheme version (Idris 2 cache key) + id: chez + run: | + set -euo pipefail + v="$(dpkg-query -W -f='${Version}' chezscheme | tr -c 'A-Za-z0-9.' '-')" + [ -n "$v" ] + echo "version=$v" >> "$GITHUB_OUTPUT" + - name: Cache Idris 2 install id: cache-idris uses: actions/cache@v6.1.0 @@ -105,7 +122,7 @@ jobs: # every cache *hit* restored a launcher with no image and died # with `idris2_app/idris2.so: not found` (exit 127) before any # proof ran. ~/.idris2/bin/{idris2,idris2_app} travels together. - key: idris2-${{ env.IDRIS2_VERSION }}-${{ runner.os }}-3 + key: idris2-${{ env.IDRIS2_VERSION }}-${{ runner.os }}-chez${{ steps.chez.outputs.version }}-3 - name: Build Idris 2 from source if: steps.cache-idris.outputs.cache-hit != 'true' From 366009ff55e2a30ff3ea641727ee3abcfd763586 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 12:16:21 +0000 Subject: [PATCH 04/47] test(fixtures): give the Bitbucket mock v8 placeholder UUIDs The governance "UUID v8 conformance" gate (and the legacy "UUID v7 conformance" job, which runs the same strict check at standards 75bd599) never reached this tree before: every governance job died at job setup on the callee-lock pin. With the repin they run, and both fail on the four placeholder UUIDs in integration/fixtures/mock-responses/ bitbucket-repo.json, the only non-v8 literals in the repository. The values are invented placeholders, and nothing loads the file: load_mock_response() has no callers, and the Bitbucket mock test builds its responses inline. Flip each placeholder's version nibble to 8 and its variant nibble into 8-b, keeping the recognisable digit patterns. The file stays valid JSON, and check-uuid-v8.sh --strict now passes on the whole tree. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LET2eHeDbN3b4WuxYjMUX9 --- .../fixtures/mock-responses/bitbucket-repo.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/integration/fixtures/mock-responses/bitbucket-repo.json b/integration/fixtures/mock-responses/bitbucket-repo.json index 628c0b6a..06fcee5e 100644 --- a/integration/fixtures/mock-responses/bitbucket-repo.json +++ b/integration/fixtures/mock-responses/bitbucket-repo.json @@ -1,6 +1,6 @@ { "type": "repository", - "uuid": "{12345678-1234-1234-1234-123456789abc}", + "uuid": "{12345678-1234-8234-8234-123456789abc}", "full_name": "hyperpolymath/test-repo", "name": "test-repo", "slug": "test-repo", @@ -15,13 +15,13 @@ "has_wiki": false, "owner": { "type": "user", - "uuid": "{87654321-4321-4321-4321-cba987654321}", + "uuid": "{87654321-4321-8321-8321-cba987654321}", "username": "hyperpolymath", "display_name": "hyperpolymath", - "account_id": "123456:abcdef12-3456-7890-abcd-ef1234567890", + "account_id": "123456:abcdef12-3456-8890-abcd-ef1234567890", "links": { "self": { - "href": "https://api.bitbucket.org/2.0/users/%7B87654321-4321-4321-4321-cba987654321%7D" + "href": "https://api.bitbucket.org/2.0/users/%7B87654321-4321-8321-8321-cba987654321%7D" }, "html": { "href": "https://bitbucket.org/hyperpolymath/" @@ -33,7 +33,7 @@ }, "workspace": { "type": "workspace", - "uuid": "{87654321-4321-4321-4321-cba987654321}", + "uuid": "{87654321-4321-8321-8321-cba987654321}", "name": "hyperpolymath", "slug": "hyperpolymath", "links": { @@ -51,7 +51,7 @@ "project": { "type": "project", "key": "TEST", - "uuid": "{11111111-1111-1111-1111-111111111111}", + "uuid": "{11111111-1111-8111-8111-111111111111}", "name": "Test Project", "links": { "self": { From abbe38233cc8a647de9ef6c41376f35117967606 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 12:16:22 +0000 Subject: [PATCH 05/47] ci(gitleaks): ignore five historical false positives by exact fingerprint Standards 75bd599 adds a gating full-history gitleaks pass to secret-scanner-reusable.yml (fetch-depth 0, every commit). It reports five generic-api-key matches, all in files that f2fe2df removed from the tree: - deploy/k8s/secrets.yaml at 0d310fa, lines 38/40/44/50: a Secret template whose values are base64 of the placeholder CHANGE_ME_IN_PRODUCTION, as the comment above each line says. Checked by comparing each value against the encoded placeholder. - deploy/terraform/modules/monitoring/main.tf at 269dac0, line 421: a Prometheus recording-rule name, not a credential. Ignore exactly those five commit:file:rule:line fingerprints, following the shape of standards' own .gitleaksignore. They cannot suppress a match in any new commit or location. Verified locally with the pinned gitleaks 8.18.4 (sha256 as in the reusable workflow) and the estate baseline from 75bd599: - history pass: 5 leaks before, none after; - working-tree pass: still clean; - negative control: a planted random secret at the same path and line is still caught by both passes. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LET2eHeDbN3b4WuxYjMUX9 --- .gitleaksignore | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .gitleaksignore diff --git a/.gitleaksignore b/.gitleaksignore new file mode 100644 index 00000000..4a0185fe --- /dev/null +++ b/.gitleaksignore @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: MPL-2.0 +# +# Exact historical fingerprints only (commit:file:rule:line). The full-history +# gitleaks pass in hyperpolymath/standards secret-scanner-reusable.yml scans +# every commit; these five matches sit in two files that f2fe2df removed from +# the tree. Each entry names what the matched value is. None of them suppresses +# a rule in any new commit or location. +# +# deploy/k8s/secrets.yaml at 0d310fa is a Secret TEMPLATE. Each of the four +# values is base64 of the placeholder CHANGE_ME_IN_PRODUCTION, as the comment +# above each line says. Checked 2026-10-19 by comparing every value against +# the encoded placeholder. +0d310faa0d74da36faa8f5cfb5b6786ebe912c1b:deploy/k8s/secrets.yaml:generic-api-key:38 +0d310faa0d74da36faa8f5cfb5b6786ebe912c1b:deploy/k8s/secrets.yaml:generic-api-key:40 +0d310faa0d74da36faa8f5cfb5b6786ebe912c1b:deploy/k8s/secrets.yaml:generic-api-key:44 +0d310faa0d74da36faa8f5cfb5b6786ebe912c1b:deploy/k8s/secrets.yaml:generic-api-key:50 +# +# deploy/terraform/modules/monitoring/main.tf at 269dac0, line 421, is a +# Prometheus recording-rule name (record = "cicd_hyper_a:api:"). It is +# a metric identifier, not a credential. +269dac03d57835a1dfe3fb336095ffac61567ccf:deploy/terraform/modules/monitoring/main.tf:generic-api-key:421 From 61624390f11c0186c4ac3065f03f3ea7fd7e5c67 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 12:24:44 +0000 Subject: [PATCH 06/47] ci: pass reusable workflows only the secrets they declare MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four callers used `secrets: inherit`, which forwards every repository and organisation secret to the callee (CWE-250). The callees at standards 75bd599 declare what they need, and their headers say not to inherit (standards#1108 §6b): - hypatia-scan-reusable declares HYPATIA_SCAN_PAT (optional); - mirror-reusable declares the seven forge keys (all optional); - scorecard-reusable and secret-scanner-reusable declare none. Pass exactly those. The `uses:` lines are unchanged, so actions.lock is too. This clears the four WH008 findings that the governance baseline gate reports on these files. It matches the change CodeRabbit asked for on hyperpolymath/palimpsest-license#187. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LET2eHeDbN3b4WuxYjMUX9 --- .github/workflows/hypatia-scan.yml | 4 +++- .github/workflows/mirror.yml | 10 +++++++++- .github/workflows/scorecard.yml | 1 - .github/workflows/secret-scanner.yml | 1 - 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/hypatia-scan.yml b/.github/workflows/hypatia-scan.yml index 62c93431..f9dfc9d7 100644 --- a/.github/workflows/hypatia-scan.yml +++ b/.github/workflows/hypatia-scan.yml @@ -22,4 +22,6 @@ permissions: jobs: hypatia: uses: hyperpolymath/standards/.github/workflows/hypatia-scan-reusable.yml@75bd5994470661a099df562d6d5ad2fe744dbd66 - secrets: inherit + # Only the one secret the callee declares (standards#1108 §6b). + secrets: + HYPATIA_SCAN_PAT: ${{ secrets.HYPATIA_SCAN_PAT }} diff --git a/.github/workflows/mirror.yml b/.github/workflows/mirror.yml index aaae2b43..95d1c65e 100644 --- a/.github/workflows/mirror.yml +++ b/.github/workflows/mirror.yml @@ -42,7 +42,15 @@ permissions: jobs: mirror: uses: hyperpolymath/standards/.github/workflows/mirror-reusable.yml@75bd5994470661a099df562d6d5ad2fe744dbd66 - secrets: inherit + # Only the seven forge keys the callee declares (standards#1108 §6b). + secrets: + GITLAB_SSH_KEY: ${{ secrets.GITLAB_SSH_KEY }} + BITBUCKET_SSH_KEY: ${{ secrets.BITBUCKET_SSH_KEY }} + CODEBERG_SSH_KEY: ${{ secrets.CODEBERG_SSH_KEY }} + SOURCEHUT_SSH_KEY: ${{ secrets.SOURCEHUT_SSH_KEY }} + DISROOT_SSH_KEY: ${{ secrets.DISROOT_SSH_KEY }} + GITEA_SSH_KEY: ${{ secrets.GITEA_SSH_KEY }} + RADICLE_KEY: ${{ secrets.RADICLE_KEY }} permissions: actions: read contents: read diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 24486170..3bc19ee8 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -22,4 +22,3 @@ jobs: contents: read security-events: write id-token: write - secrets: inherit diff --git a/.github/workflows/secret-scanner.yml b/.github/workflows/secret-scanner.yml index 542c81eb..36571a8b 100644 --- a/.github/workflows/secret-scanner.yml +++ b/.github/workflows/secret-scanner.yml @@ -23,4 +23,3 @@ jobs: pull-requests: write actions: read uses: hyperpolymath/standards/.github/workflows/secret-scanner-reusable.yml@75bd5994470661a099df562d6d5ad2fe744dbd66 - secrets: inherit From c19c465b9010f60532efba7c1fa008d957e42ad6 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 12:24:44 +0000 Subject: [PATCH 07/47] build(cargo): keep standards' staged js-runtimes crate out of the workspace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At standards 75bd599, the governance "JS runtimes — environment provisions Bun first" job checks standards' tools/js-runtimes out at .standards-js/ inside the caller's checkout, then runs `cargo install --locked --path .standards-js/tools/js-runtimes`. Here that path sits under this repository's root workspace, so cargo refuses: error: current package believes it's in a workspace when it's not Exclude .standards-js from the workspace. Reproduced locally by staging the crate the same way: `cargo metadata` fails with that error before the change and passes after. The job's full `cargo install --locked` then builds, and `js-runtimes provision .` exits 0. The root fix belongs in standards (an empty [workspace] table in the crate's manifest); this line is harmless once that lands. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LET2eHeDbN3b4WuxYjMUX9 --- Cargo.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index edc5d904..703051e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,11 @@ members = [ "integration", "tools/cii-registrar", ] +# standards' governance "JS runtimes" job checks its js-runtimes crate out at +# .standards-js/ inside this checkout and `cargo install`s it from there. +# Without this, cargo treats that crate as an undeclared member of this +# workspace and refuses to build it. +exclude = [".standards-js"] [workspace.package] version = "0.1.0" From a50c0e5ea8ceee35b3dda3fa9978d944977615b2 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 12:33:46 +0000 Subject: [PATCH 08/47] fix(actions-lock): name v0.0.3's GHE.com hostname field instead of a generic error v0.0.3 is not quite the v0.0.2 grammar. gh-actions-lock v0.2.0 adds one optional per-dependency field, `hostname` (`json:"hostname,omitempty"`, confirmed in the v0.2.0 binary). It writes the field only for a GHE.com data-residency tenant, never for github.com, so neither real v0.0.3 lock in the estate (this repo's, rsr-template-repo's) carries it. Make the parser say so instead of reporting a generic :malformed_dependency_entry: - In a v0.0.3 lock, `hostname: 'github.com'` is accepted. It states the default. - In a v0.0.3 lock, any other host is refused as {:unsupported_dependency_hostname, host}. Repository identity (owner_id/repo_id uniqueness) is validated per host, and this parser models github.com only. - In a v0.0.2 lock, a hostname line is still malformed: the field did not exist then. The moduledoc now describes the restricted schema accurately. Tests: a new case covers all three behaviours and a duplicate hostname line. The real locks still parse: hypatia v0.0.3 (40 deps), rsr-template-repo v0.0.3 (37), palimpsest-license v0.0.2 (38), standards v0.0.2 (26). Addresses CodeRabbit's finding on #929. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LET2eHeDbN3b4WuxYjMUX9 --- lib/rules/actions_lock.ex | 30 ++++++++++++++++++++++++------ test/rules/actions_lock_test.exs | 27 +++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/lib/rules/actions_lock.ex b/lib/rules/actions_lock.ex index 79fe2998..9fcec245 100644 --- a/lib/rules/actions_lock.ex +++ b/lib/rules/actions_lock.ex @@ -18,12 +18,20 @@ defmodule Hypatia.Rules.ActionsLock do closed rather than being guessed at. v0.0.3 is what gh-actions-lock v0.2.0 writes, and the version the estate gate - pins (standards#1250). Its line grammar is the v0.0.2 grammar: the same - `workflows:` and `dependencies:` sections, the same record fields (`ref`, - `commit`, `owner_id`, `repo_id`, `uses`) and the same `'': []` form for a - workflow with nothing to lock (measured on rsr-template-repo and this repo's - own lock, 2026-10-19). Both versions are therefore parsed by one grammar; any - other version string is still refused. + pins (standards#1250). For github.com dependencies its line grammar is the + v0.0.2 grammar: the same `workflows:` and `dependencies:` sections, the same + record fields (`ref`, `commit`, `owner_id`, `repo_id`, `uses`) and the same + `'': []` form for a workflow with nothing to lock (measured on + rsr-template-repo and this repo's own lock, 2026-10-19). Any other version + string is still refused. + + The one v0.0.3 addition is an optional per-dependency `hostname`, which + gh-actions-lock writes only for a GHE.com data-residency tenant + (`hostname,omitempty`). An explicit `hostname: 'github.com'` is accepted as the + default it states. Any other host is refused as + `{:unsupported_dependency_hostname, host}`. Repository identity is validated + per host, this parser models github.com only, and the estate has no tenant + locks. """ @supported_versions ["v0.0.2", "v0.0.3"] @@ -222,6 +230,16 @@ defmodule Hypatia.Rules.ActionsLock do Regex.match?(~r/^ (?:owner_id|repo_id):/, line) -> line_error(line_number, :malformed_dependency_identity) + # v0.0.3 only: an explicit github.com host is the default spelled out. + line == " hostname: 'github.com'" and state.version == "v0.0.3" -> + put_dependency_field(state, :hostname, "github.com", line_number) + + # A GHE.com tenant host is refused by name: owner/repo identity is + # validated per host, and this parser models github.com only. + match = state.version == "v0.0.3" && Regex.run(~r/^ hostname: '([^']+)'$/, line) -> + [_, host] = match + line_error(line_number, {:unsupported_dependency_hostname, host}) + line == " uses:" -> put_dependency_field(state, :uses, MapSet.new(), line_number) diff --git a/test/rules/actions_lock_test.exs b/test/rules/actions_lock_test.exs index 7c1c5432..78af2905 100644 --- a/test/rules/actions_lock_test.exs +++ b/test/rules/actions_lock_test.exs @@ -96,6 +96,33 @@ defmodule Hypatia.Rules.ActionsLockTest do assert {:error, {:line, _, :malformed_dependency_commit}} = ActionsLock.parse(malformed) end + test "v0.0.3 hostname: github.com is the default, a GHE.com tenant host is refused" do + v003 = String.replace(@valid_lock, "version: 'v0.0.2'", "version: 'v0.0.3'") + + with_host = fn lock, host -> + String.replace( + lock, + " repo_id: 197814629", + " repo_id: 197814629\n hostname: '#{host}'" + ) + end + + assert {:ok, lock} = ActionsLock.parse(with_host.(v003, "github.com")) + assert ActionsLock.pinned?(lock, "ci.yml", "actions/checkout@v7.0.1") + + assert {:error, {:line, 13, {:unsupported_dependency_hostname, "acme.ghe.com"}}} = + ActionsLock.parse(with_host.(v003, "acme.ghe.com")) + + # The field is new in v0.0.3; a v0.0.2 lock carrying it is still malformed. + assert {:error, {:line, 13, :malformed_dependency_entry}} = + ActionsLock.parse(with_host.(@valid_lock, "github.com")) + + duplicate = with_host.(with_host.(v003, "github.com"), "github.com") + + assert {:error, {:line, _, {:duplicate_dependency_field, _, :hostname}}} = + ActionsLock.parse(duplicate) + end + test "rejects every version outside the supported set, and a second version line" do for version <- ["v0.0.1", "v0.0.4", "v0.1.0", "0.0.3"] do assert {:error, {:line, 2, {:unsupported_version, ^version}}} = From d6aea2d3cbaa108650477a637474e7de28c7584f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 12:54:41 +0000 Subject: [PATCH 09/47] feat(rules): detect the three workflow-pin defect classes behind the estate's dead Rust CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Detection half of the pin-propagation architecture (hypatia detects → gitbot-fleet actuates → .git-private-farm propagates), for three classes measured 2026-09-29..2026-10-19; each has a fleet fix script carrying the same facts (gitbot-fleet scripts/fix-rust-toolchain-input.sh, fix-stale-reusable-pin.sh, fix-lock-self-sha-refs.sh). * WF026 (workflow_audit.ex) — a SHA-pinned dtolnay/rust-toolchain step with no `toolchain:` input. The action reads the channel from its @-ref; a commit SHA carries none, so the step exits 1 with "'toolchain' is a required input" before any cargo command runs and the red reads as a code failure (plasma-parser-writer#105; standards#439 fixed the reusable). Step bodies are bounded by indentation, so a later step's toolchain: cannot rescue an earlier one; tag pins never fire. * WF027 (workflow_audit.ex) — a caller pins an estate reusable at a revision MEASURED dead for that reusable: a data table keyed by reusable path, first row the 83 standards main commits of June 2026 whose rust-ci-reusable.yml carries the WF026 defect (every first-parent commit d73c93e0..49ed4715, file content checked, not dates), with the replacement a human chose (75bd599). Unlike WF016 these commits resolve: the run starts and the reusable's first step fails. * PIN-003 (pr-automerge-policy.json, read by PI001 and PA001) — the 545 standards commits reachable from main, 2026-08-19..2026-10-19, whose actions.lock carries an action record with its own SHA as ref. Since ~2026-10-15 GitHub validates a called reusable against the CALLEE's lock at the called SHA and rejects those records ("invalid lockfile pin"), so every standards reusable at those revisions dies at Set up job for every caller (hypatia#929; cicd-squabbler run 38846539059, byte-identical to its last green run). Keyed on SHA only — standards reusables are never pinned by tag; known_good_sha 75bd599, the first main commit with a clean lock. standards' own lock-selfcheck.sh rates these commits SELF-CONSISTENT because it checks that refs are keyed, not that a key's ref is a tag — which is why the list is here. * sha_bump_propagation.ex — hyperpolymath/cicd-suite's reusables (the estate audit and the Rust, Creusot, SPARK and miniKanren gates of cicd-suite#54) join @known_reusables, so a cicd-suite bump feeds the propagator like a standards bump. Tests: test/rules/workflow_audit_pins_test.exs (13: fires, silent, the indentation boundary, the per-reusable window, the table's edges, the audit's counts); pin_integrity_test.exs reads the committed policy with OTP's :json and checks PIN-003 denylists 8f2ee50 through a reusable path while 75bd599 and the June revisions are not denylisted; sha_bump_propagation_test.exs covers the cicd-suite rows. Run outside Mix in this session (Hex is unreachable here): the WF026/WF027 tests against a scratch build of the new functions, the rest against the compiled modules — all pass; the two pre-existing actions_lock_test failures outside Mix (v0.0.3 lock, Hypatia.Rules not loaded) are unrelated. CI's mix test is the first full run. AGENTIC.a2ml records the contract: tables extend only from a measured walk; an agent never names a replacement revision; the WF027 table and PIN-003 mirror gitbot-fleet's stale-reusable-pins.tsv windows A and B and are regenerated together. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- .machine_readable/descriptiles/AGENTIC.a2ml | 15 + .../pr-automerge-policy.json | 633 +++++++++++++++++- lib/rules/sha_bump_propagation.ex | 9 +- lib/rules/workflow_audit.ex | 267 +++++++- test/rules/pin_integrity_test.exs | 36 + test/rules/sha_bump_propagation_test.exs | 27 + test/rules/workflow_audit_pins_test.exs | 168 +++++ 7 files changed, 1131 insertions(+), 24 deletions(-) create mode 100644 test/rules/workflow_audit_pins_test.exs diff --git a/.machine_readable/descriptiles/AGENTIC.a2ml b/.machine_readable/descriptiles/AGENTIC.a2ml index d07ff88b..31a5f12f 100644 --- a/.machine_readable/descriptiles/AGENTIC.a2ml +++ b/.machine_readable/descriptiles/AGENTIC.a2ml @@ -40,6 +40,21 @@ require-rerun-after-fix = true release-claim-requires-hard-pass = true completion-claim-requires-behavioural-tests = true +[workflow-pin-detection] +# Hypatia's half of the pin-propagation architecture +# hypatia (this: detection) -> gitbot-fleet (actuation) -> .git-private-farm (propagation) +# Every rule here has a fleet fix script that carries the SAME facts, so a +# detection without a repair, or a repair without a detection, is a drift: +# WF026 workflow_audit.ex SHA-pinned dtolnay/rust-toolchain with no toolchain: input -> fleet fix-rust-toolchain-input.sh +# WF027 workflow_audit.ex caller pinned at a reusable revision MEASURED dead (table) -> fleet fix-stale-reusable-pin.sh (Window A) +# PIN-003 pr-automerge-policy.json standards revisions whose own actions.lock GitHub +# rejects at job setup (545 commits, 2026-08-19..2026-10-19; PI001/PA001 fire) -> fleet fix-stale-reusable-pin.sh (Window B) +# known_reusables (sha_bump_propagation.ex) now include hyperpolymath/cicd-suite's gates +may-extend-dead-revision-tables = true # only from a MEASURED walk (file content or lock record checked per commit), never from dates +may-name-a-replacement-revision = false # a human chooses it against standards/scripts/lock-selfcheck.sh; agents carry the choice into the table +must-mirror-fleet-tables = true # WF027's list == fleet stale-reusable-pins.tsv Window A; PIN-003 == Window B; regenerate together +must-not-promote-to-blocking = true # these rules report and route to the fleet; a gate that blocks on them is an owner decision + [automation-hooks] on-enter = [ "Read 0-AI-MANIFEST.a2ml or AI.a2ml first", diff --git a/.machine_readable/merge-orchestration/pr-automerge-policy.json b/.machine_readable/merge-orchestration/pr-automerge-policy.json index 89e2a09f..e8595a68 100644 --- a/.machine_readable/merge-orchestration/pr-automerge-policy.json +++ b/.machine_readable/merge-orchestration/pr-automerge-policy.json @@ -2,19 +2,24 @@ "$schema": "https://github.com/hyperpolymath/hypatia/blob/main/docs/design/merge-orchestration/schemas/pr-automerge-policy.schema.json", "$comment": "SPDX-License-Identifier: MPL-2.0 — Single source of truth for estate PR automerge + pin integrity. Read by BOTH the Elixir brain (Hypatia.Automerge.Policy) and the token-bearing actuator (scripts/estate/pr-automerge.sh). Two independent readers, one policy file: the actuator never trusts the brain's verdict, it re-derives it from the same data.", "version": "1.0.0", - "updated": "2026-09-26", + "updated": "2026-10-19", "owner": "hyperpolymath", "description": "Declarative policy for the 'safer core' of hypatia: unambiguous bumps/chores/pins get merged and their branches deleted; poisoned pins are excised or vetoed; everything else is flagged. Written after the 2026-09-26 finding that 108 workflow files across 84 hyperpolymath repos (and 23 in metadatastician) carry the codeql-action v4.38.1 commit labelled as v4.38.0.", - "pin_denylist": [ { "id": "PIN-001", "action": "github/codeql-action", "ecosystem": "github-actions", "severity": "critical", - "blocked_versions": ["4.38.1"], - "blocked_shas": ["1c5b675653bb5c22dbe9b12b556ec555138e09fd"], - "blocked_refs": ["v4.38.1"], + "blocked_versions": [ + "4.38.1" + ], + "blocked_shas": [ + "1c5b675653bb5c22dbe9b12b556ec555138e09fd" + ], + "blocked_refs": [ + "v4.38.1" + ], "reason": "GitHub rejects this commit at workflow start-up: `startup_failure`, zero jobs, no logs, wherever it is used. Kills CodeQL, Hypatia Security Scan and Scorecard on every repo it reaches.", "known_good_version": "4.38.0", "known_good_sha": "b96794f015dfd88f77b49b1c93e0fa7110f94c63", @@ -40,13 +45,582 @@ "known_good_version": null, "known_good_sha": null, "replacement": "unpin_moving_ref", - "evidence": ["hypatia/docs/DEBT-REGISTER.adoc CI-1 (run 31170993296, job 92842543720)"], + "evidence": [ + "hypatia/docs/DEBT-REGISTER.adoc CI-1 (run 31170993296, job 92842543720)" + ], "added": "2026-09-26", "added_by": "hypatia debt register CI-1", "notes": "Structural cure, not a re-pin: drop the entry from actions.lock rather than pinning a new SHA." + }, + { + "id": "PIN-003", + "action": "hyperpolymath/standards", + "ecosystem": "github-actions", + "severity": "critical", + "blocked_versions": [], + "blocked_shas": [ + "fae7f01010f44606a65f84fe53e613201a0950ac", + "e16a23170fa378bf3b1444c3ee94100413ae3ff7", + "b2cb7b685e17bd24e0122d19950a45de94ddcbc6", + "8ff9ce65f7e54f8d5c5aa94584e2ead0f2ba460f", + "ce768a6abea6fb0bca9b4c3eeb135c3a75a7f06e", + "3de430074148c49c8db8878653b3c66707c344e4", + "f7573a00ba48a2dc19c6f912d3096f95b38f7be5", + "813dcbaba53f11c06e09a0092c8466b4cbb778ee", + "f335165a8882abadc99a092f55ff950b392af2d8", + "a88f511b9bc06b430a4b4071efe75d91921cd9f7", + "0d9a814ac8d7b1d5c1f39d8510639e8fdd99f3e4", + "f78a36c9058cbab0b6610a675d3d5d7b024fd837", + "68242e650c0bd83bbf280f3b578fa949c0ab4c16", + "94572e612a7bf11f8e6343d19adadf17ef2a3de3", + "80014df42eb81f09396c948299febb258cec4719", + "95de3dfc3fcf09a1cf3325ab42723e3d81682b45", + "6a9251037760690d56022ad6fab2cfe9d887dd76", + "35cf3e2b05ab1efaecf72f8e0875ee7b6da46b88", + "ed04132e56c086795d3cc478cbdc145333ab5780", + "25824f323e76d481a3d37f22f28440ef77dc7905", + "934b879f6068295e42dac20d80e010d555ebec82", + "25d23cf3934e6c1fffd275ac78398c70aa9d58f3", + "8a39fa0bdb4d7787f7529111be417f984c85381e", + "192389bf1b40b70c594464937db7fdbf39a9a964", + "2ae085c076a4b14990bf67da0dd3821ce1c12944", + "c11c255e1b5546854e874d1486d61d6d5a4da29a", + "75064de100849f1864f8fe1c912e64c48d70a3ee", + "ae36881b5ff4b469fb904b66542ac226a15107f3", + "3daaad25e39d671dde8ad58e6427282309b02cce", + "b26d3fb548fa2c4a7920cc69128821811d4e3222", + "50cfbd55d44130399bf1215c04fd154a49277ee8", + "507b89f1b35453bf3de62622373cc6573245b664", + "bf21155871bfd8d4789f33543d0c30880b8403cd", + "e3d5107b6319dc1121e3edf12f497bed2f9fcd61", + "7d2041a2315572d8257341449257e6cfddc9aebc", + "48e0ed9138ade95a5357f76079fc56d8f6f91bb5", + "736f792321e24af85b0dfc44b7666538bf4a2ff1", + "312ff116d9acbfff3523da5e9f71ac3ca9c327d3", + "0b1898e2e6a0fc88348c0f2517970593d373ca10", + "4c101fbbc0b20e816c6beca9fc0c909994163443", + "9164d76efef88f5173fdb717b231618a2d3c701a", + "3108c1b01186741eea80c00da3f4beda912286f9", + "7712ae1568af5c872408bfa1ba27578dd2afecd0", + "8496854d8241c3a0e20abaecf67e9c70638aa3b1", + "dd58dd9659f425a3f2372b123dde524c27d045a4", + "9d629d112daa396636bbb32757213552a8fb6878", + "76773445d8b0c3a2558f33cc426bacc638206a6f", + "6d19ce0ebae047027dcfd594c7810bf35c642faa", + "d8848f8509ccd94e931f6768487c88c8423f6033", + "2e12b312b529c51f095ce3b22cd35968f5aba898", + "ca12cd467cb13d02072837f9a2a69ede429b0b09", + "900c42c70b968e11a2bca04bf0ce050be5c286dc", + "a695e37936bacff66add7ec354eb629f659477e5", + "bbcc722bb0a5969ca0b1ebb1178af2ebcd5e9c34", + "691ea2af89d4313c906672d880690f81219dbdc5", + "d8a90cf8093ab787e98b940cd7d7675b2c981ec2", + "e35431d5bce94895b675d5a135e3ff5af1ca4a63", + "a709d3a642bee6e8d22829a69d2d64e1e8656d57", + "c9dfc2dcb333c1672aa779fcc86671be037ee735", + "8639b851db282e89083351aa782d906eb537def2", + "2f9d0be57ecbb09fb5d93df411fa6c9758ab305c", + "0683ddf00237ae92c5756cf858b3da1cc706894e", + "85aa1934975a8df576c7a4912bacbb8160ab2d2d", + "69ea45cfefab795faa2122df468ce7d120ebdff2", + "f77fc28ad400f281d26431b5e6a204dfc7a2ea22", + "63d2bbf64020b5f87bc546973d21881c19e79c65", + "8453510aa0da5027dabe54743e06ed7e7648fbe2", + "d7b85cac57eb16edf51508d6f30806e86d63c9d3", + "d5a7cfa4a9e9b77e1ec45b828ea32e3bdba4634f", + "9f68971b9fa254a80559432d9fcb7652b72e941d", + "b9e78cd426a83d87408d1f5db9e798b9fb8a3bbe", + "8b5dfd0646c1232174e680ae7772250610bfa2b8", + "7d51f30f992b51fe2b33119761901c594e266e82", + "89d778d9dc5343ad3dc178656c60e9caf235588b", + "043496c2d1efb9ba48f69ec533b5444ab61bbb5d", + "ebfcd98ba6c162df2eeca854254e725095fc85d2", + "239759976b272a426a3b7d34d0d48c0009bddcfb", + "79df53059d638e699b317951f717ee9710d2618d", + "640d86efeddb1e47da230515f5af9f4c936bba80", + "ad637e5134bc9f02ddf65431827d515b26bd58ee", + "bae58d9398f6deff6187a18c59620c984b6e1c47", + "223a9da337e327246d17ad49800cf567744a3d5e", + "617f3bcb83d33f5a0e3793c41d26a3c7937dfd16", + "0ae8b773b5a0f16ca1c5389442c907ceacf75e00", + "eafcfbb5933dde8b42fdbd61483794bb75fb4c97", + "5a0891e9e58b785181a92e141505ca96ca987b42", + "8464935a8028dbf0b06eb3d2b9bc7c7a7a35b4c6", + "b935e63937b1f1b5cbaeea7e375947d7b924038f", + "44a0de481c1b10843b358e6c5c278efbfd4eaad9", + "cfc3a92e55f6e6feeed33412d9fad94812d330ed", + "78a0a61f481bbe394214002f51f41a92325c0fff", + "ce92a8cdb60b60ea4b8cd5040a14c5ec2eec9f0e", + "5751b97ed597e560573242f56f1287db159edd00", + "a5ff08d7a12399fe21fa9ed72841a9c5749d086b", + "8556163f6bb9f06e2ed9beb67a91a335aa1f8913", + "ef7e6c4cc801d1ebf5504b940ff70f72061334d0", + "0cb69026a45f756d6610d8b67e298c4c3d6acc95", + "67e148fe710381c732ee1a6a8b2898688ad13110", + "80b3a1abccfe39b16d8af8ead874e09e9529737b", + "5b56aec07917394f48d51cc95c459b12dc9c179e", + "32d37263e96600ee81ec27f343bef812c4670968", + "3bd25bd0fa9bb8f5a50d9d8288ad6c615a83fbda", + "bee32592ef4952e7e0dce8035e66f827c79839b5", + "44ca61e2deca34d8fa68a31922b1efbe7f13f241", + "8ffcaf6251be5b05fe8ae4b3b88efbc5acc203bb", + "5693b8b28b402355ab4516db4db3246b0b1019b4", + "6075712d2138d0bdcb1192d473ba50fba9501e0f", + "aa1c4dd591fa2d5ecf0d65d53bb127eab6b95486", + "c33b22b1037d6eabceb7673975e50c78030059d7", + "a41f3692d3e17193f14ede5cd7f79172f339085b", + "65d9af0140d3761b4b450a5f475d5490b31ec5e2", + "4d60617725a5eb899cc5ebba85e72d598e66560c", + "4e13946f6ff22657c7ce1592e42100222da4e554", + "1f099de983ed1ec26b8fbfdf176b69950291ead3", + "177be894c56f68c3f3a3eb428b4993a8996da892", + "e59aad7153dd7bc87c056039fcd71ba0567f37af", + "93f0c3fdc0c2dbf24d41cb888668adc4e34c6352", + "2f8c3eebad6418c04ed8dcc35d0b1ac4bf3954d4", + "b3075e041f52cb6f7e3f9bc610e22000ce388548", + "5c300a80f82d94d6faf02de0da7a1ab25700461f", + "a9a3281d620bdd195369c9a6f0bc1d661910965a", + "0ad178797da63e826742c20600a89e50c47c20ec", + "e2e0f6d61c2d2a36ef516849dc1ca9223563479f", + "c55031473e743591b649da28cc16a4f095712082", + "804b780093b3c48842913ee472e9191afe5f8329", + "f14575da71758f21a437fc875ec99d68399d7931", + "f5f48a4605aff2622330356f45cf720871aa6c54", + "aa8fd9e81585ced5ce0fa580092b21e1f84d0fe1", + "202a215d01297eb825c0d45495fcbc46a82a5e30", + "907bbfe535d21e0709eb4a67fbea1e304beec22b", + "11ba299dca221b6d4e970eeb58dd17ccbb825a9a", + "56b767ff3f70ea5427900c37659b0ea94eb49628", + "c780e55efec3ef2f765cb0223cd154102a85a1a3", + "e4b64b00cf7698a624ea8a0a8bfba551d4674dc0", + "215df8e96590dda0539d168eba000a579ab52517", + "71bb615e234133d22f04106c812aad9fcb269aa5", + "721e4a9e1beb2392ee096d38ead44c0ece962eb3", + "10e0b6ffb979f0a27fe58206512055133e67460a", + "39ee4ec5e026dbdae084989c856a655e1e342629", + "9719570729342a13fac0ead93a68d4516c5dab79", + "df20618b3ce9cefc19d603d5300952550074a3fe", + "13b872c19a9459e21a7f91aaebaf2a29e99684c9", + "0187ccd95604614009b37bb23a55f2e634cdc404", + "89813bf8e4cb806eb01232b396410c28f30ca742", + "1b6e19eaef9e81ec19d6e826837e7e1a09f968cd", + "2c06ddf3d4f0d682c70ae75643db307df4fd36dd", + "7de6a8a7965950448a3abd0fe16653fe5af21e82", + "7243deb443c38ff4a47a840ae01546c8106de245", + "9f0a8bef3c4839c14751d6223a69b03e4f33fcea", + "3c5bf1e1cefc9f3ecc2cb3b07ea5981933df9fd9", + "658ded3ffea6353492de3f082e61aa489c728fa3", + "3fbfa71e126a95a884f74e9eeff8d189902d378a", + "bd9c319f9f10d9dc3c3d94fd490d92dd4270ba40", + "8cfad82a72b94d1b362c20c5be95fbf677cc31f1", + "3238461b0930ade9172a2a449497af12aad56e1a", + "74d2f66f575246cf6e313ae7775f44df6e097ff2", + "4b2a34740425b4bc124804edd8e37c51d303c135", + "2ee32bc65b7e0d2ca8aa05a6b952817fd87f93f1", + "5fb9ad197f5bf57cf86c5f8df0c061ccb0bcf6a2", + "5f82b635c5da5c3df44d5a0caa8983d9b95e0db9", + "d0dc04d9c315b2a658bcabb3b970994365034c6d", + "51eeca7ec47cf217f8bf296f6d1b2190584d0398", + "13baaa8df7dc7ba1c8e41351b08caacfd1f2244c", + "3a845452177a9910e21ce553a3c53d9f5e5763c0", + "bd9313a6fca4a08b4f0f0763f0079f6a24262726", + "d8b9ad9a680729df98bfa947c7c1538adab02b00", + "f428562bf63be8d5004bc871a6260d91a6a20d7c", + "6e0d69c42ee3b83080a8ec82bf76cb8fe558c24a", + "6e98f4bf810afb9b03d363b4c2c0c0adc1ac4953", + "fcf6e8d66463ddcb23a0abc1ba6fa9732ea595bd", + "60fa96c63967305b9784cbd8f685824281f67bea", + "7b2060bc871878d55adadb012f300fe3f00cac16", + "2479cf769ed5f0481ccf64860a2ab954514c2b59", + "53b7a0716e36654f041e922e1c41843d7a04ff4f", + "ebe60f2fc0f71a215cf0fd7fa848769f446f7fbf", + "c56f4ecc791676a2f8ce8f4e47aaf0b1c5b3763a", + "2c91e9e30567447d038f6bbe37308e9185087ea7", + "db10ad3fa281dfbd5f682a76b808193b8e179752", + "64ad98645b13a5746996af0ac964c19512fe6472", + "f56da4fbb342e7452dfbe9fd42430615833048d0", + "520dcf63a8c6d0f194de3ed4aa72f78fa573721f", + "30815fa6e9132605d8f0e8cfdce855638d2eb553", + "65de939f110068b1761504dcbed341383ab9819b", + "779d8a9e62cf9e5661dc3abeecf62e49ca64c15a", + "0f2d9b5447d3d6e56b41fe69d54e834849d7c932", + "682fc60a5381744863443a719dfbe6b9bac93ab7", + "392e84d63946e6337901baca316c0e106d3bb7ea", + "0f13f51fafe9d2b670252aa2a18993223bffdece", + "a613156836c24bfb1b94ea014273c0b711db6e69", + "991b13e8006d44c66a609ece7a3433f67112f06a", + "faed4dd1957a6b60ecc131ddd6aa41f6fdc3f0bc", + "48dc0add6170c4b6441554449ed21644230d7165", + "a55935e103729fc834780386b77176d6b7b7b285", + "73e84b8fd767310142604b67b67e41f7fea9cc6f", + "e3b9929cdc57149874e2211e6c8934145d245855", + "58aa824437eb20147acdf72dcabb4e271874a243", + "ed5e3f651305dd1ce0d0b5d2d08b97a963634632", + "d1bd7f42fcacd3fe29328b32be361964a915788e", + "f95130bbcbc8917bfec8f8875fa810ad93b08871", + "de6810309a6165547443f9970ee311af0bc7688c", + "edac0d6a533281e3bb2eb729dcd09d1b40023bbe", + "79991c75d1419fcf76e72eddb9b0aea14016e438", + "7b931ef7f9dbb8d2386fc9170895708c0acae95c", + "68acee77cebda300e1f36fb6279f19d53d5b5cda", + "2eb9bb19d43b6285466ca36b92d1324a09e7a207", + "417094b2224ea7cc130997f8d9c36847b2efb302", + "436a89b30cc988fd5505c9cd8f5dd81110a8606b", + "8aea869bd93421128db2b4c9d4fd0b936e2a3a99", + "12d76e891ffe5eda6265c79b438a07f5dce126d3", + "051f98ec6ab2479cd46e405432811d2bb129e4e6", + "3a7c90aeacbd418f2d29d2522027e5621060dbf7", + "38d211f19a4bcb9b8b684cdb3fe1f0459f05b73d", + "9c256b67486b4b30c757730e1b65b2c2d2af935b", + "be6c9580237ec7a4f86c3d98228e4db569d6dca4", + "e977cc67456de8349dcc08143b92ce5608083f67", + "7b05a32346a40789271f401afbbb7fb3310b281f", + "a2ff696ab8f5e0b38823ad3b5cdff72772093c6e", + "5c5b894a67b967484f7c34a6be0d076fda20e246", + "4f7f02ca528212c578fd56379a202d219d01abe0", + "8fd6c915c96073c818bfc42089a95936e80f265c", + "b1a3b7b7c9c8cef5e8c8d44771426e7764b9521e", + "a521ed701f0e58cc5e7effd239b10b360908365a", + "b77c53c85e5734756a793497845735485c9dc933", + "56723d0cf7cd8ff07c5f08bf9bc4575b4e2ed895", + "07f85ad7a273e9f7b761265e4a1edf4ef1d45252", + "fb15752f38f89920dc533e20572e1f6980fbe831", + "eb0561c9c4e2d7d8a334b9d58075aeac8393eec8", + "9379fb71b66a2e1412cab7c5b5a5ff9f04632552", + "d06afc106ce644f2deb5530a36aa7878bb0fe9f7", + "a8305e82e93d03322a704cb09d3caf09d14bad1d", + "c5b5450dab4e2fbef695e16856ec44228688c9ae", + "2b0709d61427694b5c21140021b61a0d402e9c21", + "6615f70c912e4af82f2149ae08a0dfa8e0ed1a2c", + "d1ca0e8c4774c776703106c7bd1efbd22ef44bcf", + "08f77c1486bf2028cea594d13cc1c1a7a3a8ca75", + "f86fc748ae0c1744484609f65abdfd9af59d258d", + "45b4c981e5f3fbb804b711787e4d5ac98f5942a9", + "6e8770f0d1a953d23ed475b47c299cd44a9fe7ee", + "bf41f3ac094a68854f2af8ab92d7752435a6e2af", + "2b77b4130f0de2e3e3846f8c51ece58720b0bad9", + "84a438ed9baa75c499ac51e67f1ee831cbacde57", + "59a3f1806f2535298003582703b4d43d52474999", + "8d7652776678320027c8f13a3af826e28423ae2b", + "3e388f6c6905dcb158fa1969b8a23ce3638bea6e", + "2aa452611d542482c8f854b565b5311aad3542d3", + "115a28148283de8ea6314aa468212541b50906aa", + "bcf9dfa654964e72b32b3ac74a34aaef2b5ec3c0", + "a087ce57288aab8945cdf4aca4ba554eb46fec94", + "69074330666ab52f4386e0ef3fcff622d0d63c93", + "e13e2ea3dbbc9c7815e39c3749b4480514f555a6", + "82e4206fdc6e045f1c6f0f372995a5c943f15237", + "588bc92c33dfad516c730819a763175cc1b1fccf", + "000b9f6baeb60071512616d345c355a5c1ff5de3", + "6e97d0dde7947a2e2d42fc8069c4d2c0d4c8817c", + "097ac91926e57cfdcf05f76144946ebe664123e7", + "7acfded6f6171adcf2466ce83d20c4cb27e74a62", + "4885cffc1b986dfbf29184c7ec7603f2cbb8447d", + "c9257ae75101d9731483ac39d5771f382e6da39c", + "4e6ffe5568d7864f540d99c7d4ea9b9250ed30f2", + "1fcf0271646951e9b89c7303a9c3519fcf4c08ea", + "9f68279ef0e04b184bd133ddb395b3e8b13c41ba", + "699c47fa4b5c9ec4056b59b351761ccb6402072a", + "8438e359ffa7be7ba0b375404b54ce36687241ff", + "92ae1834e8ff26f9d9b15b4afb9ea36556503c2d", + "a8a2bded92d86f6ef4f16e92e303896198dc92b5", + "950bbbca54a0bd723341882fcc354a16568ec4ac", + "c5f469cf2f823889aaa53fbf90a1c41f4f4bb891", + "6054f0fd118073a08752807758629bab61cf258a", + "408d3a678a8b72d6f20e52ca5df34834fb8bc7c6", + "4e0e123f5012a4154bbe4c20194c7c021be69c77", + "4c29ea1bcf0cffc114de0f49c9162fce894f4a4f", + "71d0b9f08ab6bbdd4fe0234b0486c52dd45498e7", + "0974c3717fb71650a25898f4067eaf861d52f93f", + "d79abd10f12bd15f7be6a4b737f3c5d891a245c4", + "816905345646edb682078451ec7e1fc6c549b68b", + "598d6a5005c06d15e74ae621fa93e08dfe5884ff", + "b3b3190d0ea0a8b8e00a9378892f23d3af362b61", + "b6d5842c64f903b63cd61b79212e9c4b2a21a9a5", + "5574d3c433b99861ca3452a60be6cff1060e6a0b", + "f5ba975e5d1f9d862bb1d43621da76c14b6403e8", + "482c1feb328e8e83edf55f6053fc592b09a8b9cd", + "418d5768ee3a81f5f6c711e967f5bb2aa70ed2f6", + "7298334213411e218b7bc15aa99dcaa80e0910ca", + "61362e7b629e018ccf41293eca0b6fe382162e09", + "81a19667ebd56b604c7a158702b270b517fdf0a4", + "92e25e207bddd35facb23ee9d15ded30b753aca0", + "3cdaa69bca7bcc1fae6013095a979a95708dfba4", + "cfbb662982caf238e9006810045d70e1285e33b6", + "7521a1572358a038a77bdc3d4c4c19c8e6f0549c", + "d7cf0ba471fbc278aad355b829b442c77aad7368", + "cf7d262d74679cd874d143a245a7bf3337383dc5", + "176d1f794a0779a5c22f786828602ac4d8063a65", + "37395b3ebf25b2680159526750f9e370b8ed134f", + "8ec31edb38353f95bb9cac14a73d7ee861d22390", + "0afcfd4c3c7edd3c039057cb0ea1d8c58bee6c62", + "b584759103b9fa3d1b2688394bfee2ccb7246094", + "cdec5738cfc2c09b51c44d0958222c63b64f5606", + "56e217b077abb33045bfb2dc88dbb0f3de3f8eac", + "c2191a3fb10641d904107c54e08e34778b4abc01", + "efaec62cd396d3f71a4a522b3a7ae4948e335a12", + "4ef3f49b137ddda009973360e7b5118b535ed919", + "33ec4a7d1f46de9c0ff62c966b332689a983c008", + "41a6d7baa54df23e06b1b7513c9218d12b11cdbe", + "bbecaeeaaf38425d9085d759ded5ad845e74406f", + "613408509f896c68616a2df6c9c6d3fa6662f519", + "e9ec616d2f10323199073af1b25a586f1619ecb9", + "1af4a5ecd0c05f722ca04dfd287d892e11677b5c", + "eb01951c9598f15b2bf35b1c653aedbe87c49f73", + "f02def8ae7d7fcff0deb967f2f11d88e9d099b4d", + "b7fd0d56b2e89e4624ccd81f1dd50a2cc8c2e091", + "0db83dd13da549be953a316ba32826d974eed8c2", + "f69b35c8cff251e82fa33b82acb48c60ea05bf3f", + "ea75b9f21956341d325fdce4e38b6db41603f09c", + "245a86c80a2b2e1515a80a217aa7afdc139a8d9c", + "cf0fc461ac82eb8e9e2ca121a3cb1865533283aa", + "ddf1d2a883276c1575606fb9794df53fa0e0b400", + "31d58a849958079807963ee428d211200538d66e", + "59b4daee8f3db173f6a170b455ed840ac80679da", + "26ded1686b33f7ed2ed5d73f95eceb740b8d4387", + "f8dae005ddb31b54275ec4333adb317902a4f9f5", + "6d3cb50f1d58a0c816ee9c187c0408e6462f1f1c", + "6bbb4283c62e81f7f2788d468bbaa88685a3306b", + "4257362a3e91d29a34ccf1d27f5e5d645c468cd7", + "131f3254e45c071d618b500e06dff22f444dcf48", + "1723e19174c7ef3aa13e3b368eae0f979b26d790", + "1fbeb72867dc079fc4857de481fa1220e8362853", + "1f9128710b2dd8fb396ddf70c2b0cc6b36014271", + "ac7ba305e8216a9aa115bf63f9358915f1306b4f", + "a9cd1ddef78f4bf5476bf0c46be548bcd794ca3f", + "7c50f4f5e5f2799f05c2137453c34cdfd3ee150c", + "32b26bb692e612f1d0264b1ed3ac8a38c9dc4ac7", + "a423e48a52edc03febf258af9a86d383be728842", + "2ef97728fcc962749f5b2f4c20e772d84f342c82", + "b1987eba6fa6f5c40f987504a2fe2aaa1a34bde8", + "84ae82df0065de23110e705e4d09bb06f38bc291", + "a9ff271af5500fcd4b1609b964f80aad8105d4b9", + "4c9225a17aad6cc3e32d747b6960f4bc9697242b", + "3d516202c3259dbd9b4783c26072a0029e405e53", + "5fe2cca931aa907e46855b5f6a1c5863b6b2e762", + "abd97578e14c62c3e7127b70c46153cbd9421ed6", + "ee04eb7c0f1896611fec9f054c82be45b8246a21", + "3b4b717030770423ddb1f9da18e454203d318cc0", + "0b7c72d9a5b2dda6ceeefbbe23220d19a5076414", + "af1e59d8cb5d8eaaebe73a11132cdadad902886f", + "71439735b8e1e17d8bfa5c0ee91aa539c949c6bc", + "317101e03b8fe642589498f4bdb84541ab466062", + "a310095a5e5a0b085b5685300a9d1b757b839cf2", + "28849f5da2a875129ba4cdfd46c6ddb1f750a9a7", + "3305a76b4be86519655b5d27ea24627b3239c9c6", + "2ddb42eb26d6b19c66fcb75a626931bf5f4eb46f", + "32c69eda9af620dc6c3cf293546bd78226fcaf2e", + "c27611fffa3c59dddacc0152ce767207b3773a27", + "145618fb84bf448299e91704c3b82a851bbe87e0", + "6eaf5d4cadaf5fcbca4b9e63cbe8c9f2de95917c", + "0925f99493413ffa512b2f9278d12f0f81d43797", + "40e5c1a4dc420d35698d8d90c1daa6dc2eaff7ab", + "5331019bef5906209c7661f24adefca29e2fe9b4", + "210f14e753c80064ec1bcae72f1d654dd9b0e687", + "0e6e9ad98496ace25ae4b4c3bf1754f718061548", + "3a829c5bde408289cddac5ce23e1dc687cfbb154", + "c78f9148873c2e0f8e6dd9248fe3d0c4d93015f5", + "853424a7406f3c85d8090d697f6964cd5e9ed674", + "b2b01e6c5ca5e9d11c58a4bd7517c106affd53a2", + "593a08c8b28f8dca79230774c2b5158549976303", + "ad7ed8aa8e3acfe3d0e2efa8a0592c1119a54133", + "26aaa4f2533324ffee8f70e2aecffaf44a7a1d4e", + "a06a278fb2599177b063bfc56e1abf0591a2fde9", + "7100c159f552c6262c82954e3626ffeabeb4f421", + "c57c5efb6380da3ebc408113f836aa079d3638ee", + "7f0357977e3bfcf7484ca7032dbb0146d48f3a14", + "0cb2c4eca79737ef1cdc0d00899a519b5b630551", + "db3ebdaab2b66808a2063c4e732003606fdf8adc", + "c0058d9452a45c5192c2b47cf80d71e75c191d78", + "8ebeb45c666f938fec64fb7d0b5b7259c6cbe608", + "1730741cc09f9724a53bb23476ab0aa55828a249", + "cc88e9cc38c44719aebcf16f4333fab0e75ab60c", + "da549f59cd2113ac8d1ca133e80ecedfeb4db6bb", + "a24c4d8320cf5aecfc3797a91f3b9c76a5a7233e", + "83d8d49757658d7a742526ba5c72250244fd57d9", + "2cea69eb79fe358f63223e4a83058981461bbc77", + "73118049bb38a2f854c8c1280188a96839a1663e", + "1729091101160b4f1a78b60f86e2a8f9eb7cc9e3", + "42599e7dfd21a23dc24f18dc650066cd69a6a9cd", + "7eb8c94a909187eb08219891742e8eb60bc265fc", + "d073cb1f08ba1cc193331e6b0778b25be3397fc9", + "4a64f9a844e0af8cab93cda2f1b03bde083008f9", + "74d10cab153ab6d08f9b8c1cfa7f0a3ba78ae15e", + "27d8e086071759e4a642195e06fd952da6de2928", + "ec59d8731a967cb33044ba90d725565cf85cc07f", + "635bccec53fd04a22367ed387295f2ef38a50b79", + "5ff8eb6998ac8fa894f8db5a4f32e2e4fbed72f9", + "4394a3bc03e92c2a7c99f6584b86de85e41a96f1", + "fed2736a78077642a73256045c6e67985a2a8522", + "1d95d26d7c15aaadded91503dd461a034aae16d4", + "9d5437bc483e714468f9fa343e34ea098c40fb6d", + "3900059f7cbd83537475837d9e4a2243853cbb40", + "e22f91d9229bee5cac23b11a212315ad07088927", + "e9ef0092c9e97a415e677e0568f57a76100516bf", + "e12248c0e1996e5c2f1c618fbb47da4f6cb3ff43", + "49bf1a22fd3a906c6dde59819a404095b64834dc", + "aeb2fe1c274be79898e4a0ce1345cd2d260c48ad", + "aa5cce1e6c279364fba5a63d94d48835214dec3d", + "313c0031bbeded98d1bdbf2ed8ff785e73615d38", + "da2c748aad55c1a1dcba00b60fe4a35017bc6540", + "591cea1b074b56367f25ec0084a0c4ab66b42a7e", + "e9fa12b965897f485f8ec143e048a424b52f689a", + "874ffe5839095bd9b181eb465ec20fb473cda7c5", + "febccbdc19e0fc80761797dc52811dc056c0a39e", + "62e38a72a41d072f70ad78baa402d5ca4bb13022", + "460c375ff6889a2ad7eeffbdc246eedf767b6136", + "af24e419f48e543ced58dbca99ab39dc48ab6b11", + "5ab3a2849d68a074ebc70d0ac3aa4525804cea0f", + "486dda64980163cf4ab642b00514fcca169dd148", + "c94044ef540811ec9546606c8f641ce8c7fb7495", + "cc5f7693398ffd9844316ef7c16233f106bee8d7", + "8e50188e183222ffeb44f9729dde056a33dded38", + "953756fcbc88d5c17943faee243bd3efe9a5815c", + "0ec476f32193c33d024ec99d26e2cea0d74168db", + "d746d4625b2cb306f2c7427c5c97efc6d25fcb34", + "f69ab255a58896e066a754aeea1e4381c4e858b0", + "73856a817dcaa91a87cca97f32f47fb0edb8b9a1", + "7f89241892e2385aa1442c2afed93a5542018b45", + "1f3eef621985b2dd0e7e2d3ac265c0f0ba6b0a78", + "841eaea8885f568c5f195f22ff3c8738769f2fe6", + "704d27e3eac9b0ae42e600559f763cacd9a55959", + "9766128dbc8b3b47f154d1112e063183df0f5021", + "619713b3aa1bbaaca174387537a9b089f728f9b9", + "f90201b532678aeadd5933ad31893a5c720ef0a7", + "db5c90a92b6a3218f340af2847934c6549645082", + "5ec48f1be265535c91d4044048bb363fb17893bd", + "cf0af730bf0023484db82d4cc337d2f68541cb6e", + "60837d59cfad2b8adaba1aaf7b459171e4588792", + "14ed3374e4e70ffdbdfef740a6272e92e4a7300c", + "d8aba486f57f013a337f4a48774f161b7cc594ee", + "cd2ab490341f080607792ec86b6c4e61cfb760d3", + "ba8dbd364dd18a16396bf10f722a156241fffcf6", + "3c616d0c3b519cff42ee675fa113bd86c05a77a0", + "4f85a1b7fd2a6dda80872e28ccd1c426ece01b36", + "8f2ee50841e216cd8c192eeb68953118190f105c", + "be39d4eac8653672120843ad9bc715bad8fca05c", + "fdd54cf0d1643a1060147159e1d9ced46402a1a1", + "83a79230dc2a0e30ed0dfc03121a0ae2f9ee6de6", + "5483e909b32272ee294e3518036479498af37902", + "257869d3061d5a8ed1529bf34225d90a2416d51a", + "9e9513b11fff182b08cf8cd6c9c2272460636586", + "d200ddcafb7f7336b0ae0353ea1504153876d6f2", + "fad242d35291de1898242d6737ba02b74a59a2f2", + "d32478ac7c6cb7fdad4dc7a8fdc54f7f171a22fe", + "a8f7249c3c5aef1ed6dfa16e637f13dfde815464", + "fcb566cfb8a86cea2d3666bf65a4f177a49b1313", + "a26cc7479b14cdf70433919bfc228625f1c5c6f2", + "092dedada188f56c5915f74a5fd40aac093742c3", + "a3308ea8d0cc513727c8ad149b1132317a886c98", + "76117d7031e856687c5bd9f5f99c79b94b1316d7", + "9637782beb62abfc9a53a59ab762d39f6ae6fb29", + "51662dfb31fe27ffd77d375397c539f31ed5a648", + "a4ab5378e4cc20b30ef94848cca5700af46449fe", + "3e85179a5601bf322bff7697cfc40d5212249707", + "516bd724d2f4820c822d94af2b3b523dd0a87ae3", + "5db75ff7f6a7c6bb5ae534dd5c618c4e787e7ff3", + "48c78d90a3625cae8716bb20091bda775378b075", + "67e2663dd470950473d6183be345f6d8b1269165", + "aed079ec655046f6a876654d7c36f388682b777f", + "62f33637281ffb45ca8fa91b9411e1cd73e06dab", + "2da539ab426f347c58640a45f5428a80b4ca930f", + "25ca9884a186e07efd9c6ad0c26ab53552db2f11", + "4dd1f01c26e4b24e691075f1b78bcbf90f7f19b1", + "6036065906a8bff074116960cd7dd39db9887b56", + "ba9506eb36a78cd081ea10ffcc543e0f1abd840b", + "e083dfe79605163283d635989662937bd34dbe79", + "055c65edc75d5c6e95b96d2094aadda49b44c965", + "5ba80ae9a40991b8f01c69debd791ffdad5ceb2d", + "969a93e25e377c6990508e8d0bc86aa526812f59", + "bebca33f522165afbdd23f95118715ff6637a731", + "afd733ebdb9260f143953f817a3ce31855ba8812", + "d64ae050c7936eb0a5d00144900f892fc826707c", + "174c264c0a59e84e3c4d2a283e94d6981606a8be", + "4703b76221f1cc7194e11e8af2b9d5959c672a55", + "c59f24f76e02a8d4ecd2aea4309d185d7ed13128", + "5dda3276064d871b8372501271cd62264efb8d89", + "9b171118ea9a3207fe3992d56104582c5fb75be5", + "c51fb9761ebdbf9f6fb46300e047635b1c8afcd3", + "33c8d5f15521969a09ea245c076e541d768b5070", + "0ef5917b3a6742dc69b6758b3e40a7beb8b97d04", + "f8cc276bb0add12debc9eddce79e2a842bac5d3e", + "b5e96eef9e61e53c947e3ba407d4025d71375af9", + "571cc734cd69fb846032ec77a662aa8ee4fc32cd", + "6b38eb50104901e2fec80f9455a972bc3eced813", + "b1048c6b106073c8e51dd61d3104200dfbe7f807", + "4d104d325b96c348d685956c6678139640c15c4b", + "f83f03f59b00d9ed0efc0f372c9a153bcd5dc56d", + "28f7a2cba34c51ebccbc4e99acd4cb7cbe07c71a", + "541214b84dea7f2a94e5a5ca9506c4a0aadcda28", + "26112958664f295cd0ffa28936fc940aa001acda", + "4a329372d9257c59929624e8d05d67124d3b84a3", + "f192f08d8f7b27bbb61ad905764255c83d684cdf", + "0a70a726ce116e2d51c65a2b7282ca2134325759", + "0ecfa6323d4bcc400f1a32c6693ef11691c26448", + "77eaea773f952715656a1734e24f274d41e751a7", + "18d883be86974f1836675db59f277861de57c750", + "4e74daad2f74bd836eec04ff2f4202bee973107c", + "24a12d6f9278a4ab2928475e8ce06a0f59ae73d4", + "59c181fb6e48f7c58e7fe12a1dda10d2183d0e2b", + "cc02f9afca03970a97c2ddea6d36df3b864e7e18", + "e4326a39e36a0b72b15adc95c91671e6a6dcce66", + "1441c354eec2ab2d25319e1d96b33ef347123f47", + "3007f0538458a697749520100901aa80885aeae2", + "de3a4883c3197a29abafab5e87a0a8bead421fbe", + "84355587cb2a1f86e6882de83514a32db2646e7a", + "e9b9c9c34e06822ca1b643d0fdccea3efb4d2899", + "cf064b91ac6d89d5cc6014365d3c0d4753cd6378", + "1d74c6cc47df5cbf75b0b71151c80262459af8d8", + "6cb237c5f1c11c10377ab406385617ee783707ad", + "53eb5671945452618ad3288e6f939717b8b446ef", + "685341c6544a6ed2ce488028184430456ca07005", + "9f916c1709074435f18c6bdc6ee968078cb9cd43", + "9de5f9213b29c2469410048b94f783697734d56e", + "a63b2761ba0b97575c4f964ffc0833afb4d39986", + "4ac10c837490a0d2b68e589643b9c0e36f539297", + "e23bb022327c32995eb415991e7fefb5b26a6ec4", + "d5fe075a50ab3ce4f41614d66ed77f152fda134f", + "d67def6716c4ec2a650e7a8d18264fa22aff2750", + "11886177fe1b02839bd4f6d5f2ac12e03d651a7d", + "beb62aaaad0c1701fbce6f3f21777837a424d737", + "6fd77606f4ead7b81d4aa948a88b4064ebd50b97", + "3de9c413664d81786e789a379e9d8ce9b31402d5", + "bbfae30d95cf72ec632a5914265c3f4ab50a9ec8", + "210dc96d7d514abb943a8b314603397c5577ffff", + "14c162416ebe76d1eb4d542af75f876fc2237c1d", + "07d7b63f09b6685278032e189e9f6d52dccb0939", + "8c702cea303c5efd4348d1b6cad58a5a9b545121", + "63f31133a69fb15322546633a334864d6636834d", + "23187bb333d50fdeddba88a9eb769f4cc6df1127", + "f3139b7e8104dbe3da75ab238b86695c75525a3f", + "f451a2bb3874ca0a286a5f09cd15f7fe6470ddda", + "70cdad0e95bb2366a9b2ae9789c0e377fef6e3ef", + "c65070f9cd162346bb724d7bf4f2fa20cde8f1cf", + "a22a7abf2f6d6dda8e2ad2e41aeaeff9d7b2d02b", + "19bccaf083d56fce700beb7550414231e562b4ea", + "6f45838f3cca3ec8f25b0ee62d1d4259b0e85c2a", + "321873914f47d759dce3fc87bb18f77f8455bf78", + "6e612b12d7fee334726651eb156c57151c64ab7f", + "dfe8b4600c2f80fddba6835b21a6091939882a5e", + "8d0ae097adba4f90a2fad325abc5585ad8b9966e", + "5b5dab9fb89c0ab7187ddc334d8995bd24392e41" + ], + "blocked_refs": [], + "reason": "Since ~2026-10-15 GitHub validates a called reusable workflow against the CALLEE repository's actions.lock at the called SHA and rejects a dependency record whose ref is the bare commit SHA (`Unable to resolve action actions/checkout@3d3c42e5…: invalid lockfile pin`). Every standards commit listed here carries such a record, so every reusable of standards at these revisions dies at `Set up job` for every caller: the run reports `failure` (not startup_failure), the job that dies is the reusable's first one, and the message names an action the caller never wrote.", + "known_good_version": null, + "known_good_sha": "75bd5994470661a099df562d6d5ad2fe744dbd66", + "replacement": "pin_to_known_good", + "evidence": [ + "hyperpolymath/cicd-squabbler run 38846539059 (2026-10-16) — first red, byte-identical lock and workflows to the last green run 60e8f7a (2026-10-15)", + "hyperpolymath/hypatia#929 — five callers re-pinned to 75bd599, jobs created again", + "hyperpolymath/cicd-squabbler#135 — seven callers re-pinned, lock closure rebuilt from standards' own lock at 75bd599", + "standards/scripts/lock-selfcheck.sh rates these SELF-CONSISTENT: it checks that every ref is KEYED, not that a key's ref is a tag — this list was produced by a walk of every commit reachable from standards main (1324 commits, 545 with the rejected record shape, 2026-08-19..2026-10-19)" + ], + "added": "2026-10-19", + "added_by": "cicd-suite#54 follow-through (session 01Xn9nqHVbUTRLgBdedoXAc8)", + "notes": "Keyed on SHA only: standards reusables are pinned by commit, never by tag. The list is a measured window, not a policy about age — a standards commit older than 2026-08-19 has no actions.lock and is NOT in it. Mirrored by gitbot-fleet scripts/lib/stale-reusable-pins.tsv (Window B), which is what fix-stale-reusable-pin.sh rewrites from; regenerate both together when standards fixes or re-breaks a window. 75bd599 is the first commit on standards main whose lock carries no self-SHA action record; the toolchain-input window of June 2026 (rust-ci-reusable only) is WF027 in workflow_audit.ex, not here, because it kills one reusable, not all of them." } ], - "moving_refs_never_lockable": [ "stable", "beta", @@ -56,7 +630,6 @@ "master", "HEAD" ], - "safe_change_classes": { "bump_ci_patch_minor": { "description": "Dependabot/renovate bump of a github-actions dependency within the same major, touching only `uses:` lines.", @@ -64,7 +637,12 @@ "method": "squash", "pool": "P2", "safety": "arm_auto", - "requires": ["author_is_dependency_bot", "semver_patch_or_minor", "no_denylisted_pin", "delta_is_pin_lines_only"] + "requires": [ + "author_is_dependency_bot", + "semver_patch_or_minor", + "no_denylisted_pin", + "delta_is_pin_lines_only" + ] }, "bump_lockfile_only": { "description": "Dependency bump whose diff is confined to lockfiles (Cargo.lock, mix.lock, package-lock.json, go.sum, flake.lock).", @@ -72,7 +650,11 @@ "method": "squash", "pool": "P2", "safety": "arm_auto", - "requires": ["author_is_dependency_bot", "no_denylisted_pin", "all_files_are_lockfiles"] + "requires": [ + "author_is_dependency_bot", + "no_denylisted_pin", + "all_files_are_lockfiles" + ] }, "chore_meta_only": { "description": "Non-code chore: docs, licence headers, linguist exclusions, CODEOWNERS, issue templates, .gitattributes. No runtime path.", @@ -80,7 +662,11 @@ "method": "squash", "pool": "P3", "safety": "arm_auto", - "requires": ["no_denylisted_pin", "no_code_path_change", "no_workflow_semantic_change"] + "requires": [ + "no_denylisted_pin", + "no_code_path_change", + "no_workflow_semantic_change" + ] }, "pin_rollback_denylisted": { "description": "A pin currently on the denylist is replaced by its known-good SHA, in place, with no other edit.", @@ -88,10 +674,12 @@ "method": "squash", "pool": "P1", "safety": "arm_auto", - "requires": ["every_changed_line_is_a_denylisted_pin_site", "replacement_equals_known_good"] + "requires": [ + "every_changed_line_is_a_denylisted_pin_site", + "replacement_equals_known_good" + ] } }, - "never_auto": [ { "id": "NA-001", @@ -124,7 +712,6 @@ "reason": "Automerge is for machine-authored chores. A human PR keeps its human reviewer." } ], - "meta_guard": { "comment": "Change-level is `meta` (and therefore safety=flag) whenever the delta reaches CI, the oracle, or policy. The single exemption is a pure pin substitution — the actuator must re-prove `pin_only` from the diff itself before honouring it.", "meta_paths": [ @@ -136,7 +723,11 @@ ".machine_readable/merge-orchestration/", "stdlib/" ], - "meta_repos": ["standards", "rsr-template-repo", "hypatia"], + "meta_repos": [ + "standards", + "rsr-template-repo", + "hypatia" + ], "exemptions": [ { "id": "MGX-001", @@ -146,7 +737,6 @@ } ] }, - "lockfile_names": [ "Cargo.lock", "mix.lock", @@ -162,7 +752,6 @@ "Gemfile.lock", "composer.lock" ], - "dependency_bots": [ "dependabot[bot]", "renovate[bot]", @@ -170,7 +759,6 @@ "app/dependabot", "app/renovate" ], - "absence_rules": [ { "id": "ABS-001", @@ -197,7 +785,11 @@ "id": "ABS-003", "field": "chrome", "test": "missing_required_file", - "required_files": ["LICENSE", "README.adoc", "SECURITY.adoc"], + "required_files": [ + "LICENSE", + "README.adoc", + "SECURITY.adoc" + ], "severity": "medium", "auto_fixable": "seed_from_template", "why": "Standard project chrome. A seed from rsr-template-repo is unambiguous; a missing licence *choice* is not, so LICENSES/ is seeded but never invented.", @@ -205,7 +797,6 @@ "issue_body_template": "Missing from the repository root: {missing_list}\n\nRSR template chrome is seeded, not invented: `hypatia` can copy the template skeleton, but the licence *choice* and the security *contact* need a human.\n\nFiled by `hypatia` (ABS-003) with the absence-intake run of {date}. Deduplicated by this marker: ``" } ], - "intake_limits": { "comment": "The 2026-09-26 complaint was that issues had become unreadable. These caps are the cure: intake is capped per run, deduplicated by an in-body marker, and upgradeable — a repeat finding comments on the existing issue instead of filing a new one.", "max_new_issues_per_run": 25, @@ -214,7 +805,6 @@ "comment_on_existing_instead_of_new": true, "roll_up_when_class_exceeds": 20 }, - "stats_thresholds": { "comment": "Live thresholds for the private-farm dashboard. A metric crosses into 'recent/live tracking' when it exceeds these; below them it is reported as a headline number only.", "failing_tests_warn": 1, @@ -232,7 +822,6 @@ "open_issue_count_warn": 250, "open_issue_count_critical": 500 }, - "stats_output": { "comment": "Where the dashboard artifact goes. The .git-private-farm repo is not readable or writable from the public estate — this is the contract it consumes.", "artifact_name": "estate-stats.json", diff --git a/lib/rules/sha_bump_propagation.ex b/lib/rules/sha_bump_propagation.ex index a71fd169..c667e044 100644 --- a/lib/rules/sha_bump_propagation.ex +++ b/lib/rules/sha_bump_propagation.ex @@ -70,7 +70,14 @@ defmodule Hypatia.Rules.ShaBumpPropagation do {"hyperpolymath/standards", ".github/workflows/codeql-reusable.yml"}, {"hyperpolymath/standards", ".github/workflows/hypatia-scan-reusable.yml"}, {"hyperpolymath/standards", ".github/workflows/mirror-reusable.yml"}, - {"hyperpolymath/standards", ".github/workflows/changelog-reusable.yml"} + {"hyperpolymath/standards", ".github/workflows/changelog-reusable.yml"}, + # hyperpolymath/cicd-suite (cicd-suite#54): the estate audit and the four + # language gates, consumed as thin callers at a full cicd-suite SHA. + {"hyperpolymath/cicd-suite", ".github/workflows/main-estate-audit.yml"}, + {"hyperpolymath/cicd-suite", ".github/workflows/rust-ci.yml"}, + {"hyperpolymath/cicd-suite", ".github/workflows/creusot-gate.yml"}, + {"hyperpolymath/cicd-suite", ".github/workflows/spark-gate.yml"}, + {"hyperpolymath/cicd-suite", ".github/workflows/kanren-gate.yml"} ] # Composite actions (referenced via `uses: /@`) also flow diff --git a/lib/rules/workflow_audit.ex b/lib/rules/workflow_audit.ex index fd895b71..d1a2684a 100644 --- a/lib/rules/workflow_audit.ex +++ b/lib/rules/workflow_audit.ex @@ -93,6 +93,8 @@ defmodule Hypatia.Rules.WorkflowAudit do heading_regex_issues = check_unanchored_heading_regex(workflow_contents) d_burn_issues = check_d_burn_double_trigger(workflow_contents) secret_history = check_secret_history_coverage(workflow_contents) + toolchain_input_missing = check_sha_pinned_toolchain_without_input(workflow_contents) + broken_reusable_revisions = check_known_broken_reusable_revision(workflow_contents) %{ # `flawed_regexes` was computed at the top of this function and @@ -129,7 +131,9 @@ defmodule Hypatia.Rules.WorkflowAudit do heading_regex_issues ++ d_burn_issues ++ flawed_regexes ++ - secret_history, + secret_history ++ + toolchain_input_missing ++ + broken_reusable_revisions, missing_count: length(missing), secret_history_count: length(secret_history), unpinned_count: length(unpinned), @@ -153,6 +157,8 @@ defmodule Hypatia.Rules.WorkflowAudit do concurrency_missing_count: length(concurrency_missing), heading_regex_issues_count: length(heading_regex_issues), d_burn_issues_count: length(d_burn_issues), + toolchain_input_missing_count: length(toolchain_input_missing), + broken_reusable_revision_count: length(broken_reusable_revisions), workflow_count: length(workflow_files), standard_coverage: coverage_percentage(workflow_files) } @@ -1104,6 +1110,265 @@ defmodule Hypatia.Rules.WorkflowAudit do end) end + # ─── WF026: SHA-pinned dtolnay/rust-toolchain with no toolchain: input ── + + @toolchain_sha_pin ~r/^\s*(?:-\s+)?uses:\s*["']?dtolnay\/rust-toolchain@([0-9a-f]{40})/ + + @doc """ + WF026: a SHA-pinned `dtolnay/rust-toolchain` step that passes no + `toolchain:` input. + + The action reads the toolchain from its `@`-ref (`@stable`, `@1.85.0`). + A full-SHA ref carries no channel, so a SHA-pinned step with no + `with: toolchain:` dies in its first step with + `'toolchain' is a required input` — before any cargo command runs — and + the red reads as a code failure. plasma-parser-writer's + `rust-ci / Cargo check + clippy + fmt` was red this way from 2026-09-29 + to 2026-10-14 (plasma-parser-writer#105); standards' rust-ci-reusable.yml + carried the defect from d73c93e0 to 49ed4715 (standards#439 fixed it). + WF027 covers the CALLER side of that window; this rule is the defect + itself, wherever it is written — a reusable, a plain workflow, a fork. + + Sensitivity / specificity: + * Fires only on `uses: dtolnay/rust-toolchain@<40-hex>` whose step + body (the lines below `uses:` up to the next list item at the same + or shallower indentation) names no `toolchain:` key. A tag-pinned + step carries its channel in the ref and never fires. + * The step body is bounded by indentation, so a later step's + `toolchain:` cannot rescue an earlier step. + + Auto-fixable: gitbot-fleet `scripts/fix-rust-toolchain-input.sh` inserts + `toolchain: stable` (the channel the estate's reusables pass). + """ + def check_sha_pinned_toolchain_without_input(workflow_contents) do + Enum.flat_map(workflow_contents, fn {filename, content} -> + lines = String.split(content, "\n") + + lines + |> Enum.with_index(1) + |> Enum.flat_map(fn {line, number} -> + case Regex.run(@toolchain_sha_pin, line) do + [_full, sha] -> + step_indent = wf026_indent(line) + + body = + lines + |> Enum.drop(number) + |> Enum.take_while(&wf026_step_line?(&1, step_indent)) + + if Enum.any?(body, &Regex.match?(~r/^\s*toolchain:\s*\S/, &1)) do + [] + else + [ + %{ + rule: "WF026", + type: :sha_pinned_toolchain_without_input, + file: filename, + line: number, + current_sha: sha, + severity: :high, + auto_fixable: true, + fix_script: "fix-rust-toolchain-input.sh", + reason: + "`dtolnay/rust-toolchain` is pinned at `@#{sha}` with no `toolchain:` " <> + "input. The action reads the channel from its `@`-ref; a commit SHA " <> + "carries none, so the step exits 1 with \"'toolchain' is a required " <> + "input\" before any cargo command runs and the red reads as a code " <> + "failure. Add `with: toolchain: stable` (or the channel you mean).", + action: :add_toolchain_input + } + ] + end + + _ -> + [] + end + end) + end) + end + + # Indentation width of a line (spaces only; workflow YAML never uses tabs). + defp wf026_indent(line), do: String.length(line) - String.length(String.trim_leading(line, " ")) + + # A line belongs to the step opened at `step_indent` until the next list item + # at that indentation or shallower, or a shallower key. Blank lines and + # comment lines belong to whatever step they sit in. + defp wf026_step_line?(line, step_indent) do + trimmed = String.trim(line) + + cond do + trimmed == "" -> true + String.starts_with?(trimmed, "#") -> true + Regex.match?(~r/^\s*-\s/, line) and wf026_indent(line) <= step_indent -> false + wf026_indent(line) < step_indent -> false + true -> true + end + end + + # ─── WF027: caller pinned at a reusable revision the estate has measured dead ── + + # One entry per (reusable, window). The SHAs are the commits of + # hyperpolymath/standards main whose rust-ci-reusable.yml pins + # dtolnay/rust-toolchain by SHA and passes no `toolchain:` input — a walk of + # every first-parent commit from d73c93e0 (2026-06-01) to 49ed4715 + # (2026-06-24), each one's file content checked, not the dates. The + # replacement is the current standards main whose own actions.lock GitHub + # accepts (see pr-automerge-policy.json PIN-003 for the lock-side window, + # which kills EVERY standards reusable and so lives in the pin denylist + # rather than here). gitbot-fleet scripts/lib/stale-reusable-pins.tsv carries + # the same rows (Window A) for the repair; regenerate both together. + @broken_reusable_revisions [ + %{ + reusable: "hyperpolymath/standards/.github/workflows/rust-ci-reusable.yml", + window: "standards main 2026-06-01..2026-06-24 (d73c93e0..49ed4715)", + fixed_by: "standards#439 (9fe0d8ffa33a6fa0c6ba64117bef9981dacd3020)", + replacement: "75bd5994470661a099df562d6d5ad2fe744dbd66", + reason: + "rust-ci-reusable.yml at this revision pins dtolnay/rust-toolchain by SHA and " <> + "passes no toolchain: input, so every caller fails in its first step with " <> + "\"'toolchain' is a required input\" before any cargo command runs.", + shas: [ + "bf32531befa4bb7f692a5f7ecac1eab71af20506", + "583a9346ebf28e0fd8114f22aef140eaa1be4ac0", + "d7c22711e830e1f383846472f6e9b99debdb201e", + "747b2f0f690505b24c90991a030a01e564ff59c6", + "509142da4ec04d74dccfc9e52813aeabc2c6fa02", + "e2ef79eecdd623ff631fcbf3fe0268ba4f4ce166", + "0597948e81c0be7570f805d7ec49f64fa0eeefd6", + "da1ee5ebaec4de70e7530a9fcb4b2c693434bcc1", + "0f49246bc5055f7159c92d649c4d3fb97b75cf66", + "f15ced3f44146de9931bcb70157e1825d07b83ed", + "5e29ef7923aff017419bdce9354e9df4c3823cb5", + "ad8c45387878f2b00217330b0e1260edc6f878b5", + "3ce22c244bd4b31ed99397a585cd458735a79819", + "8c785d41adbbee534b158747a5c6796402e450f0", + "c1070339c80c4f1af71bb35cacc58c19f611f3a2", + "49ed471515bdf587aeb9931af2d1314cd6aa5de2", + "7282b1ac828d89eaecd3403b59591975746ec3ff", + "d135b05bfc647d0c0fbfedc7e80f37ea50f49236", + "4762ba66441de05bccb36e074ba3d2202219126a", + "59429dc666d347d030114cd6ef6f4ff7b7f6a6b4", + "e8fe153b568af5e3534a261eee0258fa84fa6c5e", + "280cb7e3f2db8b0e5f26c42877ea5fa80243b06f", + "c6c095b38f7e729d3d355629b80e771dfe1a351b", + "09e7023d24682621bea4e11965a1ef5e87d86c3b", + "42455b19727fa0ccf73a9b914f0e9781d4c34577", + "4ddc926b4b86451c2711a16bf927ed16ad2db45e", + "0809b8e3630d38229b6c3629956e0dff4b937c33", + "d72fe5a14e841ac6d78514b53624b6173038ee20", + "5a93d9d57cc04de4002d6d0ecd336fc7a8698910", + "3e57141d38672438a7a9110bd10ea0f76e6099a7", + "78b29005efe954822c86c553b40523b9fdae78d4", + "f290d4a696b8af78f9141daeced486b7b8a85d11", + "a8c69ecfcd87d5418dd4559019ac860fff20e834", + "a04c1d9f32e3c6cb5e91b7ba220080475449a4d2", + "c43aafaa2d707f3f195cad9742d3ccc8d068e3db", + "a955527810b82dbf7a9bc541502c5235f119499d", + "2ff85f8f84212ae2cbedb1ea232c6f5a470acef3", + "8e6ba7d4a0890d25425855a77404d4690d0ab4b5", + "426368b72949d8235c8cf47f8bdc5e0e5cd5b54f", + "025bba3f35f5040bf76d4954f02c744d61f68e8f", + "8466cee8adbb814a2507a9a33e36ba415c5d7eba", + "f5f0506a6ec88e574753eee701a268e0d4b3a7f2", + "ae5b6731e0c1ce192edc14737a226d6b201341aa", + "742424ec9cffeb414174276d9321325859e3aa64", + "19995ace5f1179e9e2d4783fe1d36df0f343492d", + "524523c1a6057942aadaef5999912fbf13a355c5", + "e4c7d968fa9aad6fcecab23bd60d1025679d1ad9", + "e96096c6614469e29e02edcc923fdacc0e611b34", + "ca29b1950ac493e4a29db3b61b6991fbdc8a57b2", + "e59affd85741ac786867a667bc1526ade31b1dd4", + "7694d58cd4c82d6d706ef9653bcf545e28126794", + "33a82ee5e3635bd137b2bc8ef2b1d71367195546", + "52db79bee40380820e01c454efe6e85ae6115b2c", + "6022e498a06552d71e51662283b9453855185b94", + "c97b2bdd12aeaff4624efcaec874ea7665dadae6", + "9726a3814c895bf531f22c2b82cc15a3b1efa1f3", + "bec61613c821fdb1035f18b88e6e5d19d23328f7", + "4236493691a06a6393b5ec4480f38e5d804f5b69", + "6cd3772824e59c8c9affeab66061e25383544242", + "e763d82b93f46e7201d95e6d817ac61d739a8cc1", + "aa665acc04fff66632ede8368dbc9c2b5a630d3c", + "3a5c23cb5888eca79df090fde6d52a8dc4820285", + "0b96f61a3f043bd5409b913e526b29f3fd38a217", + "ae6619ecddd0eff0f957c73280efc36bb9302d75", + "3b3549e293f87be99d7dbb939d380c8128842c8e", + "2b88f34c97a9478b7a96b07425bb37ee5822bb8e", + "9bdaff9479d53328ed907bdd5ba885398800fb00", + "142dd7a0b9cd6101de8c4279c55b7051c537f76b", + "ac6b3252e7170d9459779c03bf31ae04c3629a61", + "f2b9d5eb5fb68a60b30468b8289a0f8cefbdd558", + "e401e47106ce0b0e32e3d0e2b7d910410e5b165e", + "e3ad4ca266e4dc93b7cebce5e3b3b48c6d523e45", + "74748546395bf3dcedd5b660f58feb36be9d53d6", + "fc7abf589b93f9a33b8af74dbfe98ba22707156f", + "7814dc67a489afc39f0289af5d3b6c3e23645156", + "c98915a225c98f2b90980cf045a00eb603e6af0f", + "18dd92c19d5b9e50740826a4be8301afbab91bae", + "82a8ac8e2fd446b710139a759bf22f175dc5e299", + "55da6b331922f0ff7389c783d363ae12d8c61068", + "1376eb6251fa9beeaea75241739b1d00dd6b4028", + "28946811fb7fe2d05e9ac11c9ebee7295e966d40", + "d7c3b8f75d70ab640e6f37a4c8f9e6e8f2b6df56", + "d73c93e0e7d38f673b68e616d51a62f5860081fd" + ] + } + ] + + @doc "The (reusable, window) table WF027 fires on." + def broken_reusable_revisions, do: @broken_reusable_revisions + + @doc """ + WF027: a caller pins an estate reusable workflow at a revision the estate + has MEASURED dead for that reusable. + + Unlike WF016 (a SHA no branch reaches any more), these commits are + reachable and resolve: the run starts, the reusable's first step fails, + and the failure names something the caller never wrote. The table is + data, keyed by reusable path, so a newly measured window is a row, not a + rule. Full 40-hex pins only — that is the only form sha_pinning_required + lets a caller write. + + Auto-fixable: gitbot-fleet `scripts/fix-stale-reusable-pin.sh` rewrites + the pin to the replacement the table names, never to "newest". + """ + def check_known_broken_reusable_revision(workflow_contents) do + Enum.flat_map(workflow_contents, fn {filename, content} -> + stripped = strip_comments(content) + + Enum.flat_map(@broken_reusable_revisions, fn rev -> + ref_re = ~r/uses:\s*["']?#{Regex.escape(rev.reusable)}@([0-9a-f]{40})/ + + Regex.scan(ref_re, stripped) + |> Enum.flat_map(fn [_full, sha] -> + if sha in rev.shas do + [ + %{ + rule: "WF027", + type: :known_broken_reusable_revision, + file: filename, + reusable: rev.reusable, + current_sha: sha, + replacement_sha: rev.replacement, + severity: :critical, + auto_fixable: true, + fix_script: "fix-stale-reusable-pin.sh", + reason: + "`#{rev.reusable}` is pinned at `@#{sha}`, inside #{rev.window}: " <> + rev.reason <> + " Fixed by #{rev.fixed_by}; move the pin to " <> + "`@#{rev.replacement}`.", + action: :rewrite_stale_reusable_pin + } + ] + else + [] + end + end) + end) + end) + end + # ─── WF017: Secret-consuming action without secret-presence gate ────── # Actions whose primary input is a secret and that fail outright when diff --git a/test/rules/pin_integrity_test.exs b/test/rules/pin_integrity_test.exs index 59c26da7..e48a7273 100644 --- a/test/rules/pin_integrity_test.exs +++ b/test/rules/pin_integrity_test.exs @@ -421,4 +421,40 @@ defmodule Hypatia.Rules.PinIntegrityTest do assert PI.pi005_stale_pin(content, %{}) == [] end end + + describe "PIN-003 in the committed policy (standards revisions whose lock GitHub rejects)" do + @policy_path Path.expand( + "../../.machine_readable/merge-orchestration/pr-automerge-policy.json", + __DIR__ + ) + @rejected "8f2ee50841e216cd8c192eeb68953118190f105c" + @accepted "75bd5994470661a099df562d6d5ad2fe744dbd66" + @governance "hyperpolymath/standards/.github/workflows/governance-reusable.yml" + + defp committed_policy, do: :json.decode(File.read!(@policy_path)) + + test "the entry exists, is keyed on SHA only, and names the known-good revision" do + policy = committed_policy() + entry = Enum.find(policy["pin_denylist"], &(&1["id"] == "PIN-003")) + assert entry["action"] == "hyperpolymath/standards" + assert entry["severity"] == "critical" + assert entry["blocked_versions"] == [] + assert entry["blocked_refs"] == [] + assert entry["known_good_sha"] == @accepted + assert entry["replacement"] == "pin_to_known_good" + assert length(entry["blocked_shas"]) > 500 + assert Enum.all?(entry["blocked_shas"], &Regex.match?(~r/^[0-9a-f]{40}$/, &1)) + refute @accepted in entry["blocked_shas"] + end + + test "a caller pinned inside the window is denylisted through the reusable path; the replacement is not" do + policy = committed_policy() + assert %{"id" => "PIN-003"} = PI.denylisted?(policy, @governance, @rejected) + assert PI.denylisted?(policy, @governance, @accepted) == nil + # The June toolchain window is WF027's, not this entry's: a June revision + # has no lock and must not be denylisted here. + assert PI.denylisted?(policy, @governance, "d135b05bfc647d0c0fbfedc7e80f37ea50f49236") == + nil + end + end end diff --git a/test/rules/sha_bump_propagation_test.exs b/test/rules/sha_bump_propagation_test.exs index 13506113..60113480 100644 --- a/test/rules/sha_bump_propagation_test.exs +++ b/test/rules/sha_bump_propagation_test.exs @@ -27,6 +27,33 @@ defmodule Hypatia.Rules.ShaBumpPropagationTest do ) end + describe "check/1 — cicd-suite reusables (cicd-suite#54)" do + test "a change to a cicd-suite gate fires, with the gate's path" do + [finding] = + ShaBumpPropagation.check( + event(%{ + source_repo: "hyperpolymath/cicd-suite", + files_changed: [".github/workflows/rust-ci.yml", "actions/rust-ci-check/check.sh"], + pr_title: "feat(gates): strict Rust CI" + }) + ) + + assert finding.source_repo == "hyperpolymath/cicd-suite" + assert finding.source_workflow == ".github/workflows/rust-ci.yml" + end + + test "every cicd-suite gate and the estate audit are registered" do + paths = + ShaBumpPropagation.known_reusables() + |> Enum.filter(fn {repo, _} -> repo == "hyperpolymath/cicd-suite" end) + |> Enum.map(fn {_, path} -> path end) + + for gate <- ~w(main-estate-audit rust-ci creusot-gate spark-gate kanren-gate) do + assert ".github/workflows/#{gate}.yml" in paths + end + end + end + describe "check/1 — sensitivity" do test "fires on a known reusable workflow change" do [finding] = ShaBumpPropagation.check(event(%{})) diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs new file mode 100644 index 00000000..901ee040 --- /dev/null +++ b/test/rules/workflow_audit_pins_test.exs @@ -0,0 +1,168 @@ +# SPDX-License-Identifier: MPL-2.0 + +defmodule Hypatia.Rules.WorkflowAuditPinsTest do + use ExUnit.Case, async: true + alias Hypatia.Rules.WorkflowAudit + + # The two anchors of the 2026-09-29..10-14 incident (plasma-parser-writer#105): + # the SHA standards' rust-ci-reusable.yml pinned dtolnay/rust-toolchain at, + # and the standards revision plasma was pinned to. + @toolchain_sha "6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772" + @broken_standards "d135b05bfc647d0c0fbfedc7e80f37ea50f49236" + @good_standards "75bd5994470661a099df562d6d5ad2fe744dbd66" + @rust_ci "hyperpolymath/standards/.github/workflows/rust-ci-reusable.yml" + + describe "check_sha_pinned_toolchain_without_input/1 (WF026)" do + test "fires on a SHA-pinned step with no with: block at all" do + wf = """ + jobs: + check: + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + - run: cargo check --locked + """ + + [f] = WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + assert f.rule == "WF026" + assert f.severity == :high + assert f.file == "rust.yml" + assert f.line == 5 + assert f.current_sha == @toolchain_sha + assert f.auto_fixable + assert f.fix_script == "fix-rust-toolchain-input.sh" + assert f.reason =~ "'toolchain' is a required input" + end + + test "fires when the with: block names components but no toolchain" do + wf = """ + jobs: + check: + steps: + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@#{@toolchain_sha} # stable + with: + components: clippy, rustfmt + - run: cargo check + """ + + assert [%{rule: "WF026", line: 5}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "silent when the step passes a toolchain" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + toolchain: stable + components: clippy + """ + + assert WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) == [] + end + + test "silent on a tag-pinned step: the ref carries the channel" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + - uses: dtolnay/rust-toolchain@1.85.0 + """ + + assert WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) == [] + end + + test "a later step's toolchain: does not rescue an earlier bare step" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + toolchain: stable + """ + + assert [%{line: 4}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "a shallower key ends the step: a job-level toolchain env is not an input" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + env: + toolchain: stable + """ + + assert [%{line: 4}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "the audit carries WF026 findings and a count" do + wf = "jobs:\n a:\n steps:\n - uses: dtolnay/rust-toolchain@#{@toolchain_sha}\n" + result = WorkflowAudit.audit(["rust.yml"], %{"rust.yml" => wf}) + assert Enum.any?(result.findings, &(&1[:rule] == "WF026")) + assert result.toolchain_input_missing_count == 1 + end + end + + describe "check_known_broken_reusable_revision/1 (WF027)" do + test "fires on rust-ci-reusable at a revision inside the toolchain window" do + wf = "jobs:\n rust-ci:\n uses: #{@rust_ci}@#{@broken_standards}\n" + [f] = WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => wf}) + assert f.rule == "WF027" + assert f.severity == :critical + assert f.reusable == @rust_ci + assert f.current_sha == @broken_standards + assert f.replacement_sha == @good_standards + assert f.auto_fixable + assert f.fix_script == "fix-stale-reusable-pin.sh" + assert f.reason =~ "standards#439" + end + + test "silent at the replacement revision" do + wf = "jobs:\n rust-ci:\n uses: #{@rust_ci}@#{@good_standards}\n" + assert WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => wf}) == [] + end + + test "silent for another reusable at the same revision: the window is per reusable" do + wf = + "jobs:\n mirror:\n uses: hyperpolymath/standards/.github/workflows/mirror-reusable.yml@#{@broken_standards}\n" + + assert WorkflowAudit.check_known_broken_reusable_revision(%{"mirror.yml" => wf}) == [] + end + + test "a commented-out pin does not fire" do + wf = + "jobs:\n rust-ci:\n # uses: #{@rust_ci}@#{@broken_standards}\n uses: #{@rust_ci}@#{@good_standards}\n" + + assert WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => wf}) == [] + end + + test "the table is the measured window: 83 revisions, edges included, replacement excluded" do + [row] = WorkflowAudit.broken_reusable_revisions() + assert row.reusable == @rust_ci + assert length(row.shas) == 83 + assert "d73c93e0e7d38f673b68e616d51a62f5860081fd" in row.shas + assert "49ed4715" |> then(&Enum.any?(row.shas, fn s -> String.starts_with?(s, &1) end)) + refute row.replacement in row.shas + assert Enum.all?(row.shas, &Regex.match?(~r/^[0-9a-f]{40}$/, &1)) + end + + test "the audit carries WF027 findings and a count" do + wf = "jobs:\n rust-ci:\n uses: #{@rust_ci}@#{@broken_standards}\n" + result = WorkflowAudit.audit(["rust-ci.yml"], %{"rust-ci.yml" => wf}) + assert Enum.any?(result.findings, &(&1[:rule] == "WF027")) + assert result.broken_reusable_revision_count == 1 + end + end +end From 59fdd36d5dc8727a992d439023f30b63df99cb79 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 16:27:28 +0000 Subject: [PATCH 10/47] rules(WF026/WF027): inspect the whole step, with:-scoped, non-blocking severity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review findings (CodeRabbit, Codex) on the two new workflow-audit rules: * WF026 now reads the whole list item the `uses:` belongs to — from its opening `- ` line, so a `with:` written above `uses:` counts — and only the step's `with:` mapping decides, in block form (a direct child) or flow form (`with: { toolchain: stable }`). `toolchain:` under `env:` is not an action input and no longer rescues a step. * WF026 and WF027 report at medium, not high/critical: AGENTIC marks these rules must-not-promote-to-blocking, and the hypatia-scan gate refuses high and critical, so the old severities would have turned every stale pin in the estate into a required-check failure. * WF027's match is anchored to a `uses:` key at the start of a line (quoted or bare); the reusable's path quoted in a string or comment is not a pin. * PrAutomerge.parse_pin/1 accepts a quoted `"uses"` key and a quoted value. * pin_integrity_test uses Jason, not OTP 27's `:json`. * estate-pin-integrity.sh `--via search` searches a window-sized denylist entry (PIN-003: 545 SHAs) by its action path — one query — instead of one query and a two-second sleep per SHA; phase 2 still confirms every hit against the full token alternation. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- lib/rules/pr_automerge.ex | 3 +- lib/rules/workflow_audit.ex | 106 ++++++++++++++++++++---- scripts/sweeps/estate-pin-integrity.sh | 13 ++- test/rules/pin_integrity_test.exs | 2 +- test/rules/workflow_audit_pins_test.exs | 4 +- 5 files changed, 107 insertions(+), 21 deletions(-) diff --git a/lib/rules/pr_automerge.ex b/lib/rules/pr_automerge.ex index 0056c1fd..bb287308 100644 --- a/lib/rules/pr_automerge.ex +++ b/lib/rules/pr_automerge.ex @@ -456,7 +456,8 @@ defmodule Hypatia.Rules.PrAutomerge do # - - uses: actions/checkout@v4.1.7 # so the marker is part of the match. (Without this, every real diff looks # like a non-pin change and every PR falls through to `flag`.) - @pin_re ~r/^[-+]?\s*-?\s*uses:\s*(?[A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@(?[^\s#]+)/ + # The key and the value may each be quoted (`"uses": "owner/repo@sha"`). + @pin_re ~r/^[-+]?\s*-?\s*["']?uses["']?\s*:\s*["']?(?[A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@(?[^\s#"']+)/ @doc "Parse a `uses:` line into `%{action, base, ref}`; `nil` when it is not one." def parse_pin(line) do diff --git a/lib/rules/workflow_audit.ex b/lib/rules/workflow_audit.ex index d1a2684a..70df95c2 100644 --- a/lib/rules/workflow_audit.ex +++ b/lib/rules/workflow_audit.ex @@ -1130,12 +1130,18 @@ defmodule Hypatia.Rules.WorkflowAudit do itself, wherever it is written — a reusable, a plain workflow, a fork. Sensitivity / specificity: - * Fires only on `uses: dtolnay/rust-toolchain@<40-hex>` whose step - body (the lines below `uses:` up to the next list item at the same - or shallower indentation) names no `toolchain:` key. A tag-pinned + * Fires only on `uses: dtolnay/rust-toolchain@<40-hex>` whose step — + the whole list item, from its opening `- ` line, so a `with:` written + above `uses:` counts (YAML mapping keys carry no order) — has no + `with:` mapping naming `toolchain:`, in block form (a direct child of + `with:`) or flow form (`with: { toolchain: stable }`). A tag-pinned step carries its channel in the ref and never fires. - * The step body is bounded by indentation, so a later step's - `toolchain:` cannot rescue an earlier step. + * Only `with:` decides: `toolchain:` under `env:` or any other key is + not an action input and does not rescue the step. + * The step is bounded by indentation, so a later step's `toolchain:` + cannot rescue an earlier step. + * Severity is medium: the rule reports and routes to the fleet; it is + never a blocking gate (AGENTIC `must-not-promote-to-blocking`). Auto-fixable: gitbot-fleet `scripts/fix-rust-toolchain-input.sh` inserts `toolchain: stable` (the channel the estate's reusables pass). @@ -1149,14 +1155,7 @@ defmodule Hypatia.Rules.WorkflowAudit do |> Enum.flat_map(fn {line, number} -> case Regex.run(@toolchain_sha_pin, line) do [_full, sha] -> - step_indent = wf026_indent(line) - - body = - lines - |> Enum.drop(number) - |> Enum.take_while(&wf026_step_line?(&1, step_indent)) - - if Enum.any?(body, &Regex.match?(~r/^\s*toolchain:\s*\S/, &1)) do + if wf026_with_names_toolchain?(wf026_step(lines, number)) do [] else [ @@ -1166,7 +1165,7 @@ defmodule Hypatia.Rules.WorkflowAudit do file: filename, line: number, current_sha: sha, - severity: :high, + severity: :medium, auto_fixable: true, fix_script: "fix-rust-toolchain-input.sh", reason: @@ -1205,6 +1204,75 @@ defmodule Hypatia.Rules.WorkflowAudit do end end + # The whole list item the `uses:` line at `number` (1-based) belongs to: + # from its opening `- ` line — which may sit above `uses:`, since YAML + # mapping keys carry no order — to the next list item at that indentation + # or shallower, or a shallower key. + defp wf026_step(lines, number) do + uses_line = Enum.at(lines, number - 1) + uses_indent = wf026_indent(uses_line) + + {opener, step_indent} = + if Regex.match?(~r/^\s*-\s/, uses_line) do + {number, uses_indent} + else + lines + |> Enum.take(number - 1) + |> Enum.with_index(1) + |> Enum.reverse() + |> Enum.find(fn {l, _} -> + Regex.match?(~r/^\s*-\s/, l) and wf026_indent(l) < uses_indent + end) + |> case do + {l, i} -> {i, wf026_indent(l)} + nil -> {number, uses_indent} + end + end + + rest = + lines + |> Enum.drop(opener) + |> Enum.take_while(&wf026_step_line?(&1, step_indent)) + + [Enum.at(lines, opener - 1) | rest] + end + + # True when the step's `with:` mapping names `toolchain:` — block form (a + # direct child of `with:`) or flow form (`with: { toolchain: stable }`). + # `toolchain:` under `env:` or any other key is not an action input. + defp wf026_with_names_toolchain?(step) do + step + |> Enum.with_index() + |> Enum.any?(fn {line, i} -> + cond do + Regex.match?(~r/^\s*(?:-\s+)?with:\s*\{/, line) -> + Regex.match?(~r/[{,]\s*["']?toolchain["']?\s*:\s*[^\s,}]/, line) + + Regex.match?(~r/^\s*(?:-\s+)?with:\s*(?:#.*)?$/, line) -> + with_indent = wf026_key_indent(line) + + step + |> Enum.drop(i + 1) + |> Enum.take_while(fn l -> + trimmed = String.trim(l) + trimmed == "" or String.starts_with?(trimmed, "#") or wf026_indent(l) > with_indent + end) + |> Enum.any?(&Regex.match?(~r/^\s*["']?toolchain["']?\s*:\s*\S/, &1)) + + true -> + false + end + end) + end + + # Column of a key, with a list dash counted as indentation (`- with:` is + # the column of `with`). + defp wf026_key_indent(line) do + line + |> String.replace(~r/^(\s*)-\s/, "\\1 ") + |> wf026_indent() + end + # ─── WF027: caller pinned at a reusable revision the estate has measured dead ── # One entry per (reusable, window). The SHAs are the commits of @@ -1331,13 +1399,19 @@ defmodule Hypatia.Rules.WorkflowAudit do Auto-fixable: gitbot-fleet `scripts/fix-stale-reusable-pin.sh` rewrites the pin to the replacement the table names, never to "newest". + + Severity is medium: like WF026 the rule reports and routes to the fleet; + it is never a blocking gate (AGENTIC `must-not-promote-to-blocking`). """ def check_known_broken_reusable_revision(workflow_contents) do Enum.flat_map(workflow_contents, fn {filename, content} -> stripped = strip_comments(content) Enum.flat_map(@broken_reusable_revisions, fn rev -> - ref_re = ~r/uses:\s*["']?#{Regex.escape(rev.reusable)}@([0-9a-f]{40})/ + # Anchored to a `uses:` key at the start of a line (quoted or not): + # the reusable's path inside a string value or a comment is not a pin. + ref_re = + ~r/^\s*(?:-\s+)?["']?uses["']?\s*:\s*["']?#{Regex.escape(rev.reusable)}@([0-9a-f]{40})/m Regex.scan(ref_re, stripped) |> Enum.flat_map(fn [_full, sha] -> @@ -1350,7 +1424,7 @@ defmodule Hypatia.Rules.WorkflowAudit do reusable: rev.reusable, current_sha: sha, replacement_sha: rev.replacement, - severity: :critical, + severity: :medium, auto_fixable: true, fix_script: "fix-stale-reusable-pin.sh", reason: diff --git a/scripts/sweeps/estate-pin-integrity.sh b/scripts/sweeps/estate-pin-integrity.sh index 727f9178..b2e855c9 100755 --- a/scripts/sweeps/estate-pin-integrity.sh +++ b/scripts/sweeps/estate-pin-integrity.sh @@ -140,9 +140,20 @@ log "policy $(jq -r '.version' "$POLICY") · blocked SHAs ${#DENY_SHAS[@]} · bl HITS="$OUT_DIR/hits.tsv" : > "$HITS" +# Search mode is reconnaissance: one code-search query per token, two seconds +# apart. An entry that blocks a window of commits (PIN-003 names 545) would +# cost one query per SHA — ~18 minutes of sleeps for one action — so such an +# entry is searched by its action path instead (one query), and phase 2 +# confirms each hit against the full token alternation. Small denylists keep +# the exact per-SHA search. +mapfile -t SEARCH_TOKENS < <(jq -r ' + .pin_denylist[] + | (if ((.blocked_shas // []) | length) > 8 then [.action] else (.blocked_shas // []) end)[], + (.blocked_versions // [])[]' "$POLICY" | sort -u) + if [ "$VIA" = "search" ]; then for org in "${ORGS[@]}"; do - for token in "${DENY_SHAS[@]}" "${DENY_VERSIONS[@]}"; do + for token in "${SEARCH_TOKENS[@]}"; do page=1 while :; do n=$(gh api "search/code?q=${token}+org:${org}&per_page=100&page=${page}" \ diff --git a/test/rules/pin_integrity_test.exs b/test/rules/pin_integrity_test.exs index e48a7273..94768a7e 100644 --- a/test/rules/pin_integrity_test.exs +++ b/test/rules/pin_integrity_test.exs @@ -431,7 +431,7 @@ defmodule Hypatia.Rules.PinIntegrityTest do @accepted "75bd5994470661a099df562d6d5ad2fe744dbd66" @governance "hyperpolymath/standards/.github/workflows/governance-reusable.yml" - defp committed_policy, do: :json.decode(File.read!(@policy_path)) + defp committed_policy, do: Jason.decode!(File.read!(@policy_path)) test "the entry exists, is keyed on SHA only, and names the known-good revision" do policy = committed_policy() diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs index 901ee040..454e3f7c 100644 --- a/test/rules/workflow_audit_pins_test.exs +++ b/test/rules/workflow_audit_pins_test.exs @@ -25,7 +25,7 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do [f] = WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) assert f.rule == "WF026" - assert f.severity == :high + assert f.severity == :medium assert f.file == "rust.yml" assert f.line == 5 assert f.current_sha == @toolchain_sha @@ -120,7 +120,7 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do wf = "jobs:\n rust-ci:\n uses: #{@rust_ci}@#{@broken_standards}\n" [f] = WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => wf}) assert f.rule == "WF027" - assert f.severity == :critical + assert f.severity == :medium assert f.reusable == @rust_ci assert f.current_sha == @broken_standards assert f.replacement_sha == @good_standards From 73f8c384085dac1316c95599d4d78c74d41d907e Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 16:32:52 +0000 Subject: [PATCH 11/47] test(WF026/WF027): cover with: above uses:, flow with:, env.toolchain, anchored pins Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- test/rules/workflow_audit_pins_test.exs | 82 +++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs index 454e3f7c..0061006c 100644 --- a/test/rules/workflow_audit_pins_test.exs +++ b/test/rules/workflow_audit_pins_test.exs @@ -107,6 +107,64 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) end + test "silent when with: is written above uses: (YAML keys carry no order)" do + wf = """ + jobs: + check: + steps: + - with: + toolchain: stable + uses: dtolnay/rust-toolchain@#{@toolchain_sha} + """ + + assert [] = WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "silent on a flow with: that names a toolchain" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: { toolchain: stable, components: clippy } + """ + + assert [] = WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "fires on a flow with: without a toolchain" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: { components: clippy } + """ + + assert [%{rule: "WF026", line: 4}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "env.toolchain is not an input: the step fires whether env: sits below or above uses:" do + wf = """ + jobs: + check: + steps: + - name: toolchain via env only + uses: dtolnay/rust-toolchain@#{@toolchain_sha} + env: + toolchain: stable + - env: + toolchain: beta + uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + components: clippy + """ + + assert [%{line: 5}, %{line: 10}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + test "the audit carries WF026 findings and a count" do wf = "jobs:\n a:\n steps:\n - uses: dtolnay/rust-toolchain@#{@toolchain_sha}\n" result = WorkflowAudit.audit(["rust.yml"], %{"rust.yml" => wf}) @@ -158,6 +216,30 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do assert Enum.all?(row.shas, &Regex.match?(~r/^[0-9a-f]{40}$/, &1)) end + test "the reusable's path inside a run: script or a string value is not a pin" do + wf = """ + jobs: + note: + steps: + - run: echo "uses: #{@rust_ci}@#{@broken_standards}" + - name: "uses: #{@rust_ci}@#{@broken_standards}" + run: true + """ + + assert [] = WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => wf}) + end + + test "a quoted uses: key with a quoted value still fires" do + wf = """ + jobs: + rust: + "uses": "#{@rust_ci}@#{@broken_standards}" + """ + + assert [%{rule: "WF027", current_sha: @broken_standards}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => wf}) + end + test "the audit carries WF027 findings and a count" do wf = "jobs:\n rust-ci:\n uses: #{@rust_ci}@#{@broken_standards}\n" result = WorkflowAudit.audit(["rust-ci.yml"], %{"rust-ci.yml" => wf}) From bdadab035abd8257bfd5ddf66bd85b616131455b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 16:37:24 +0000 Subject: [PATCH 12/47] test(PA001): a quoted uses: key and value still reach the denylist Also shortens two WF026/WF027 test names past the line limit. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- test/rules/pr_automerge_test.exs | 13 +++++++++++++ test/rules/workflow_audit_pins_test.exs | 4 ++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/test/rules/pr_automerge_test.exs b/test/rules/pr_automerge_test.exs index acd3ad97..cd7dc93b 100644 --- a/test/rules/pr_automerge_test.exs +++ b/test/rules/pr_automerge_test.exs @@ -162,6 +162,19 @@ defmodule Hypatia.Rules.PrAutomergeTest do assert decision.pool == "P1" end + test "a quoted uses: key and a quoted value are still a pin PA001 sees" do + patch = """ + @@ -1,1 +1,1 @@ + - - "uses": "github/codeql-action/init@#{@good}" + + - "uses": "github/codeql-action/init@#{@poison}" + """ + + decision = PA.classify(pr([file(".github/workflows/codeql.yml", patch)]), @policy) + + assert decision.disposition == :close_poison_only + assert decision.blocked_by == "introduces_denylisted_pin" + end + test "poison alongside a wanted update is excised, then merged" do patch = """ @@ -1,4 +1,4 @@ diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs index 0061006c..7bc68622 100644 --- a/test/rules/workflow_audit_pins_test.exs +++ b/test/rules/workflow_audit_pins_test.exs @@ -145,7 +145,7 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) end - test "env.toolchain is not an input: the step fires whether env: sits below or above uses:" do + test "env.toolchain is not an input, below or above uses:" do wf = """ jobs: check: @@ -216,7 +216,7 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do assert Enum.all?(row.shas, &Regex.match?(~r/^[0-9a-f]{40}$/, &1)) end - test "the reusable's path inside a run: script or a string value is not a pin" do + test "the path inside a run: script or a string value is not a pin" do wf = """ jobs: note: From eba2eb224746676777dbad1cc804c563a86a9cbc Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 17:06:44 +0000 Subject: [PATCH 13/47] rules(WF026): step-key scoping, quoted uses: keys, bare-dash openers - `"uses":` is the same key to GitHub: the pin regex accepts a quoted key, as WF027 and PrAutomerge.parse_pin/1 already do. - Only a `uses:` at the step's key column is the step's action. The same text nested under `env:` or inside a `run: |` script that writes a fixture is data: no finding, so the fleet is never sent to rewrite script text. wf026_step/2 now returns the step and the column of its keys, and the with: check only honours a `with:` at that column. - A bare `-` on its own line opens a step; a `with:` written under it and above `uses:` rescues the step instead of being discarded. Tests: a quoted key fires; run: | and env: copies are silent; a bare-dash step with with: above uses: is silent and a bare-dash bare uses: fires. Mirrors gitbot-fleet 9838040 (the repair side of the same cases). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- lib/rules/workflow_audit.ex | 126 ++++++++++++++++-------- test/rules/workflow_audit_pins_test.exs | 47 +++++++++ 2 files changed, 132 insertions(+), 41 deletions(-) diff --git a/lib/rules/workflow_audit.ex b/lib/rules/workflow_audit.ex index 70df95c2..c416fa60 100644 --- a/lib/rules/workflow_audit.ex +++ b/lib/rules/workflow_audit.ex @@ -1112,7 +1112,7 @@ defmodule Hypatia.Rules.WorkflowAudit do # ─── WF026: SHA-pinned dtolnay/rust-toolchain with no toolchain: input ── - @toolchain_sha_pin ~r/^\s*(?:-\s+)?uses:\s*["']?dtolnay\/rust-toolchain@([0-9a-f]{40})/ + @toolchain_sha_pin ~r/^\s*(?:-\s+)?["']?uses["']?\s*:\s*["']?dtolnay\/rust-toolchain@([0-9a-f]{40})/ @doc """ WF026: a SHA-pinned `dtolnay/rust-toolchain` step that passes no @@ -1131,11 +1131,17 @@ defmodule Hypatia.Rules.WorkflowAudit do Sensitivity / specificity: * Fires only on `uses: dtolnay/rust-toolchain@<40-hex>` whose step — - the whole list item, from its opening `- ` line, so a `with:` written - above `uses:` counts (YAML mapping keys carry no order) — has no - `with:` mapping naming `toolchain:`, in block form (a direct child of - `with:`) or flow form (`with: { toolchain: stable }`). A tag-pinned - step carries its channel in the ref and never fires. + the whole list item, from its opening `- ` line (a bare `-` on its own + line opens a step too), so a `with:` written above `uses:` counts + (YAML mapping keys carry no order) — has no `with:` mapping naming + `toolchain:`, in block form (a direct child of `with:`) or flow form + (`with: { toolchain: stable }`). A tag-pinned step carries its channel + in the ref and never fires. Keys may be quoted (`"uses":`): the same + key to GitHub, the same key here. + * Only a `uses:` that is a KEY OF THE STEP MAPPING is the step's action. + The same text nested under `env:`, inside a `run: |` script that + writes a fixture, or under any other key is data: no finding, so the + fleet is never sent to rewrite script text. * Only `with:` decides: `toolchain:` under `env:` or any other key is not an action input and does not rescue the step. * The step is bounded by indentation, so a later step's `toolchain:` @@ -1155,28 +1161,37 @@ defmodule Hypatia.Rules.WorkflowAudit do |> Enum.flat_map(fn {line, number} -> case Regex.run(@toolchain_sha_pin, line) do [_full, sha] -> - if wf026_with_names_toolchain?(wf026_step(lines, number)) do - [] - else - [ - %{ - rule: "WF026", - type: :sha_pinned_toolchain_without_input, - file: filename, - line: number, - current_sha: sha, - severity: :medium, - auto_fixable: true, - fix_script: "fix-rust-toolchain-input.sh", - reason: - "`dtolnay/rust-toolchain` is pinned at `@#{sha}` with no `toolchain:` " <> - "input. The action reads the channel from its `@`-ref; a commit SHA " <> - "carries none, so the step exits 1 with \"'toolchain' is a required " <> - "input\" before any cargo command runs and the red reads as a code " <> - "failure. Add `with: toolchain: stable` (or the channel you mean).", - action: :add_toolchain_input - } - ] + {step, key_indent} = wf026_step(lines, number) + + cond do + # `uses:` nested under `env:`, inside a `run: |` script or under + # any other key is data, not the step's action. + wf026_key_indent(line) != key_indent -> + [] + + wf026_with_names_toolchain?(step, key_indent) -> + [] + + true -> + [ + %{ + rule: "WF026", + type: :sha_pinned_toolchain_without_input, + file: filename, + line: number, + current_sha: sha, + severity: :medium, + auto_fixable: true, + fix_script: "fix-rust-toolchain-input.sh", + reason: + "`dtolnay/rust-toolchain` is pinned at `@#{sha}` with no `toolchain:` " <> + "input. The action reads the channel from its `@`-ref; a commit SHA " <> + "carries none, so the step exits 1 with \"'toolchain' is a required " <> + "input\" before any cargo command runs and the red reads as a code " <> + "failure. Add `with: toolchain: stable` (or the channel you mean).", + action: :add_toolchain_input + } + ] end _ -> @@ -1198,22 +1213,29 @@ defmodule Hypatia.Rules.WorkflowAudit do cond do trimmed == "" -> true String.starts_with?(trimmed, "#") -> true - Regex.match?(~r/^\s*-\s/, line) and wf026_indent(line) <= step_indent -> false + Regex.match?(~r/^\s*-(?:\s|$)/, line) and wf026_indent(line) <= step_indent -> false wf026_indent(line) < step_indent -> false true -> true end end + defp wf026_plain?(line) do + trimmed = String.trim(line) + trimmed != "" and not String.starts_with?(trimmed, "#") + end + # The whole list item the `uses:` line at `number` (1-based) belongs to: # from its opening `- ` line — which may sit above `uses:`, since YAML - # mapping keys carry no order — to the next list item at that indentation - # or shallower, or a shallower key. + # mapping keys carry no order, and may be a bare `-` on its own line — to + # the next list item at that indentation or shallower, or a shallower key. + # Returns the step's lines and the column of its keys, so a caller can tell + # a key of the step mapping from the same text nested under another key. defp wf026_step(lines, number) do uses_line = Enum.at(lines, number - 1) uses_indent = wf026_indent(uses_line) {opener, step_indent} = - if Regex.match?(~r/^\s*-\s/, uses_line) do + if Regex.match?(~r/^\s*-(?:\s|$)/, uses_line) do {number, uses_indent} else lines @@ -1221,7 +1243,7 @@ defmodule Hypatia.Rules.WorkflowAudit do |> Enum.with_index(1) |> Enum.reverse() |> Enum.find(fn {l, _} -> - Regex.match?(~r/^\s*-\s/, l) and wf026_indent(l) < uses_indent + Regex.match?(~r/^\s*-(?:\s|$)/, l) and wf026_indent(l) < uses_indent end) |> case do {l, i} -> {i, wf026_indent(l)} @@ -1234,28 +1256,50 @@ defmodule Hypatia.Rules.WorkflowAudit do |> Enum.drop(opener) |> Enum.take_while(&wf026_step_line?(&1, step_indent)) - [Enum.at(lines, opener - 1) | rest] + step = [Enum.at(lines, opener - 1) | rest] + {step, wf026_step_key_indent(step)} + end + + # The column of the step's keys: the opener's key (`- uses:` -> the column + # of `uses`), or for a bare `-` the indentation of the first plain line + # under it. + defp wf026_step_key_indent([opener | rest]) do + cond do + Regex.match?(~r/^\s*-\s+\S/, opener) -> + wf026_key_indent(opener) + + Regex.match?(~r/^\s*-\s*$/, opener) -> + case Enum.find(rest, &wf026_plain?/1) do + nil -> wf026_indent(opener) + 2 + l -> wf026_indent(l) + end + + true -> + wf026_indent(opener) + end end # True when the step's `with:` mapping names `toolchain:` — block form (a # direct child of `with:`) or flow form (`with: { toolchain: stable }`). - # `toolchain:` under `env:` or any other key is not an action input. - defp wf026_with_names_toolchain?(step) do + # Only a `with:` at the step's key column is the step's own; `toolchain:` + # under `env:` or any other key is not an action input. + defp wf026_with_names_toolchain?(step, key_indent) do step |> Enum.with_index() |> Enum.any?(fn {line, i} -> cond do - Regex.match?(~r/^\s*(?:-\s+)?with:\s*\{/, line) -> - Regex.match?(~r/[{,]\s*["']?toolchain["']?\s*:\s*[^\s,}]/, line) + wf026_key_indent(line) != key_indent -> + false - Regex.match?(~r/^\s*(?:-\s+)?with:\s*(?:#.*)?$/, line) -> - with_indent = wf026_key_indent(line) + Regex.match?(~r/^\s*(?:-\s+)?["']?with["']?\s*:\s*\{/, line) -> + Regex.match?(~r/[{,]\s*["']?toolchain["']?\s*:\s*[^\s,}]/, line) + Regex.match?(~r/^\s*(?:-\s+)?["']?with["']?\s*:\s*(?:#.*)?$/, line) -> step |> Enum.drop(i + 1) |> Enum.take_while(fn l -> trimmed = String.trim(l) - trimmed == "" or String.starts_with?(trimmed, "#") or wf026_indent(l) > with_indent + trimmed == "" or String.starts_with?(trimmed, "#") or wf026_indent(l) > key_indent end) |> Enum.any?(&Regex.match?(~r/^\s*["']?toolchain["']?\s*:\s*\S/, &1)) diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs index 7bc68622..6b885abe 100644 --- a/test/rules/workflow_audit_pins_test.exs +++ b/test/rules/workflow_audit_pins_test.exs @@ -165,6 +165,53 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) end + test "a quoted \"uses\" key is the same key: it fires like the bare one" do + wf = """ + jobs: + check: + steps: + - "uses": "dtolnay/rust-toolchain@#{@toolchain_sha}" + """ + + assert [%{rule: "WF026", line: 4}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "the same text inside a run: | script or under env: is not the step's action" do + wf = """ + jobs: + check: + steps: + - name: write a fixture + run: | + cat > fixture.yml <<'EOF' + uses: dtolnay/rust-toolchain@#{@toolchain_sha} + EOF + - uses: dtolnay/rust-toolchain@stable + env: + uses: dtolnay/rust-toolchain@#{@toolchain_sha} + """ + + assert [] = WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "a bare - opener is a step: with: above uses: rescues it, a bare uses: still fires" do + wf = """ + jobs: + check: + steps: + - + with: + toolchain: nightly + uses: dtolnay/rust-toolchain@#{@toolchain_sha} + - + uses: dtolnay/rust-toolchain@#{@toolchain_sha} + """ + + assert [%{line: 9}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + test "the audit carries WF026 findings and a count" do wf = "jobs:\n a:\n steps:\n - uses: dtolnay/rust-toolchain@#{@toolchain_sha}\n" result = WorkflowAudit.audit(["rust.yml"], %{"rust.yml" => wf}) From ad8560e1f69ab7704640131fb6690878590ede44 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 17:16:01 +0000 Subject: [PATCH 14/47] rules(PI001, WF026): a quoted "uses" key reaches the denylist; only direct children of with: are inputs - PinIntegrity.@uses_regex accepts a quoted key (`"uses": "a/b@ref"`), the same key GitHub reads, so PI001 and substitute_denylisted_pins/2 see a quoted caller of a PIN-003 revision as PA001 already did. Test: the quoted pin is caught at its line and repaired in place. - WF026: only a DIRECT child of `with:` is an input. `toolchain: stable` inside another input's block scalar (`components: |`) is text, sits deeper than the first child, and no longer rescues the step. Test added. Mirrors gitbot-fleet b77414a. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- lib/rules/pin_integrity.ex | 4 +++- lib/rules/workflow_audit.ex | 31 +++++++++++++++++++------ test/rules/pin_integrity_test.exs | 15 ++++++++++++ test/rules/workflow_audit_pins_test.exs | 15 ++++++++++++ 4 files changed, 57 insertions(+), 8 deletions(-) diff --git a/lib/rules/pin_integrity.ex b/lib/rules/pin_integrity.ex index c7cd6727..cd057d66 100644 --- a/lib/rules/pin_integrity.ex +++ b/lib/rules/pin_integrity.ex @@ -58,7 +58,9 @@ defmodule Hypatia.Rules.PinIntegrity do # must open the line — after indentation and an optional `-` or `{` — so a # `uses:` inside a `run:` string never counts. The value may be quoted; the # quote, trailing comma and closing brace are not part of the ref. - @uses_regex ~r/^\s*(?:-\s*|\{\s*)?uses:\s*(?["']?)(?[A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@(?[^\s#"',}]+)\k\s*\}?\s*,?\s*(?:#\s*(?.*?))?\s*$/ + # The key itself may be quoted too (`"uses": "a/b@ref"`): GitHub reads + # the same key, so the scanner and the repair must as well. + @uses_regex ~r/^\s*(?:-\s*|\{\s*)?(?["']?)uses\k\s*:\s*(?["']?)(?[A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@(?[^\s#"',}]+)\k\s*\}?\s*,?\s*(?:#\s*(?.*?))?\s*$/ @doc """ Parse every `uses:` pin site in a workflow file. diff --git a/lib/rules/workflow_audit.ex b/lib/rules/workflow_audit.ex index c416fa60..ccebacf6 100644 --- a/lib/rules/workflow_audit.ex +++ b/lib/rules/workflow_audit.ex @@ -1295,13 +1295,30 @@ defmodule Hypatia.Rules.WorkflowAudit do Regex.match?(~r/[{,]\s*["']?toolchain["']?\s*:\s*[^\s,}]/, line) Regex.match?(~r/^\s*(?:-\s+)?["']?with["']?\s*:\s*(?:#.*)?$/, line) -> - step - |> Enum.drop(i + 1) - |> Enum.take_while(fn l -> - trimmed = String.trim(l) - trimmed == "" or String.starts_with?(trimmed, "#") or wf026_indent(l) > key_indent - end) - |> Enum.any?(&Regex.match?(~r/^\s*["']?toolchain["']?\s*:\s*\S/, &1)) + # Only a DIRECT child of `with:` is an input: the contents of another + # input's block scalar (`components: |` carrying `toolchain: stable`) + # sit deeper than the first child and are text, not a key. + children = + step + |> Enum.drop(i + 1) + |> Enum.take_while(fn l -> + trimmed = String.trim(l) + trimmed == "" or String.starts_with?(trimmed, "#") or wf026_indent(l) > key_indent + end) + |> Enum.filter(&wf026_plain?/1) + + case children do + [] -> + false + + [first | _] -> + child_indent = wf026_indent(first) + + Enum.any?(children, fn l -> + wf026_indent(l) == child_indent and + Regex.match?(~r/^\s*["']?toolchain["']?\s*:\s*\S/, l) + end) + end true -> false diff --git a/test/rules/pin_integrity_test.exs b/test/rules/pin_integrity_test.exs index 94768a7e..69ad371d 100644 --- a/test/rules/pin_integrity_test.exs +++ b/test/rules/pin_integrity_test.exs @@ -276,6 +276,21 @@ defmodule Hypatia.Rules.PinIntegrityTest do assert PI.pi001_denylisted_pin(new_content, @policy) == [] end + test "a quoted \"uses\" key is the same key: caught by PI001 and repaired in place" do + content = """ + jobs: + a: + steps: + - "uses": "github/codeql-action/init@#{@poison}" # v4.38.1 + """ + + assert [%{line: 4}] = PI.pi001_denylisted_pin(content, @policy) + + assert {:ok, new_content, [_]} = PI.substitute_denylisted_pins(content, @policy) + assert new_content =~ ~s("uses": "github/codeql-action/init@#{@good}" # v4.38.0) + assert PI.pi001_denylisted_pin(new_content, @policy) == [] + end + test "a file with no denylisted site is returned untouched" do clean = "- uses: actions/checkout@v7.0.1\n" diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs index 6b885abe..8ebcaa24 100644 --- a/test/rules/workflow_audit_pins_test.exs +++ b/test/rules/workflow_audit_pins_test.exs @@ -212,6 +212,21 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) end + test "toolchain: inside another input's block scalar is text, not an input: still fires" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + components: | + toolchain: stable + """ + + assert [%{rule: "WF026", line: 4}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + test "the audit carries WF026 findings and a count" do wf = "jobs:\n a:\n steps:\n - uses: dtolnay/rust-toolchain@#{@toolchain_sha}\n" result = WorkflowAudit.audit(["rust.yml"], %{"rust.yml" => wf}) From 1e8a8176217146519394b8d74b647d21d79f5d3f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 17:30:34 +0000 Subject: [PATCH 15/47] rules(WF026): a step is a list item under steps:; a flow with: is read to its brace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A list item is a step only when its parent key is `steps:`. The rule now walks up the item's ancestors (the parent may sit at the item's own indentation, YAML's `steps:\n- uses:` shape) and refuses any ancestor that opens a block scalar, so a dtolnay line inside a `run: |` script or inside a fixture kept in `env:` — even one that spells out `steps:` — is text and never sends the fleet to rewrite scalar data. - A flow `with: {` spread over several lines is read to its closing brace, so a toolchain on the next line counts and the step is silent. Tests: multi-line flow with/without a toolchain (fires at line 8 only); fixture in env: and run: | item are silent, the real step fires (line 12). Mirrors gitbot-fleet 513bf93. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- lib/rules/workflow_audit.ex | 71 +++++++++++++++++++++++-- test/rules/workflow_audit_pins_test.exs | 39 ++++++++++++++ 2 files changed, 107 insertions(+), 3 deletions(-) diff --git a/lib/rules/workflow_audit.ex b/lib/rules/workflow_audit.ex index ccebacf6..45b5f8fb 100644 --- a/lib/rules/workflow_audit.ex +++ b/lib/rules/workflow_audit.ex @@ -1161,7 +1161,7 @@ defmodule Hypatia.Rules.WorkflowAudit do |> Enum.flat_map(fn {line, number} -> case Regex.run(@toolchain_sha_pin, line) do [_full, sha] -> - {step, key_indent} = wf026_step(lines, number) + {step, key_indent, opener} = wf026_step(lines, number) cond do # `uses:` nested under `env:`, inside a `run: |` script or under @@ -1169,6 +1169,11 @@ defmodule Hypatia.Rules.WorkflowAudit do wf026_key_indent(line) != key_indent -> [] + # A list item is a step only under `steps:`: one inside a block + # scalar (a fixture kept in `env:`, a `run: |` script) is text. + not wf026_under_steps?(lines, opener) -> + [] + wf026_with_names_toolchain?(step, key_indent) -> [] @@ -1257,7 +1262,53 @@ defmodule Hypatia.Rules.WorkflowAudit do |> Enum.take_while(&wf026_step_line?(&1, step_indent)) step = [Enum.at(lines, opener - 1) | rest] - {step, wf026_step_key_indent(step)} + {step, wf026_step_key_indent(step), opener} + end + + # A list item is a step only when its parent key is `steps:`. Walk up the + # ancestors — each the nearest preceding plain, non-list line shallower + # than the last; the parent may sit at the item's own indentation, YAML's + # `steps:\n- uses:` shape — and refuse any ancestor that opens a block + # scalar: an item inside `run: |`, or inside a fixture kept in `env:`, is + # text however much it looks like a step, and must never send the fleet to + # rewrite scalar data. + defp wf026_under_steps?(lines, opener) do + opener_indent = wf026_indent(Enum.at(lines, opener - 1)) + + lines + |> Enum.take(opener - 1) + |> Enum.reverse() + |> Enum.filter(&wf026_plain?/1) + |> wf026_walk_up(opener_indent, nil) + end + + defp wf026_walk_up([], _limit, parent), do: parent == :steps + + defp wf026_walk_up([line | rest], limit, parent) do + indent = wf026_indent(line) + list_item? = Regex.match?(~r/^\s*-(?:\s|$)/, line) + + cond do + indent > limit -> + wf026_walk_up(rest, limit, parent) + + # A sibling item of the one being placed. + list_item? and parent == nil and indent == limit -> + wf026_walk_up(rest, limit, parent) + + Regex.match?(~r/:\s*[|>][-+0-9]*\s*(?:#.*)?$/, line) -> + false + + list_item? -> + wf026_walk_up(rest, indent - 1, parent) + + parent == nil -> + found = if Regex.match?(~r/^\s*["']?steps["']?\s*:/, line), do: :steps, else: :other + wf026_walk_up(rest, indent - 1, found) + + true -> + wf026_walk_up(rest, indent - 1, parent) + end end # The column of the step's keys: the opener's key (`- uses:` -> the column @@ -1292,7 +1343,21 @@ defmodule Hypatia.Rules.WorkflowAudit do false Regex.match?(~r/^\s*(?:-\s+)?["']?with["']?\s*:\s*\{/, line) -> - Regex.match?(~r/[{,]\s*["']?toolchain["']?\s*:\s*[^\s,}]/, line) + # A flow `with: {` on one line, or spread over several up to its + # closing brace: every entry in it is an input. + {inside, rest} = + step + |> Enum.drop(i + 1) + |> Enum.split_while(&(not String.contains?(&1, "}"))) + + flow = + if String.contains?(line, "}") do + [line] + else + [line | inside ++ Enum.take(rest, 1)] + end + + Enum.any?(flow, &Regex.match?(~r/(?:^|[{,])\s*["']?toolchain["']?\s*:\s*[^\s,}]/, &1)) Regex.match?(~r/^\s*(?:-\s+)?["']?with["']?\s*:\s*(?:#.*)?$/, line) -> # Only a DIRECT child of `with:` is an input: the contents of another diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs index 8ebcaa24..34f11012 100644 --- a/test/rules/workflow_audit_pins_test.exs +++ b/test/rules/workflow_audit_pins_test.exs @@ -227,6 +227,45 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) end + test "a flow with: spread over several lines is read to its closing brace" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: { + toolchain: nightly, + } + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: { + components: clippy, + } + """ + + assert [%{line: 8}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "a list item is a step only under steps: — fixtures in env: and run: | are text" do + wf = """ + jobs: + check: + env: + FIXTURE: | + jobs: + a: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + steps: + - run: | + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + """ + + assert [%{line: 12}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + test "the audit carries WF026 findings and a count" do wf = "jobs:\n a:\n steps:\n - uses: dtolnay/rust-toolchain@#{@toolchain_sha}\n" result = WorkflowAudit.audit(["rust.yml"], %{"rust.yml" => wf}) From 5365212736d4900035f2dab672c2e286229b498f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 17:35:46 +0000 Subject: [PATCH 16/47] rules(WF027, CLI, sweep): job-level calls only; keep repair metadata; no clean verdict past the search cap - WF027 fires only on a `uses:` that is a key of a job under `jobs:`, the one place a reusable is called: the same text inside a `run: |` script or under `env:` is data. The ancestor walk WF026 uses is now shared (wf_ancestor_keys/2) and WF027 findings carry their line. - The CLI's workflow-audit normaliser keeps :line, :auto_fixable, :fix_script, :current_sha, :replacement_sha and :reusable, so SARIF points WF026/WF027 at the line and the fleet receives the advertised script and replacement instead of a line-1 finding with no repair. - estate-pin-integrity.sh --via search reads total_count on the first page of every query; past GitHub's 1,000-result cap it logs the overflow, refuses the "clean" verdict (INCOMPLETE, exit 2) and names --via clone as the only way to clear the org. Tests: WF027 fires at line 11 for the job-level call only; the quoted case now asserts its line. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- lib/hypatia/cli.ex | 17 +++++- lib/rules/workflow_audit.ex | 79 +++++++++++++++---------- scripts/sweeps/estate-pin-integrity.sh | 23 ++++++- test/rules/workflow_audit_pins_test.exs | 21 ++++++- 4 files changed, 106 insertions(+), 34 deletions(-) diff --git a/lib/hypatia/cli.ex b/lib/hypatia/cli.ex index 0d06a4f8..cd937d31 100644 --- a/lib/hypatia/cli.ex +++ b/lib/hypatia/cli.ex @@ -486,7 +486,22 @@ defmodule Hypatia.CLI do # structural_drift normalizer's `Map.take` above. Before this, # 7 `fix_recipe:` and 7 `recipe_id:` declarations in # workflow_audit.ex were read by nothing at all. - |> Map.merge(Map.take(f, [:fix_recipe, :recipe_id, :job])) + # WF026/WF027 anchor to a line and name their repair script and + # the SHAs involved; keep those so SARIF points at the line and + # the fleet receives the advertised script and replacement. + |> Map.merge( + Map.take(f, [ + :fix_recipe, + :recipe_id, + :job, + :line, + :auto_fixable, + :fix_script, + :current_sha, + :replacement_sha, + :reusable + ]) + ) end) results ++ normalized diff --git a/lib/rules/workflow_audit.ex b/lib/rules/workflow_audit.ex index 45b5f8fb..06e69f76 100644 --- a/lib/rules/workflow_audit.ex +++ b/lib/rules/workflow_audit.ex @@ -1265,52 +1265,64 @@ defmodule Hypatia.Rules.WorkflowAudit do {step, wf026_step_key_indent(step), opener} end - # A list item is a step only when its parent key is `steps:`. Walk up the - # ancestors — each the nearest preceding plain, non-list line shallower - # than the last; the parent may sit at the item's own indentation, YAML's - # `steps:\n- uses:` shape — and refuse any ancestor that opens a block - # scalar: an item inside `run: |`, or inside a fixture kept in `env:`, is - # text however much it looks like a step, and must never send the fleet to - # rewrite scalar data. + # A list item is a step only when its parent key is `steps:`; an item + # inside `run: |`, or inside a fixture kept in `env:`, is text however much + # it looks like a step, and must never send the fleet to rewrite scalar data. defp wf026_under_steps?(lines, opener) do - opener_indent = wf026_indent(Enum.at(lines, opener - 1)) + match?(["steps" | _], wf_ancestor_keys(lines, opener)) + end + + # The mapping keys enclosing the line at `number`, nearest first — list + # items are nodes, not keys, and are skipped — or nil when an ancestor + # opens a block scalar: the line is then text, not YAML structure. The + # parent of a list item may sit at the item's own indentation (YAML's + # `steps:\n- uses:` shape); every further ancestor is strictly shallower. + defp wf_ancestor_keys(lines, number) do + line = Enum.at(lines, number - 1) + indent = wf026_indent(line) + list_item? = Regex.match?(~r/^\s*-(?:\s|$)/, line) + limit = if list_item?, do: indent, else: indent - 1 lines - |> Enum.take(opener - 1) + |> Enum.take(number - 1) |> Enum.reverse() |> Enum.filter(&wf026_plain?/1) - |> wf026_walk_up(opener_indent, nil) + |> wf_walk_up(limit, list_item?, []) end - defp wf026_walk_up([], _limit, parent), do: parent == :steps + defp wf_walk_up([], _limit, _sibling_ok?, keys), do: Enum.reverse(keys) - defp wf026_walk_up([line | rest], limit, parent) do + defp wf_walk_up([line | rest], limit, sibling_ok?, keys) do indent = wf026_indent(line) list_item? = Regex.match?(~r/^\s*-(?:\s|$)/, line) cond do indent > limit -> - wf026_walk_up(rest, limit, parent) + wf_walk_up(rest, limit, sibling_ok?, keys) # A sibling item of the one being placed. - list_item? and parent == nil and indent == limit -> - wf026_walk_up(rest, limit, parent) + list_item? and sibling_ok? and indent == limit -> + wf_walk_up(rest, limit, sibling_ok?, keys) Regex.match?(~r/:\s*[|>][-+0-9]*\s*(?:#.*)?$/, line) -> - false + nil list_item? -> - wf026_walk_up(rest, indent - 1, parent) - - parent == nil -> - found = if Regex.match?(~r/^\s*["']?steps["']?\s*:/, line), do: :steps, else: :other - wf026_walk_up(rest, indent - 1, found) + wf_walk_up(rest, indent - 1, false, keys) true -> - wf026_walk_up(rest, indent - 1, parent) + wf_walk_up(rest, indent - 1, false, [wf_key_name(line) | keys]) end end + defp wf_key_name(line) do + line + |> String.replace(~r/^\s*(?:-\s+)?/, "") + |> String.replace(~r/\s*:.*$/, "") + |> String.trim("\"") + |> String.trim("'") + end + # The column of the step's keys: the opener's key (`- uses:` -> the column # of `uses`), or for a bare `-` the indentation of the first plain line # under it. @@ -1531,22 +1543,29 @@ defmodule Hypatia.Rules.WorkflowAudit do """ def check_known_broken_reusable_revision(workflow_contents) do Enum.flat_map(workflow_contents, fn {filename, content} -> - stripped = strip_comments(content) + lines = String.split(content, "\n") Enum.flat_map(@broken_reusable_revisions, fn rev -> - # Anchored to a `uses:` key at the start of a line (quoted or not): - # the reusable's path inside a string value or a comment is not a pin. + # Anchored to a `uses:` key at the start of a line (quoted or not) that + # is a key of a job under `jobs:` — the only place a reusable is + # called. The reusable's path inside a `run: |` script, under `env:`, + # in a string value or in a comment is not a call, so the fleet is + # never sent to rewrite script data. ref_re = - ~r/^\s*(?:-\s+)?["']?uses["']?\s*:\s*["']?#{Regex.escape(rev.reusable)}@([0-9a-f]{40})/m + ~r/^\s*(?:-\s+)?["']?uses["']?\s*:\s*["']?#{Regex.escape(rev.reusable)}@([0-9a-f]{40})/ - Regex.scan(ref_re, stripped) - |> Enum.flat_map(fn [_full, sha] -> - if sha in rev.shas do + lines + |> Enum.with_index(1) + |> Enum.flat_map(fn {line, number} -> + with [_full, sha] <- Regex.run(ref_re, line), + true <- sha in rev.shas, + [_job, "jobs" | _] <- wf_ancestor_keys(lines, number) do [ %{ rule: "WF027", type: :known_broken_reusable_revision, file: filename, + line: number, reusable: rev.reusable, current_sha: sha, replacement_sha: rev.replacement, @@ -1562,7 +1581,7 @@ defmodule Hypatia.Rules.WorkflowAudit do } ] else - [] + _ -> [] end end) end) diff --git a/scripts/sweeps/estate-pin-integrity.sh b/scripts/sweeps/estate-pin-integrity.sh index b2e855c9..3e952ee9 100755 --- a/scripts/sweeps/estate-pin-integrity.sh +++ b/scripts/sweeps/estate-pin-integrity.sh @@ -151,14 +151,28 @@ mapfile -t SEARCH_TOKENS < <(jq -r ' | (if ((.blocked_shas // []) | length) > 8 then [.action] else (.blocked_shas // []) end)[], (.blocked_versions // [])[]' "$POLICY" | sort -u) +# GitHub's code search returns at most 1,000 results per query (ten pages +# of 100), whatever the true count. A broad query that overflows the cap +# leaves candidates phase 2 never sees, so the sweep says so and refuses a +# clean verdict: a search that could not see everything is INCOMPLETE, and +# only `--via clone` can clear the org. +incomplete=0 if [ "$VIA" = "search" ]; then for org in "${ORGS[@]}"; do for token in "${SEARCH_TOKENS[@]}"; do page=1 while :; do - n=$(gh api "search/code?q=${token}+org:${org}&per_page=100&page=${page}" \ - --jq '.items[] | "'"${org}"'\t\(.repository.name)\t\(.path)"' 2>/dev/null \ + gh api "search/code?q=${token}+org:${org}&per_page=100&page=${page}" \ + > "$OUT_DIR/.search.json" 2>/dev/null || : > "$OUT_DIR/.search.json" + n=$(jq -r '.items[]? | "'"${org}"'\t\(.repository.name)\t\(.path)"' "$OUT_DIR/.search.json" 2>/dev/null \ | tee -a "$HITS" | wc -l) + if [ "$page" -eq 1 ]; then + total=$(jq -r '.total_count // 0' "$OUT_DIR/.search.json" 2>/dev/null || echo 0) + if [ "${total:-0}" -gt 1000 ] 2>/dev/null; then + log " ! ${org}: '${token}' has ${total} results; GitHub shows only the first 1000 — phase 1 is INCOMPLETE for this org, use --via clone" + incomplete=$((incomplete + 1)) + fi + fi [ "$n" -lt 100 ] && break page=$((page + 1)) [ "$page" -gt 10 ] && break @@ -191,6 +205,10 @@ total_hits=$(wc -l < "$HITS") log "phase 1: ${total_hits} candidate file(s)" if [ "$total_hits" -eq 0 ]; then + if [ "$incomplete" -gt 0 ]; then + echo "INCOMPLETE: no denylisted pin tokens in the visible search results for ${ORGS[*]}, but ${incomplete} search(es) overflowed GitHub's 1,000-result cap; a clean verdict needs --via clone" + exit 2 + fi echo "clean: no denylisted pin tokens found in ${ORGS[*]}" exit 0 fi @@ -321,5 +339,6 @@ log " repair plan : ${PLAN}" [ "$REWRITE" -eq 1 ] && log " repaired trees : ${OUT_DIR}/patched" log " NOTE: this sweep never writes to a remote. Branch + PR creation is" log " the token-bearing actuator's job — docs/operations/estate-automerge.adoc" +[ "$incomplete" -gt 0 ] && log " INCOMPLETE: ${incomplete} search(es) overflowed GitHub's 1,000-result cap; candidates beyond it were not seen — rerun with --via clone" exit 2 diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs index 34f11012..cd3b259c 100644 --- a/test/rules/workflow_audit_pins_test.exs +++ b/test/rules/workflow_audit_pins_test.exs @@ -337,7 +337,26 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do "uses": "#{@rust_ci}@#{@broken_standards}" """ - assert [%{rule: "WF027", current_sha: @broken_standards}] = + assert [%{rule: "WF027", current_sha: @broken_standards, line: 3}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => wf}) + end + + test "only a key of a job is a call: run: | and env: copies are not" do + wf = """ + jobs: + fixture: + env: + uses: #{@rust_ci}@#{@broken_standards} + steps: + - run: | + cat > caller.yml <<'EOF' + uses: #{@rust_ci}@#{@broken_standards} + EOF + rust-ci: + uses: #{@rust_ci}@#{@broken_standards} + """ + + assert [%{rule: "WF027", line: 11}] = WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => wf}) end From f5a0e27185fb1d171f67ea2b5e3d1fdc76c1e6ec Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 17:44:39 +0000 Subject: [PATCH 17/47] rules(WF026): a flow with: closes when its brace depth returns to zero An expression inside a multi-line flow mapping (`components: ${{ matrix.component }},`) opens and closes its own braces; the scan ended at the first `}` and missed a toolchain written after it, reporting the step and offering a repair that would have duplicated the key. The mapping now ends when the net depth returns to zero. Test added. Mirrors the fleet repair script's brace tracking. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- lib/rules/workflow_audit.ex | 31 ++++++++++++++++--------- test/rules/workflow_audit_pins_test.exs | 15 ++++++++++++ 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/lib/rules/workflow_audit.ex b/lib/rules/workflow_audit.ex index 06e69f76..28124f9e 100644 --- a/lib/rules/workflow_audit.ex +++ b/lib/rules/workflow_audit.ex @@ -1315,6 +1315,10 @@ defmodule Hypatia.Rules.WorkflowAudit do end end + defp wf_brace_delta(line) do + length(String.split(line, "{")) - length(String.split(line, "}")) + end + defp wf_key_name(line) do line |> String.replace(~r/^\s*(?:-\s+)?/, "") @@ -1355,18 +1359,23 @@ defmodule Hypatia.Rules.WorkflowAudit do false Regex.match?(~r/^\s*(?:-\s+)?["']?with["']?\s*:\s*\{/, line) -> - # A flow `with: {` on one line, or spread over several up to its - # closing brace: every entry in it is an input. - {inside, rest} = - step - |> Enum.drop(i + 1) - |> Enum.split_while(&(not String.contains?(&1, "}"))) - + # A flow `with: {` on one line, or spread over several up to the + # brace that closes it — an expression `${{ matrix.x }}` inside it + # opens and closes its own braces, so the mapping ends when the net + # depth returns to zero: every entry in it is an input. flow = - if String.contains?(line, "}") do - [line] - else - [line | inside ++ Enum.take(rest, 1)] + case wf_brace_delta(line) do + depth when depth <= 0 -> + [line] + + depth -> + step + |> Enum.drop(i + 1) + |> Enum.reduce_while({depth, [line]}, fn l, {d, acc} -> + d = d + wf_brace_delta(l) + if d <= 0, do: {:halt, {d, [l | acc]}}, else: {:cont, {d, [l | acc]}} + end) + |> elem(1) end Enum.any?(flow, &Regex.match?(~r/(?:^|[{,])\s*["']?toolchain["']?\s*:\s*[^\s,}]/, &1)) diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs index cd3b259c..835d3db2 100644 --- a/test/rules/workflow_audit_pins_test.exs +++ b/test/rules/workflow_audit_pins_test.exs @@ -246,6 +246,21 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) end + test "an expression's braces inside a multi-line flow with: do not close it early" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: { + components: ${{ matrix.component }}, + toolchain: nightly, + } + """ + + assert [] = WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + test "a list item is a step only under steps: — fixtures in env: and run: | are text" do wf = """ jobs: From 7a95c15dd69896a7c03db16dcf303ee6d841db6a Mon Sep 17 00:00:00 2001 From: hyperpolymath Date: Mon, 19 Oct 2026 18:09:28 +0000 Subject: [PATCH 18/47] rules(WF026): KYAML and flow-style steps, anchors and aliases; sweep: a search that failed or a candidate unread is never clean WF026: - a KYAML workflow (YAML-POLICY Y-3, the output of `yq -o kyaml`) is read as its block equivalent through ResearchExtensions.flow_to_block/1, line for line, so a quoted, braced step is the same step and a finding points at the source line; - a flow-style step written inline in a block document (`- { uses: ..., with: { toolchain: stable } }`, `steps: [ { uses: ... } ]`, a `- {` item spread over lines) is read through its braces with quotes, comments and nested braces respected; a `uses:` inside a quoted scalar is text; - `with: &inputs` (block or flow), `with: *inputs` and a `<<: *inputs` child are resolved from the document and the anchored mapping's direct entries decide; an alias with no anchor (or one that reaches itself) is reported as unreadable, never passed silently; - only a plain block-style step is handed to the fixer: a finding on a KYAML document, a flow-style step, or a `with:` that goes through an anchor or alias carries auto_fixable: false (the fleet script refuses those shapes, exit 2) and says why; a block `with:` whose merged mapping lacks the input stays repairable, since an explicit child overrides a merge key. estate-pin-integrity.sh (--via search): - a `gh api` call that fails (expired auth, rate limit, outage) is a failed search, counted and logged, not an empty page: the run is INCOMPLETE (exit 2) rather than `clean`; - the exit code follows confirmed denylisted pins, not phase-1 candidates: a broad action-path query lists healthy callers too, and a run whose candidates all turn out clean now exits 0, while a candidate that could not be read makes the run INCOMPLETE. Tests: seven WF026 cases (KYAML document, inline flow steps, a flow list in env: as decoy, `- {` items with nested expression braces, anchors/aliases/ merge keys, an unresolved alias, a plain block finding still the fixer's). Sweep behaviour exercised offline with a fake gh (failing API, healthy caller, unreadable candidate). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- .machine_readable/descriptiles/AGENTIC.a2ml | 2 + lib/rules/workflow_audit.ex | 611 ++++++++++++++++---- scripts/sweeps/estate-pin-integrity.sh | 56 +- test/rules/workflow_audit_pins_test.exs | 154 +++++ 4 files changed, 714 insertions(+), 109 deletions(-) diff --git a/.machine_readable/descriptiles/AGENTIC.a2ml b/.machine_readable/descriptiles/AGENTIC.a2ml index 31a5f12f..ecc2bc39 100644 --- a/.machine_readable/descriptiles/AGENTIC.a2ml +++ b/.machine_readable/descriptiles/AGENTIC.a2ml @@ -46,6 +46,8 @@ completion-claim-requires-behavioural-tests = true # Every rule here has a fleet fix script that carries the SAME facts, so a # detection without a repair, or a repair without a detection, is a drift: # WF026 workflow_audit.ex SHA-pinned dtolnay/rust-toolchain with no toolchain: input -> fleet fix-rust-toolchain-input.sh +# (detected in block YAML and KYAML/flow style alike, Y-3; repaired in block style only — +# a KYAML, flow-step, anchored or aliased finding carries auto_fixable: false and routes to review) # WF027 workflow_audit.ex caller pinned at a reusable revision MEASURED dead (table) -> fleet fix-stale-reusable-pin.sh (Window A) # PIN-003 pr-automerge-policy.json standards revisions whose own actions.lock GitHub # rejects at job setup (545 commits, 2026-08-19..2026-10-19; PI001/PA001 fire) -> fleet fix-stale-reusable-pin.sh (Window B) diff --git a/lib/rules/workflow_audit.ex b/lib/rules/workflow_audit.ex index 28124f9e..d290cfd8 100644 --- a/lib/rules/workflow_audit.ex +++ b/lib/rules/workflow_audit.ex @@ -1114,6 +1114,10 @@ defmodule Hypatia.Rules.WorkflowAudit do @toolchain_sha_pin ~r/^\s*(?:-\s+)?["']?uses["']?\s*:\s*["']?dtolnay\/rust-toolchain@([0-9a-f]{40})/ + # The same key written inside a flow mapping on the line that opens it: + # `- { uses: dtolnay/rust-toolchain@ }`, `steps: [ { uses: ... } ]`. + @toolchain_sha_pin_inline ~r/[{,]\s*["']?uses["']?\s*:\s*["']?dtolnay\/rust-toolchain@([0-9a-f]{40})/ + @doc """ WF026: a SHA-pinned `dtolnay/rust-toolchain` step that passes no `toolchain:` input. @@ -1148,62 +1152,172 @@ defmodule Hypatia.Rules.WorkflowAudit do cannot rescue an earlier step. * Severity is medium: the rule reports and routes to the fleet; it is never a blocking gate (AGENTIC `must-not-promote-to-blocking`). + * A KYAML workflow (standards YAML-POLICY Y-3, the output of + `yq -o kyaml`) is judged as its block equivalent, line for line + (`ResearchExtensions.flow_to_block/1`), so a quoted, braced step is + the same step. A flow-style step written inline in a block document + (`- { uses: ..., with: { toolchain: stable } }`, + `steps: [ { uses: ... } ]`, or a `- {` item spread over lines) is read + through its braces: its `with` entry decides, quotes and nested + braces respected, and a `uses:` inside a quoted string is text. + * `with:` may be an anchor (`with: &inputs`, block or flow), an alias + (`with: *inputs`) or carry a merge key (`<<: *inputs`): the anchored + mapping is resolved from the document and its direct entries decide. + An alias with no anchor in the file cannot be read; the step is + reported so, not passed silently. Auto-fixable: gitbot-fleet `scripts/fix-rust-toolchain-input.sh` inserts - `toolchain: stable` (the channel the estate's reusables pass). + `toolchain: stable` (the channel the estate's reusables pass) into a + block-style step whose `with:` is absent, a block mapping or a flow + mapping. The fixer refuses every other shape rather than guess (exit 2, + file untouched), so a finding on a KYAML document, a flow-style step, or + a `with:` that goes through an anchor or alias carries + `auto_fixable: false` and routes to review instead of to the fixer. """ def check_sha_pinned_toolchain_without_input(workflow_contents) do Enum.flat_map(workflow_contents, fn {filename, content} -> - lines = String.split(content, "\n") + # KYAML is read as the block layout it denotes; line numbers survive. + normalised = Hypatia.Rules.ResearchExtensions.flow_to_block(content) + kyaml? = normalised != content + lines = String.split(normalised, "\n") lines |> Enum.with_index(1) |> Enum.flat_map(fn {line, number} -> case Regex.run(@toolchain_sha_pin, line) do [_full, sha] -> - {step, key_indent, opener} = wf026_step(lines, number) + wf026_block_step_finding(filename, lines, line, number, sha, kyaml?) - cond do - # `uses:` nested under `env:`, inside a `run: |` script or under - # any other key is data, not the step's action. - wf026_key_indent(line) != key_indent -> - [] + _ -> + case Regex.run(@toolchain_sha_pin_inline, line, return: :index) do + [{_, _}, {sha_at, 40}] -> + sha = binary_part(line, sha_at, 40) + wf026_inline_step_finding(filename, lines, line, number, sha, kyaml?) - # A list item is a step only under `steps:`: one inside a block - # scalar (a fixture kept in `env:`, a `run: |` script) is text. - not wf026_under_steps?(lines, opener) -> + _ -> [] + end + end + end) + end) + end - wf026_with_names_toolchain?(step, key_indent) -> - [] + # The block-style path: `uses:` is a key on its own line. + defp wf026_block_step_finding(filename, lines, line, number, sha, kyaml?) do + {step, key_indent, opener} = wf026_step(lines, number) + flow_item? = Regex.match?(~r/^\s*-\s+\{/, Enum.at(lines, opener - 1)) - true -> - [ - %{ - rule: "WF026", - type: :sha_pinned_toolchain_without_input, - file: filename, - line: number, - current_sha: sha, - severity: :medium, - auto_fixable: true, - fix_script: "fix-rust-toolchain-input.sh", - reason: - "`dtolnay/rust-toolchain` is pinned at `@#{sha}` with no `toolchain:` " <> - "input. The action reads the channel from its `@`-ref; a commit SHA " <> - "carries none, so the step exits 1 with \"'toolchain' is a required " <> - "input\" before any cargo command runs and the red reads as a code " <> - "failure. Add `with: toolchain: stable` (or the channel you mean).", - action: :add_toolchain_input - } - ] - end + cond do + # `uses:` nested under `env:`, inside a `run: |` script or under + # any other key is data, not the step's action. + wf026_key_indent(line) != key_indent -> + [] - _ -> + # A list item is a step only under `steps:`: one inside a block + # scalar (a fixture kept in `env:`, a `run: |` script) is text. + not wf026_under_steps?(lines, opener) -> + [] + + true -> + case wf026_with(step, key_indent, lines) do + {:toolchain, _form} -> [] + + verdict -> + form = if flow_item?, do: :flow_step, else: elem(verdict, 1) + [wf026_finding(filename, number, sha, verdict, form, kyaml?)] end - end) - end) + end + end + + # The inline path: `uses:` sits inside a flow mapping on the line that + # opens it. The mapping is read through its braces (quote-aware, across + # lines) and its `with` entry decides. + defp wf026_inline_step_finding(filename, lines, line, number, sha, kyaml?) do + uses_at = + case Regex.run(@toolchain_sha_pin_inline, line, return: :index) do + [{at, _} | _] -> at + 1 + _ -> nil + end + + with true <- is_integer(uses_at), + {:ok, open_col} <- wf_flow_opener_before(line, uses_at), + true <- wf026_inline_item?(line, open_col), + true <- wf026_inline_under_steps?(lines, line, number, open_col), + {:ok, interior, _} <- wf_flow_interior(lines, number, open_col) do + case wf026_with_entry(wf_flow_entries(interior), lines) do + {:toolchain, _} -> [] + verdict -> [wf026_finding(filename, number, sha, verdict, :flow_step, kyaml?)] + end + else + _ -> [] + end + end + + # The text before the mapping's `{` must be a sequence position: a list + # dash, the `steps:` key opening its sequence, a `[`, a comma, or nothing. + # Anything else (`run: "echo { uses: ... }"`) is a scalar carrying braces. + defp wf026_inline_item?(line, open_col) do + prefix = binary_part(line, 0, open_col) + Regex.match?(~r/^\s*(?:-\s*)?(?:["']?steps["']?\s*:\s*)?[\[,\s]*$/, prefix) + end + + defp wf026_inline_under_steps?(lines, line, number, open_col) do + prefix = binary_part(line, 0, open_col) + + if Regex.match?(~r/["']?steps["']?\s*:/, prefix) do + # The line is the `steps:` key itself; it belongs to a job. + match?([_job, "jobs" | _], wf_ancestor_keys(lines, number) || []) + else + wf026_under_steps?(lines, number) + end + end + + # The finding. Only a plain block-style step is handed to the fixer: + # the fleet script refuses flow-style steps, anchors and aliases (exit + # 2, file untouched), and a KYAML document is a flow-style document. + defp wf026_finding(filename, number, sha, verdict, form, kyaml?) do + repairable? = not kyaml? and form in [:none, :block, :flow] + + shape = + case {verdict, form} do + {{:unreadable, :alias, name}, _} -> + " Its `with:` is the alias `*#{name}` and no `&#{name}` anchor is defined in " <> + "the file, so the inputs cannot be read; the alias must be resolved by hand." + + {_, :flow_step} -> + " The step is written in flow style; the repair script rewrites block-style " <> + "steps only, so this one routes to review." + + {_, f} when f in [:anchor, :alias, :merge] -> + " Its `with:` goes through a YAML anchor or alias; the repair script does not " <> + "rewrite anchored mappings, so this one routes to review." + + _ when kyaml? -> + " The workflow is KYAML; the repair script rewrites block-style steps only, " <> + "so this one routes to review." + + _ -> + "" + end + + %{ + rule: "WF026", + type: :sha_pinned_toolchain_without_input, + file: filename, + line: number, + current_sha: sha, + severity: :medium, + auto_fixable: repairable?, + fix_script: if(repairable?, do: "fix-rust-toolchain-input.sh", else: nil), + reason: + "`dtolnay/rust-toolchain` is pinned at `@#{sha}` with no `toolchain:` " <> + "input. The action reads the channel from its `@`-ref; a commit SHA " <> + "carries none, so the step exits 1 with \"'toolchain' is a required " <> + "input\" before any cargo command runs and the red reads as a code " <> + "failure. Add `with: toolchain: stable` (or the channel you mean)." <> shape, + action: :add_toolchain_input + } end # Indentation width of a line (spaces only; workflow YAML never uses tabs). @@ -1304,10 +1418,17 @@ defmodule Hypatia.Rules.WorkflowAudit do list_item? and sibling_ok? and indent == limit -> wf_walk_up(rest, limit, sibling_ok?, keys) + # A flow closer (`}`, `],`) on its own line ends a collection some + # earlier line opened; it names nothing and bounds nothing. + Regex.match?(~r/^\s*[}\]]/, line) -> + wf_walk_up(rest, limit, sibling_ok?, keys) + Regex.match?(~r/:\s*[|>][-+0-9]*\s*(?:#.*)?$/, line) -> nil - list_item? -> + # A list item, or a `{`/`[` opening a flow item on its own line, is a + # node, not a key. + list_item? or Regex.match?(~r/^\s*[{\[]/, line) -> wf_walk_up(rest, indent - 1, false, keys) true -> @@ -1315,10 +1436,6 @@ defmodule Hypatia.Rules.WorkflowAudit do end end - defp wf_brace_delta(line) do - length(String.split(line, "{")) - length(String.split(line, "}")) - end - defp wf_key_name(line) do line |> String.replace(~r/^\s*(?:-\s+)?/, "") @@ -1328,14 +1445,14 @@ defmodule Hypatia.Rules.WorkflowAudit do end # The column of the step's keys: the opener's key (`- uses:` -> the column - # of `uses`), or for a bare `-` the indentation of the first plain line - # under it. + # of `uses`), or for a bare `-` — or a `- {` opening a flow-style item — the + # indentation of the first plain line under it. defp wf026_step_key_indent([opener | rest]) do cond do - Regex.match?(~r/^\s*-\s+\S/, opener) -> + Regex.match?(~r/^\s*-\s+\S/, opener) and not Regex.match?(~r/^\s*-\s+\{\s*$/, opener) -> wf026_key_indent(opener) - Regex.match?(~r/^\s*-\s*$/, opener) -> + Regex.match?(~r/^\s*-\s*(?:\{\s*)?$/, opener) -> case Enum.find(rest, &wf026_plain?/1) do nil -> wf026_indent(opener) + 2 l -> wf026_indent(l) @@ -1346,72 +1463,362 @@ defmodule Hypatia.Rules.WorkflowAudit do end end - # True when the step's `with:` mapping names `toolchain:` — block form (a - # direct child of `with:`) or flow form (`with: { toolchain: stable }`). - # Only a `with:` at the step's key column is the step's own; `toolchain:` - # under `env:` or any other key is not an action input. - defp wf026_with_names_toolchain?(step, key_indent) do + # What the step's `with:` says about `toolchain:`, as a verdict: + # `{:toolchain, form}`, `{:missing, form}` or `{:unreadable, :alias, name}`, + # where form is :none (no `with:`), :block, :flow, :anchor (`with: &x`), + # :alias (`with: *x`) or :merge (a `<<: *x` child). Only a `with:` at the + # step's key column is the step's own; `toolchain:` under `env:` or any + # other key is not an action input. + defp wf026_with(step, key_indent, lines) do step |> Enum.with_index() - |> Enum.any?(fn {line, i} -> - cond do - wf026_key_indent(line) != key_indent -> - false - - Regex.match?(~r/^\s*(?:-\s+)?["']?with["']?\s*:\s*\{/, line) -> - # A flow `with: {` on one line, or spread over several up to the - # brace that closes it — an expression `${{ matrix.x }}` inside it - # opens and closes its own braces, so the mapping ends when the net - # depth returns to zero: every entry in it is an input. - flow = - case wf_brace_delta(line) do - depth when depth <= 0 -> - [line] - - depth -> - step - |> Enum.drop(i + 1) - |> Enum.reduce_while({depth, [line]}, fn l, {d, acc} -> - d = d + wf_brace_delta(l) - if d <= 0, do: {:halt, {d, [l | acc]}}, else: {:cont, {d, [l | acc]}} - end) - |> elem(1) + |> Enum.find_value({:missing, :none}, fn {line, i} -> + if wf026_key_indent(line) == key_indent do + case Regex.run(~r/^\s*(?:-\s+)?["']?with["']?\s*:(.*)$/, line) do + [_, value] -> + value = value |> wf_strip_comment() |> String.trim() + wf026_with_value(value, line, Enum.drop(step, i + 1), key_indent, lines) + + nil -> + nil + end + end + end) + end + + defp wf026_with_value(value, line, rest, key_indent, lines) do + cond do + # `with: *inputs` — the anchored mapping, wherever it is defined. + match?([_ | _], Regex.run(~r/^\*([\w.-]+)$/, value)) -> + [_, name] = Regex.run(~r/^\*([\w.-]+)$/, value) + wf026_alias_verdict(name, lines, :alias) + + # `with: {` / `with: &inputs {` — a flow mapping, read to its brace. + String.contains?(value, "{") -> + form = if String.starts_with?(value, "&"), do: :anchor, else: :flow + [{at, _}] = Regex.run(~r/\{/, line, return: :index) + + case wf_flow_interior([line | rest], 1, at) do + {:ok, interior, _} -> wf026_entries_verdict(wf_flow_entries(interior), lines, form) + :error -> {:missing, form} + end + + # `with:` / `with: &inputs` — a block mapping: only a DIRECT child is + # an input; the contents of another input's block scalar + # (`components: |` carrying `toolchain: stable`) sit deeper and are text. + true -> + form = if String.starts_with?(value, "&"), do: :anchor, else: :block + wf026_entries_verdict(wf_block_children(rest, key_indent), lines, form) + end + end + + # The `with` entry of a flow-style step's own entries. + defp wf026_with_entry(entries, lines) do + case List.keyfind(entries, "with", 0) do + nil -> + {:missing, :none} + + {_, value} -> + value = String.trim(value) + + cond do + match?([_ | _], Regex.run(~r/^\*([\w.-]+)$/, value)) -> + [_, name] = Regex.run(~r/^\*([\w.-]+)$/, value) + wf026_alias_verdict(name, lines, :alias) + + String.contains?(value, "{") -> + form = if String.starts_with?(value, "&"), do: :anchor, else: :flow + [{at, _}] = Regex.run(~r/\{/, value, return: :index) + + case wf_flow_interior([value], 1, at) do + {:ok, interior, _} -> wf026_entries_verdict(wf_flow_entries(interior), lines, form) + :error -> {:missing, form} end - Enum.any?(flow, &Regex.match?(~r/(?:^|[{,])\s*["']?toolchain["']?\s*:\s*[^\s,}]/, &1)) - - Regex.match?(~r/^\s*(?:-\s+)?["']?with["']?\s*:\s*(?:#.*)?$/, line) -> - # Only a DIRECT child of `with:` is an input: the contents of another - # input's block scalar (`components: |` carrying `toolchain: stable`) - # sit deeper than the first child and are text, not a key. - children = - step - |> Enum.drop(i + 1) - |> Enum.take_while(fn l -> - trimmed = String.trim(l) - trimmed == "" or String.starts_with?(trimmed, "#") or wf026_indent(l) > key_indent - end) - |> Enum.filter(&wf026_plain?/1) - - case children do - [] -> - false - - [first | _] -> - child_indent = wf026_indent(first) - - Enum.any?(children, fn l -> - wf026_indent(l) == child_indent and - Regex.match?(~r/^\s*["']?toolchain["']?\s*:\s*\S/, l) - end) + true -> + {:missing, :flow} + end + end + end + + # Does a list of `{key, value}` entries name a toolchain? A `<<: *x` merge + # key brings the anchored mapping's entries in. + defp wf026_entries_verdict(entries, lines, form, seen \\ []) do + cond do + Enum.any?(entries, fn {k, v} -> k == "toolchain" and wf_unquote(v) != "" end) -> + {:toolchain, form} + + true -> + entries + |> Enum.filter(fn {k, v} -> + k == "<<" and Regex.match?(~r/^\*[\w.-]+$/, String.trim(v)) + end) + |> Enum.find_value({:missing, form}, fn {_, v} -> + name = v |> String.trim() |> String.trim_leading("*") + + case wf026_alias_verdict(name, lines, :merge, seen) do + {:missing, _} -> nil + verdict -> verdict end + end) + end + end - true -> - false + # An alias that reaches itself (`&x` whose mapping merges `*x`) is as + # unreadable as one with no anchor. + defp wf026_alias_verdict(name, lines, form, seen \\ []) do + if name in seen do + {:unreadable, :alias, name} + else + case wf_anchor_entries(lines, name) do + {:ok, entries} -> wf026_entries_verdict(entries, lines, form, [name | seen]) + :error -> {:unreadable, :alias, name} + end + end + end + + # The entries of the mapping anchored `&name`: the first `&name` in the + # document, read as a flow mapping when a `{` follows it, else as the + # block mapping under its line. + defp wf_anchor_entries(lines, name) do + re = ~r/&#{Regex.escape(name)}(?=[\s,{}\[\]]|$)/ + + lines + |> Enum.with_index(1) + |> Enum.find(fn {l, _} -> wf026_plain?(l) and Regex.match?(re, wf_strip_comment(l)) end) + |> case do + nil -> + :error + + {line, number} -> + [{at, len}] = Regex.run(re, line, return: :index) + + after_anchor = + line |> binary_part(at + len, byte_size(line) - at - len) |> wf_strip_comment() + + case Regex.run(~r/\{/, after_anchor, return: :index) do + [{brace, _}] -> + case wf_flow_interior(lines, number, at + len + brace) do + {:ok, interior, _} -> {:ok, wf_flow_entries(interior)} + :error -> :error + end + + nil -> + {:ok, wf_block_children(Enum.drop(lines, number), wf026_key_indent(line))} + end + end + end + + # The direct children of a block mapping whose key sits at `key_indent`, + # as `{key, value}` entries: the plain lines deeper than the key, at the + # first child's indentation. Deeper lines are nested mappings or scalar + # text, not entries. + defp wf_block_children(rest, key_indent) do + children = + rest + |> Enum.take_while(fn l -> + trimmed = String.trim(l) + trimmed == "" or String.starts_with?(trimmed, "#") or wf026_indent(l) > key_indent + end) + |> Enum.filter(&wf026_plain?/1) + + case children do + [] -> + [] + + [first | _] -> + child_indent = wf026_indent(first) + + children + |> Enum.filter(&(wf026_indent(&1) == child_indent)) + |> Enum.flat_map(fn l -> + case Regex.run(~r/^\s*(?:-\s+)?["']?([^"':\s]+)["']?\s*:(.*)$/, l) do + [_, k, v] -> [{k, v |> wf_strip_comment() |> String.trim()}] + nil -> [] + end + end) + end + end + + # ─── flow-mapping reading (quote- and depth-aware) ────────────────── + + # Byte column of the innermost `{` still open at byte `at` of `line`, read + # from the start with quotes and comments respected; :error when `at` is + # not inside a flow mapping opened on this line (it is inside a quoted + # scalar, or no brace precedes it). + defp wf_flow_opener_before(line, at) do + line + |> binary_part(0, at) + |> :binary.bin_to_list() + |> wf_flow_prefix(0, nil, [], ?\s) + end + + # State: nil (plain text), ?" or ?' (inside that quote), :comment. The + # stack holds the byte column of each open `{` (`:seq` for a `[`); the + # innermost must be a mapping for `at` to be a key position. + defp wf_flow_prefix([], _pos, nil, [col | _], _prev) when is_integer(col), do: {:ok, col} + defp wf_flow_prefix([], _pos, _state, _stack, _prev), do: :error + + defp wf_flow_prefix([c | rest], pos, nil, stack, prev) do + cond do + c == ?# and prev in [?\s, ?\t] -> wf_flow_prefix(rest, pos + 1, :comment, stack, c) + c in [?", ?'] -> wf_flow_prefix(rest, pos + 1, c, stack, c) + c == ?{ -> wf_flow_prefix(rest, pos + 1, nil, [pos | stack], c) + c == ?[ -> wf_flow_prefix(rest, pos + 1, nil, [:seq | stack], c) + c in [?}, ?]] -> wf_flow_prefix(rest, pos + 1, nil, Enum.drop(stack, 1), c) + true -> wf_flow_prefix(rest, pos + 1, nil, stack, c) + end + end + + defp wf_flow_prefix([?\\, c | rest], pos, ?", stack, _prev), + do: wf_flow_prefix(rest, pos + 2, ?", stack, c) + + defp wf_flow_prefix([q | rest], pos, q, stack, _prev), + do: wf_flow_prefix(rest, pos + 1, nil, stack, q) + + defp wf_flow_prefix([c | rest], pos, state, stack, _prev), + do: wf_flow_prefix(rest, pos + 1, state, stack, c) + + # The interior of the flow collection whose opener sits at byte `open_col` + # of line `number` (1-based): the text between that brace and the one + # closing it, which may be lines later. Quoted scalars and comments are + # skipped for depth, comments are dropped from the text. :error when the + # document ends first. + defp wf_flow_interior(lines, number, open_col) do + first = Enum.at(lines, number - 1) + head = binary_part(first, open_col + 1, byte_size(first) - open_col - 1) + + text = + [head | Enum.drop(lines, number)] + |> Enum.join("\n") + |> :binary.bin_to_list() + + wf_flow_take(text, 1, nil, [], 0) + end + + defp wf_flow_take([], _depth, _state, _acc, _nl), do: :error + + defp wf_flow_take([c | rest], depth, nil, acc, nl) do + cond do + c == ?# and (acc == [] or hd(acc) in [?\s, ?\t, ?\n]) -> + wf_flow_take(rest, depth, :comment, acc, nl) + + c in [?", ?'] -> + wf_flow_take(rest, depth, c, [c | acc], nl) + + c in [?{, ?[] -> + wf_flow_take(rest, depth + 1, nil, [c | acc], nl) + + c in [?}, ?]] and depth == 1 -> + {:ok, acc |> Enum.reverse() |> List.to_string(), nl} + + c in [?}, ?]] -> + wf_flow_take(rest, depth - 1, nil, [c | acc], nl) + + c == ?\n -> + wf_flow_take(rest, depth, nil, [c | acc], nl + 1) + + true -> + wf_flow_take(rest, depth, nil, [c | acc], nl) + end + end + + defp wf_flow_take([?\\, c | rest], depth, ?", acc, nl), + do: wf_flow_take(rest, depth, ?", [c, ?\\ | acc], nl) + + defp wf_flow_take([?\n | rest], depth, :comment, acc, nl), + do: wf_flow_take(rest, depth, nil, [?\n | acc], nl + 1) + + defp wf_flow_take([_ | rest], depth, :comment, acc, nl), + do: wf_flow_take(rest, depth, :comment, acc, nl) + + defp wf_flow_take([q | rest], depth, q, acc, nl), + do: wf_flow_take(rest, depth, nil, [q | acc], nl) + + defp wf_flow_take([c | rest], depth, q, acc, nl), + do: wf_flow_take(rest, depth, q, [c | acc], nl) + + # The top-level `{key, value}` entries of a flow mapping's interior: + # commas inside nested `{}`/`[]` or quoted scalars do not split, and the + # key ends at the first top-level colon. + defp wf_flow_entries(interior) do + interior + |> wf_flow_split(?,) + |> Enum.map(&String.trim/1) + |> Enum.reject(&(&1 == "")) + |> Enum.map(fn entry -> + case wf_flow_split(entry, ?:) do + [key] -> {wf_unquote(key), ""} + [key | value] -> {wf_unquote(key), value |> Enum.join(":") |> String.trim()} end end) end + # Split at `sep` where it sits at depth 0 and outside quotes. + defp wf_flow_split(text, sep) do + text + |> :binary.bin_to_list() + |> wf_flow_split(sep, 0, nil, [], []) + end + + defp wf_flow_split([], _sep, _depth, _q, cur, out), + do: Enum.reverse([cur |> Enum.reverse() |> List.to_string() | out]) + + defp wf_flow_split([c | rest], sep, depth, nil, cur, out) do + cond do + c == sep and depth == 0 -> + wf_flow_split(rest, sep, depth, nil, [], [cur |> Enum.reverse() |> List.to_string() | out]) + + c in [?", ?'] -> + wf_flow_split(rest, sep, depth, c, [c | cur], out) + + c in [?{, ?[] -> + wf_flow_split(rest, sep, depth + 1, nil, [c | cur], out) + + c in [?}, ?]] -> + wf_flow_split(rest, sep, max(depth - 1, 0), nil, [c | cur], out) + + true -> + wf_flow_split(rest, sep, depth, nil, [c | cur], out) + end + end + + defp wf_flow_split([?\\, c | rest], sep, depth, ?", cur, out), + do: wf_flow_split(rest, sep, depth, ?", [c, ?\\ | cur], out) + + defp wf_flow_split([q | rest], sep, depth, q, cur, out), + do: wf_flow_split(rest, sep, depth, nil, [q | cur], out) + + defp wf_flow_split([c | rest], sep, depth, q, cur, out), + do: wf_flow_split(rest, sep, depth, q, [c | cur], out) + + # Drop a trailing ` # comment` that sits outside quotes. + defp wf_strip_comment(text) do + text + |> :binary.bin_to_list() + |> wf_strip_comment(nil, []) + end + + defp wf_strip_comment([], _q, acc), do: acc |> Enum.reverse() |> List.to_string() + + defp wf_strip_comment([?# | _], nil, acc) when acc == [] or hd(acc) in [?\s, ?\t], + do: acc |> Enum.reverse() |> List.to_string() + + defp wf_strip_comment([c | rest], nil, acc) when c in [?", ?'], + do: wf_strip_comment(rest, c, [c | acc]) + + defp wf_strip_comment([?\\, c | rest], ?", acc), do: wf_strip_comment(rest, ?", [c, ?\\ | acc]) + defp wf_strip_comment([q | rest], q, acc), do: wf_strip_comment(rest, nil, [q | acc]) + defp wf_strip_comment([c | rest], q, acc), do: wf_strip_comment(rest, q, [c | acc]) + + defp wf_unquote(text) do + text + |> String.trim() + |> String.replace(~r/^"(.*)"$/s, "\\1") + |> String.replace(~r/^'(.*)'$/s, "\\1") + |> String.trim() + end + # Column of a key, with a list dash counted as indentation (`- with:` is # the column of `with`). defp wf026_key_indent(line) do diff --git a/scripts/sweeps/estate-pin-integrity.sh b/scripts/sweeps/estate-pin-integrity.sh index 3e952ee9..f8bb64cc 100755 --- a/scripts/sweeps/estate-pin-integrity.sh +++ b/scripts/sweeps/estate-pin-integrity.sh @@ -28,7 +28,10 @@ # * `--via clone` is the authoritative mode (reads the files themselves); # `--via search` is the fast reconnaissance mode and can lag the index. # -# EXIT CODES: 0 clean, 1 error, 2 findings present, 3 nothing to do +# EXIT CODES: 0 clean, 1 error, 2 findings present (or the sweep could not +# see everything: a search that failed or overflowed, a +# candidate that could not be read — never reported clean), +# 3 nothing to do # # usage: estate-pin-integrity.sh [--org ORG]... [--policy FILE] [--out DIR] # [--via search|clone] [--rewrite] @@ -155,15 +158,23 @@ mapfile -t SEARCH_TOKENS < <(jq -r ' # of 100), whatever the true count. A broad query that overflows the cap # leaves candidates phase 2 never sees, so the sweep says so and refuses a # clean verdict: a search that could not see everything is INCOMPLETE, and -# only `--via clone` can clear the org. +# only `--via clone` can clear the org. A query that FAILS (expired auth, +# rate limit, outage: `gh api` exits non-zero) is the same case, not an +# empty page — zero results from a request that never answered is not a +# clean org. incomplete=0 +failed=0 if [ "$VIA" = "search" ]; then for org in "${ORGS[@]}"; do for token in "${SEARCH_TOKENS[@]}"; do page=1 while :; do - gh api "search/code?q=${token}+org:${org}&per_page=100&page=${page}" \ - > "$OUT_DIR/.search.json" 2>/dev/null || : > "$OUT_DIR/.search.json" + if ! gh api "search/code?q=${token}+org:${org}&per_page=100&page=${page}" \ + > "$OUT_DIR/.search.json" 2> "$OUT_DIR/.search.err"; then + log " ! ${org}: search for '${token}' (page ${page}) failed: $(head -c 200 "$OUT_DIR/.search.err" | tr '\n' ' ') — phase 1 is INCOMPLETE for this org, use --via clone" + failed=$((failed + 1)) + break + fi n=$(jq -r '.items[]? | "'"${org}"'\t\(.repository.name)\t\(.path)"' "$OUT_DIR/.search.json" 2>/dev/null \ | tee -a "$HITS" | wc -l) if [ "$page" -eq 1 ]; then @@ -204,9 +215,20 @@ fi total_hits=$(wc -l < "$HITS") log "phase 1: ${total_hits} candidate file(s)" +# What phase 1 could not see. A sweep that saw everything and found nothing +# is clean; one that could not see everything never is. +unseen() { + local parts=() + [ "$incomplete" -gt 0 ] && parts+=("${incomplete} search(es) overflowed GitHub's 1,000-result cap") + [ "$failed" -gt 0 ] && parts+=("${failed} search(es) failed") + [ "${unread:-0}" -gt 0 ] && parts+=("${unread} candidate file(s) could not be read") + local IFS=';' + printf '%s' "${parts[*]}" +} + if [ "$total_hits" -eq 0 ]; then - if [ "$incomplete" -gt 0 ]; then - echo "INCOMPLETE: no denylisted pin tokens in the visible search results for ${ORGS[*]}, but ${incomplete} search(es) overflowed GitHub's 1,000-result cap; a clean verdict needs --via clone" + if [ "$incomplete" -gt 0 ] || [ "$failed" -gt 0 ]; then + echo "INCOMPLETE: no denylisted pin tokens in the visible search results for ${ORGS[*]}, but $(unseen); a clean verdict needs --via clone" exit 2 fi echo "clean: no denylisted pin tokens found in ${ORGS[*]}" @@ -214,8 +236,14 @@ if [ "$total_hits" -eq 0 ]; then fi # ─── Phase 2: confirm, classify, and build the repaired tree ───────────── +# A phase-1 candidate is a file the search (or grep) associated with a +# token; for a broad action-path query that includes every healthy caller. +# Only a file in which a denylisted token is CONFIRMED is a finding, and +# only confirmed findings decide the exit code. repairable=0 flagged=0 +confirmed=0 +unread=0 while IFS=$'\t' read -r org repo path; do [ -n "${path:-}" ] || continue @@ -223,6 +251,7 @@ while IFS=$'\t' read -r org repo path; do content=$(gh api "repos/${org}/${repo}/contents/${path}?ref=HEAD" --jq '.content' 2>/dev/null | base64 -d 2>/dev/null || true) if [ -z "$content" ]; then log " ! cannot read ${org}/${repo}/${path} (binary, missing, or no permission)" + unread=$((unread + 1)) continue fi @@ -231,6 +260,7 @@ while IFS=$'\t' read -r org repo path; do sites_before=$(grep -cE '^[[:space:]]*-?[[:space:]]*uses:' "$OUT_DIR/.current" || true) denied_before=$(grep -cE "@(${REF_ALTERNATION})([[:space:]#]|$)" "$OUT_DIR/.current" || true) [ "$denied_before" -eq 0 ] && continue + confirmed=$((confirmed + 1)) # Scope guard: a repair is only meaningful where a pin is *live*. A # documentation code block quoting the poisoned SHA is a finding, not a @@ -310,6 +340,15 @@ while IFS=$'\t' read -r org repo path; do fi done < "$HITS" +if [ "$confirmed" -eq 0 ]; then + if [ "$incomplete" -gt 0 ] || [ "$failed" -gt 0 ] || [ "$unread" -gt 0 ]; then + echo "INCOMPLETE: none of the ${total_hits} candidate file(s) carries a denylisted pin, but $(unseen); a clean verdict needs --via clone" + exit 2 + fi + echo "clean: ${total_hits} candidate file(s) read, none carries a denylisted pin in ${ORGS[*]}" + exit 0 +fi + # One repair PR per repository. if [ "$repairable" -gt 0 ]; then jq -src ' @@ -331,6 +370,7 @@ fi log "" log "─── pin-integrity sweep ────────────────────────────────────────────" log " candidate files : ${total_hits}" +log " confirmed denylisted : ${confirmed}" log " mechanically repairable : ${repairable}" log " needs owner review : ${flagged}" log " repos with a repair ready : $(grep -c . "$PLAN" 2>/dev/null || echo 0)" @@ -339,6 +379,8 @@ log " repair plan : ${PLAN}" [ "$REWRITE" -eq 1 ] && log " repaired trees : ${OUT_DIR}/patched" log " NOTE: this sweep never writes to a remote. Branch + PR creation is" log " the token-bearing actuator's job — docs/operations/estate-automerge.adoc" -[ "$incomplete" -gt 0 ] && log " INCOMPLETE: ${incomplete} search(es) overflowed GitHub's 1,000-result cap; candidates beyond it were not seen — rerun with --via clone" +if [ "$incomplete" -gt 0 ] || [ "$failed" -gt 0 ] || [ "$unread" -gt 0 ]; then + log " INCOMPLETE: $(unseen); what they hid was not seen — rerun with --via clone" +fi exit 2 diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs index 835d3db2..188cab7f 100644 --- a/test/rules/workflow_audit_pins_test.exs +++ b/test/rules/workflow_audit_pins_test.exs @@ -281,6 +281,160 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) end + test "a KYAML workflow is judged as its block equivalent, line for line" do + # The literal shape of `yq -o kyaml` (standards YAML-POLICY Y-3): every + # value quoted, every mapping braced, step items opening with `{`. + wf = """ + { + jobs: { + check: { + runs-on: "ubuntu-latest", + steps: [ + { + uses: "dtolnay/rust-toolchain@#{@toolchain_sha}", + with: { + toolchain: "stable", + }, + }, + { + uses: "dtolnay/rust-toolchain@#{@toolchain_sha}", + }, + { uses: "dtolnay/rust-toolchain@#{@toolchain_sha}" }, + ], + }, + }, + } + """ + + assert [%{line: 13} = a, %{line: 15} = b] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + + # The repair script rewrites block-style steps only: a KYAML finding is + # real but routes to review, never to the fixer. + refute a.auto_fixable + assert a.fix_script == nil + assert a.reason =~ "KYAML" + refute b.auto_fixable + end + + test "a flow-style step inline in a block document is read through its braces" do + wf = """ + jobs: + check: + steps: + - { uses: dtolnay/rust-toolchain@#{@toolchain_sha}, with: { toolchain: stable } } + - { name: "rust", uses: "dtolnay/rust-toolchain@#{@toolchain_sha}" } + - { with: { components: "clippy, rustfmt" }, uses: dtolnay/rust-toolchain@#{@toolchain_sha} } + - run: echo "{ uses: dtolnay/rust-toolchain@#{@toolchain_sha} }" + """ + + assert [%{line: 5} = a, %{line: 6}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + + refute a.auto_fixable + assert a.reason =~ "flow style" + end + + test "steps: [ { uses: ... } ] on one line is a step; a flow list in env: is not" do + wf = """ + jobs: + check: + env: + fixture: [ { uses: dtolnay/rust-toolchain@#{@toolchain_sha} } ] + steps: [ { uses: dtolnay/rust-toolchain@#{@toolchain_sha} } ] + """ + + assert [%{line: 5, auto_fixable: false}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "a - { item spread over lines: its with entry decides, nested braces respected" do + wf = """ + jobs: + check: + steps: + - { + uses: dtolnay/rust-toolchain@#{@toolchain_sha}, + with: { components: "${{ matrix.component }}", toolchain: nightly }, + } + - { + uses: dtolnay/rust-toolchain@#{@toolchain_sha}, + with: { components: "${{ matrix.component }}" }, + } + - { + env: { uses: dtolnay/rust-toolchain@#{@toolchain_sha} }, + uses: dtolnay/rust-toolchain@#{@toolchain_sha}, + } + """ + + assert [%{line: 9, auto_fixable: false}, %{line: 14, auto_fixable: false}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "with: through an anchor or alias is resolved; the finding is not auto-fixable" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: &rust-inputs + toolchain: stable + components: clippy + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: *rust-inputs + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: &bare-inputs { components: clippy } + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: *bare-inputs + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + <<: *rust-inputs + components: rustfmt + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + <<: *bare-inputs + """ + + assert [%{line: 10} = a, %{line: 12} = b, %{line: 18} = c] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + + refute a.auto_fixable + assert a.reason =~ "anchor or alias" + refute b.auto_fixable + # A block `with:` whose merged mapping lacks the input is still the + # fixer's: an explicit `toolchain:` child overrides a merge key, so + # inserting one is a sound repair. + assert c.auto_fixable + end + + test "an alias with no anchor cannot be read: reported, never passed silently" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: *nowhere + """ + + assert [%{line: 4, auto_fixable: false} = f] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + + assert f.reason =~ "no `&nowhere` anchor" + end + + test "a plain block-style finding is still the fixer's" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: { components: clippy } # a flow mapping the fixer extends + """ + + assert [%{line: 4, auto_fixable: true, fix_script: "fix-rust-toolchain-input.sh"}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + test "the audit carries WF026 findings and a count" do wf = "jobs:\n a:\n steps:\n - uses: dtolnay/rust-toolchain@#{@toolchain_sha}\n" result = WorkflowAudit.audit(["rust.yml"], %{"rust.yml" => wf}) From e681d1f9e784840762038462d34f07ff32c6494e Mon Sep 17 00:00:00 2001 From: hyperpolymath Date: Mon, 19 Oct 2026 18:19:41 +0000 Subject: [PATCH 19/47] rules(WF026): a toolchain: key with no value (null, ~) is no input The fleet fixer (gitbot-fleet e3eca1c) fills an empty `toolchain:` key in place; the rule now agrees that `toolchain:`, `toolchain: null` and `toolchain: ~` leave the step without its input, in block and flow form, so the two sides describe the same defect. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- lib/rules/workflow_audit.ex | 6 +++++- test/rules/workflow_audit_pins_test.exs | 22 ++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/lib/rules/workflow_audit.ex b/lib/rules/workflow_audit.ex index d290cfd8..4021a6c9 100644 --- a/lib/rules/workflow_audit.ex +++ b/lib/rules/workflow_audit.ex @@ -1543,9 +1543,13 @@ defmodule Hypatia.Rules.WorkflowAudit do # Does a list of `{key, value}` entries name a toolchain? A `<<: *x` merge # key brings the anchored mapping's entries in. + # The key present with no value (`toolchain:`, `null`, `~`) is not an input + # the action can read; the fleet fixer fills such a key in place. defp wf026_entries_verdict(entries, lines, form, seen \\ []) do cond do - Enum.any?(entries, fn {k, v} -> k == "toolchain" and wf_unquote(v) != "" end) -> + Enum.any?(entries, fn {k, v} -> + k == "toolchain" and wf_unquote(v) not in ["", "null", "~"] + end) -> {:toolchain, form} true -> diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs index 188cab7f..ede997af 100644 --- a/test/rules/workflow_audit_pins_test.exs +++ b/test/rules/workflow_audit_pins_test.exs @@ -422,6 +422,28 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do assert f.reason =~ "no `&nowhere` anchor" end + test "a toolchain: key with no value is no input: null and ~ included" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + toolchain: + components: clippy + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: { components: clippy, toolchain: null } + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + toolchain: ~ # chosen later + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: { toolchain: "1.85.0" } + """ + + assert [%{line: 4, auto_fixable: true}, %{line: 8, auto_fixable: true}, %{line: 10}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + test "a plain block-style finding is still the fixer's" do wf = """ jobs: From 0c1610beb78dddfeed6678d0588fde944254483a Mon Sep 17 00:00:00 2001 From: hyperpolymath Date: Mon, 19 Oct 2026 18:26:28 +0000 Subject: [PATCH 20/47] rules(WF026): a SHA pin ends at a value boundary `@-patched` is a tag that merely begins with 40 hex characters; a tag carries its own channel, so the step is not reported. Both the block-style and the inline regex now require a quote, whitespace, `#`, `,`, `}`, `]` or the end of the line after the SHA. Mirrors gitbot-fleet's fixer (tc13). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- lib/rules/workflow_audit.ex | 6 ++++-- test/rules/workflow_audit_pins_test.exs | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/lib/rules/workflow_audit.ex b/lib/rules/workflow_audit.ex index 4021a6c9..72801f25 100644 --- a/lib/rules/workflow_audit.ex +++ b/lib/rules/workflow_audit.ex @@ -1112,11 +1112,13 @@ defmodule Hypatia.Rules.WorkflowAudit do # ─── WF026: SHA-pinned dtolnay/rust-toolchain with no toolchain: input ── - @toolchain_sha_pin ~r/^\s*(?:-\s+)?["']?uses["']?\s*:\s*["']?dtolnay\/rust-toolchain@([0-9a-f]{40})/ + # The 40 hex characters must end the ref: `@-patched` is a tag that + # merely begins with one, and a tag carries its own channel. + @toolchain_sha_pin ~r/^\s*(?:-\s+)?["']?uses["']?\s*:\s*["']?dtolnay\/rust-toolchain@([0-9a-f]{40})(?=["'\s#,}\]]|$)/ # The same key written inside a flow mapping on the line that opens it: # `- { uses: dtolnay/rust-toolchain@ }`, `steps: [ { uses: ... } ]`. - @toolchain_sha_pin_inline ~r/[{,]\s*["']?uses["']?\s*:\s*["']?dtolnay\/rust-toolchain@([0-9a-f]{40})/ + @toolchain_sha_pin_inline ~r/[{,]\s*["']?uses["']?\s*:\s*["']?dtolnay\/rust-toolchain@([0-9a-f]{40})(?=["'\s#,}\]]|$)/ @doc """ WF026: a SHA-pinned `dtolnay/rust-toolchain` step that passes no diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs index ede997af..7b1ebdc3 100644 --- a/test/rules/workflow_audit_pins_test.exs +++ b/test/rules/workflow_audit_pins_test.exs @@ -444,6 +444,20 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) end + test "a tag that merely begins with a SHA is a tag: it carries its channel" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha}-patched + - { uses: dtolnay/rust-toolchain@#{@toolchain_sha}-patched } + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} # pinned + """ + + assert [%{line: 6}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + test "a plain block-style finding is still the fixer's" do wf = """ jobs: From 74c7277845a46621fddc98442bfc41cbb29099a1 Mon Sep 17 00:00:00 2001 From: "coderabbitai[bot]" <136622811+coderabbitai[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:45:19 +0000 Subject: [PATCH 21/47] fix(security): harden shell paths and refresh Rust dependencies --- Cargo.lock | 16 ++++++++-------- hypatia-cli-bash.sh | 2 +- scripts/hypatia-sitrep.sh | 4 +--- scripts/ramdisk-orchestrator.sh | 6 +++--- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9d44e46d..d01f31a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -824,9 +824,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] @@ -1445,9 +1445,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.15" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +checksum = "a9f37a958b41b3b19ee2707c06439c0e9e547e847223eb791ecb0cb821c65e27" dependencies = [ "atomic-waker", "bytes", @@ -2861,9 +2861,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.16" +version = "0.11.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +checksum = "a9746dbde176634f4f2f1faf2404e30a31b2bc1e9cafb5329c95d8177a18c9fc" dependencies = [ "aws-lc-rs", "bytes", @@ -3213,9 +3213,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "aws-lc-rs", "once_cell", diff --git a/hypatia-cli-bash.sh b/hypatia-cli-bash.sh index 3bcf1cfc..491e5dea 100755 --- a/hypatia-cli-bash.sh +++ b/hypatia-cli-bash.sh @@ -331,7 +331,7 @@ scan_code_patterns() { # the banned-language filter above). run_pattern "$dir" "*.{js,jsx,mjs,res}" '\beval\s*\(' \ "critical" "code_injection" "eval_usage" "CWE-95" \ - "Replace eval() with JSON.parse, switch, or safe alternatives" + "Replace eval() with JSON.parse, switch, or safe alternatives" # hypatia: allow content_patterns/eval_in_shell -- remediation text, not shell execution # Dynamic Function() constructor run_pattern "$dir" "*.{js,jsx,mjs,res}" '\bFunction\s*\(' \ diff --git a/scripts/hypatia-sitrep.sh b/scripts/hypatia-sitrep.sh index 14648063..5772e56c 100755 --- a/scripts/hypatia-sitrep.sh +++ b/scripts/hypatia-sitrep.sh @@ -135,11 +135,9 @@ echo "$ALL_OUTPUT" | sed -n '/SECTION:GATE/,/SECTION:END/p' | grep -E "^\s+(Appr # --- 6. Top findings by frequency --- header "TOP PATTERNS (by repo count)" if [[ -f "$MANIFEST" ]]; then - jq -r '.pattern_id' "$MANIFEST" 2>/dev/null | sort | uniq -c | sort -rn | head -10 > /tmp/hypatia-top-patterns.txt || true while read -r count pid; do printf " %4d repos %s\n" "$count" "$pid" - done < /tmp/hypatia-top-patterns.txt - rm -f /tmp/hypatia-top-patterns.txt + done < <(jq -r '.pattern_id' "$MANIFEST" 2>/dev/null | sort | uniq -c | sort -rn | head -10 || true) else log " No dispatch manifest found" fi diff --git a/scripts/ramdisk-orchestrator.sh b/scripts/ramdisk-orchestrator.sh index bbf2e06f..8f7c3b57 100755 --- a/scripts/ramdisk-orchestrator.sh +++ b/scripts/ramdisk-orchestrator.sh @@ -19,7 +19,7 @@ # --push Push committed fixes to remotes # --dry-run Show what would be done without doing it # --keep Do not unmount RAM disk on completion -# --mount PATH Custom mount point (default: /tmp/hypatia-workspace) +# --mount PATH Custom mount point (default: per-user Hypatia workspace) # --max-repos N Limit number of repos to process # --fallback-disk Fall back to Eclipse drive if RAM is insufficient # --quiet Suppress non-essential output @@ -43,7 +43,7 @@ FARM_DIR="${FARM_DIR:-${HYPATIA_DIR}/../.git-private-farm}" ECLIPSE_REPOS="${HYPATIA_REPOS_DIR:-$(dirname "$HYPATIA_DIR")}" # Defaults -MOUNT_POINT="/tmp/hypatia-workspace" +MOUNT_POINT="${XDG_RUNTIME_DIR:-${XDG_STATE_HOME:-$HOME/.local/state}}/hypatia/workspace" RAMDISK_SIZE="2G" ORG="hyperpolymath" SEVERITY="high" @@ -176,7 +176,7 @@ mount_workspace() { if ! sudo -n true 2>/dev/null; then log_warn "No passwordless sudo — falling back to disk workspace" USE_TMPFS=false - MOUNT_POINT="/tmp/hypatia-workspace-$$" + MOUNT_POINT=$(mktemp -d) fi fi From 62a4a8c28460afc49dece72b7700f60b7b94b4d5 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 22:03:32 +0000 Subject: [PATCH 22/47] fix(rules): WF026 judges every inline step, YAML null spellings and explicit keys over merges; WF027 reads flow-style jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review round 7 on #931 (Codex): - WF026 inline path: every `{ uses: dtolnay/rust-toolchain@ }` on a physical line is judged, not only the first — `Regex.scan` over the inline pin, and the sequence-position test strips the closed sibling mappings before it. - YAML null is unquoted: `""`, `''`, `~`, `null`, `Null`, `NULL` and nothing are the key with no value; `"null"` quoted is the string null, an input the step names. The test is on the raw value now, not on the unquoted one. - An explicit `toolchain:` key wins over a `<<: *anchor` merge, empty or not (YAML merge semantics): `<<: *inputs` beside a bare `toolchain:` is the empty value whatever the anchor names. - Upper-case hex is the same commit: both WF026 regexes and the WF027 reusable regex accept `[0-9a-fA-F]{40}`; WF027 compares in lower case and reports the pin as written. - WF027: the 40 hex characters must end the ref (`@-patched` is a tag), and a job written as a flow mapping is a call — on its own line under `jobs:` (`rust-ci: { uses: ... }`) or inside a one-line `jobs: { ... }` mapping. A `run:` string carrying the same braces is not. Tests for each shape in test/rules/workflow_audit_pins_test.exs. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- lib/rules/workflow_audit.ex | 181 +++++++++++++++++------- test/rules/workflow_audit_pins_test.exs | 120 ++++++++++++++++ 2 files changed, 246 insertions(+), 55 deletions(-) diff --git a/lib/rules/workflow_audit.ex b/lib/rules/workflow_audit.ex index 72801f25..7ccf0bfd 100644 --- a/lib/rules/workflow_audit.ex +++ b/lib/rules/workflow_audit.ex @@ -1113,12 +1113,13 @@ defmodule Hypatia.Rules.WorkflowAudit do # ─── WF026: SHA-pinned dtolnay/rust-toolchain with no toolchain: input ── # The 40 hex characters must end the ref: `@-patched` is a tag that - # merely begins with one, and a tag carries its own channel. - @toolchain_sha_pin ~r/^\s*(?:-\s+)?["']?uses["']?\s*:\s*["']?dtolnay\/rust-toolchain@([0-9a-f]{40})(?=["'\s#,}\]]|$)/ + # merely begins with one, and a tag carries its own channel. Git resolves + # a commit ID in either case, so upper-case hex is the same pin. + @toolchain_sha_pin ~r/^\s*(?:-\s+)?["']?uses["']?\s*:\s*["']?dtolnay\/rust-toolchain@([0-9a-fA-F]{40})(?=["'\s#,}\]]|$)/ # The same key written inside a flow mapping on the line that opens it: # `- { uses: dtolnay/rust-toolchain@ }`, `steps: [ { uses: ... } ]`. - @toolchain_sha_pin_inline ~r/[{,]\s*["']?uses["']?\s*:\s*["']?dtolnay\/rust-toolchain@([0-9a-f]{40})(?=["'\s#,}\]]|$)/ + @toolchain_sha_pin_inline ~r/[{,]\s*["']?uses["']?\s*:\s*["']?dtolnay\/rust-toolchain@([0-9a-fA-F]{40})(?=["'\s#,}\]]|$)/ @doc """ WF026: a SHA-pinned `dtolnay/rust-toolchain` step that passes no @@ -1191,14 +1192,14 @@ defmodule Hypatia.Rules.WorkflowAudit do wf026_block_step_finding(filename, lines, line, number, sha, kyaml?) _ -> - case Regex.run(@toolchain_sha_pin_inline, line, return: :index) do - [{_, _}, {sha_at, 40}] -> - sha = binary_part(line, sha_at, 40) - wf026_inline_step_finding(filename, lines, line, number, sha, kyaml?) - - _ -> - [] - end + # Several flow-style steps may share one physical line + # (`steps: [ { uses: ... }, { uses: ... } ]`): each is judged. + @toolchain_sha_pin_inline + |> Regex.scan(line, return: :index) + |> Enum.flat_map(fn [{at, _}, {sha_at, 40}] -> + sha = binary_part(line, sha_at, 40) + wf026_inline_step_finding(filename, lines, line, number, at + 1, sha, kyaml?) + end) end end) end) @@ -1235,15 +1236,8 @@ defmodule Hypatia.Rules.WorkflowAudit do # The inline path: `uses:` sits inside a flow mapping on the line that # opens it. The mapping is read through its braces (quote-aware, across # lines) and its `with` entry decides. - defp wf026_inline_step_finding(filename, lines, line, number, sha, kyaml?) do - uses_at = - case Regex.run(@toolchain_sha_pin_inline, line, return: :index) do - [{at, _} | _] -> at + 1 - _ -> nil - end - - with true <- is_integer(uses_at), - {:ok, open_col} <- wf_flow_opener_before(line, uses_at), + defp wf026_inline_step_finding(filename, lines, line, number, uses_at, sha, kyaml?) do + with {:ok, open_col} <- wf_flow_opener_before(line, uses_at), true <- wf026_inline_item?(line, open_col), true <- wf026_inline_under_steps?(lines, line, number, open_col), {:ok, interior, _} <- wf_flow_interior(lines, number, open_col) do @@ -1259,11 +1253,20 @@ defmodule Hypatia.Rules.WorkflowAudit do # The text before the mapping's `{` must be a sequence position: a list # dash, the `steps:` key opening its sequence, a `[`, a comma, or nothing. # Anything else (`run: "echo { uses: ... }"`) is a scalar carrying braces. + # An earlier sibling on the same line (`[ { uses: a }, { uses: b } ]`) is + # a closed mapping before this one; it is removed before the test. defp wf026_inline_item?(line, open_col) do - prefix = binary_part(line, 0, open_col) + prefix = line |> binary_part(0, open_col) |> wf_drop_closed_mappings() Regex.match?(~r/^\s*(?:-\s*)?(?:["']?steps["']?\s*:\s*)?[\[,\s]*$/, prefix) end + defp wf_drop_closed_mappings(text) do + case Regex.replace(~r/\{[^{}]*\}/, text, "") do + ^text -> text + shorter -> wf_drop_closed_mappings(shorter) + end + end + defp wf026_inline_under_steps?(lines, line, number, open_col) do prefix = binary_part(line, 0, open_col) @@ -1544,16 +1547,21 @@ defmodule Hypatia.Rules.WorkflowAudit do end # Does a list of `{key, value}` entries name a toolchain? A `<<: *x` merge - # key brings the anchored mapping's entries in. + # key brings the anchored mapping's entries in, but an explicit key wins + # over a merged one (YAML merge semantics): `toolchain:` written empty + # beside `<<: *inputs` is the empty value, whatever the anchor names. # The key present with no value (`toolchain:`, `null`, `~`) is not an input # the action can read; the fleet fixer fills such a key in place. defp wf026_entries_verdict(entries, lines, form, seen \\ []) do + explicit = Enum.find(entries, fn {k, _} -> k == "toolchain" end) + cond do - Enum.any?(entries, fn {k, v} -> - k == "toolchain" and wf_unquote(v) not in ["", "null", "~"] - end) -> + match?({_, v} when is_binary(v), explicit) and not wf026_null_value?(elem(explicit, 1)) -> {:toolchain, form} + explicit != nil -> + {:missing, form} + true -> entries |> Enum.filter(fn {k, v} -> @@ -1570,6 +1578,19 @@ defmodule Hypatia.Rules.WorkflowAudit do end end + # YAML's null: nothing, `~`, or `null` in the three spellings the core + # schema admits, unquoted. `"null"` is the string null, an input the + # action receives; `""` and `''` are the empty string, which it does not. + defp wf026_null_value?(value) do + case String.trim(value) do + "" -> true + "~" -> true + ~s("") -> true + "''" -> true + bare -> bare in ["null", "Null", "NULL"] + end + end + # An alias that reaches itself (`&x` whose mapping merges `*x`) is as # unreadable as one with no anchor. defp wf026_alias_verdict(name, lines, form, seen \\ []) do @@ -1970,46 +1991,96 @@ defmodule Hypatia.Rules.WorkflowAudit do Enum.flat_map(@broken_reusable_revisions, fn rev -> # Anchored to a `uses:` key at the start of a line (quoted or not) that # is a key of a job under `jobs:` — the only place a reusable is - # called. The reusable's path inside a `run: |` script, under `env:`, - # in a string value or in a comment is not a call, so the fleet is - # never sent to rewrite script data. + # called — or to the same key inside the flow mapping a job opens on + # its own line (`rust: { uses: ... }`). The reusable's path inside a + # `run: |` script, under `env:`, in a string value or in a comment is + # not a call, so the fleet is never sent to rewrite script data. + # The 40 hex characters end the ref (`@-patched` is a tag), in + # either case: git resolves the commit ID case-insensitively. + reusable = Regex.escape(rev.reusable) + ref_re = - ~r/^\s*(?:-\s+)?["']?uses["']?\s*:\s*["']?#{Regex.escape(rev.reusable)}@([0-9a-f]{40})/ + ~r/^\s*(?:-\s+)?["']?uses["']?\s*:\s*["']?#{reusable}@([0-9a-fA-F]{40})(?=["'\s#,}\]]|$)/ + + inline_re = + ~r/[{,]\s*["']?uses["']?\s*:\s*["']?#{reusable}@([0-9a-fA-F]{40})(?=["'\s#,}\]]|$)/ lines |> Enum.with_index(1) |> Enum.flat_map(fn {line, number} -> - with [_full, sha] <- Regex.run(ref_re, line), - true <- sha in rev.shas, - [_job, "jobs" | _] <- wf_ancestor_keys(lines, number) do - [ - %{ - rule: "WF027", - type: :known_broken_reusable_revision, - file: filename, - line: number, - reusable: rev.reusable, - current_sha: sha, - replacement_sha: rev.replacement, - severity: :medium, - auto_fixable: true, - fix_script: "fix-stale-reusable-pin.sh", - reason: - "`#{rev.reusable}` is pinned at `@#{sha}`, inside #{rev.window}: " <> - rev.reason <> - " Fixed by #{rev.fixed_by}; move the pin to " <> - "`@#{rev.replacement}`.", - action: :rewrite_stale_reusable_pin - } - ] - else - _ -> [] - end + shas = + case Regex.run(ref_re, line) do + [_full, sha] -> + if match?([_job, "jobs" | _], wf_ancestor_keys(lines, number)), + do: [sha], + else: [] + + nil -> + inline_re + |> Regex.scan(line, return: :index) + |> Enum.filter(fn [{at, _} | _] -> + wf027_inline_job?(lines, line, number, at + 1) + end) + |> Enum.map(fn [_, {sha_at, 40}] -> binary_part(line, sha_at, 40) end) + end + + Enum.flat_map(shas, fn sha -> + if String.downcase(sha) in rev.shas do + [wf027_finding(filename, number, sha, rev)] + else + [] + end + end) end) end) end) end + # Is the `uses` at `uses_at` the entry of a flow mapping a job key opens? + # Either the job's own line under `jobs:` (`rust: { uses: ... }`) or the + # whole `jobs:` mapping written on one line (`jobs: { rust: { uses: ... } }`). + # Earlier siblings on the line (`a: { ... }, `) are closed entries before + # this one; they are removed before the key opening this mapping is read. + defp wf027_inline_job?(lines, line, number, uses_at) do + with {:ok, open_col} <- wf_flow_opener_before(line, uses_at), + prefix = line |> binary_part(0, open_col) |> wf_drop_closed_mappings(), + prefix = Regex.replace(~r/["']?[\w.-]+["']?\s*:\s*[^,{}]*,\s*/, prefix, ""), + [_, lead, _job] <- + Regex.run( + ~r/^(\s*(?:["']?jobs["']?\s*:\s*\{\s*,?\s*)?)["']?([\w.-]+)["']?\s*:\s*$/, + prefix + ) do + if String.contains?(lead, "jobs") do + wf_ancestor_keys(lines, number) == [] + else + match?(["jobs" | _], wf_ancestor_keys(lines, number)) + end + else + _ -> false + end + end + + defp wf027_finding(filename, number, sha, rev) do + %{ + rule: "WF027", + type: :known_broken_reusable_revision, + file: filename, + line: number, + reusable: rev.reusable, + current_sha: sha, + replacement_sha: rev.replacement, + severity: :medium, + auto_fixable: true, + fix_script: "fix-stale-reusable-pin.sh", + reason: + "`#{rev.reusable}` is pinned at `@#{sha}`, inside #{rev.window}: " <> + rev.reason <> + " Fixed by #{rev.fixed_by}; move the pin to " <> + "`@#{rev.replacement}`.", + action: :rewrite_stale_reusable_pin + } + end + # ─── WF017: Secret-consuming action without secret-presence gate ────── # Actions whose primary input is a secret and that fail outright when diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs index 7b1ebdc3..4ba1b00f 100644 --- a/test/rules/workflow_audit_pins_test.exs +++ b/test/rules/workflow_audit_pins_test.exs @@ -444,6 +444,80 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) end + test "YAML null is unquoted: \"null\" is a string, Null/NULL/\"\"/'' are not inputs" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: { toolchain: "null" } + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: { toolchain: Null } + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + toolchain: NULL + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + toolchain: "" + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: { toolchain: '' } + """ + + assert [%{line: 6}, %{line: 8}, %{line: 11}, %{line: 14}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "an explicit toolchain: key wins over a merged anchor, empty or not" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: &inputs { toolchain: stable, components: clippy } + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + <<: *inputs + toolchain: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: { <<: *inputs, toolchain: null } + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: { <<: *inputs, toolchain: nightly } + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: { <<: *inputs } + """ + + assert [%{line: 6, auto_fixable: true}, %{line: 10, auto_fixable: true}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "every flow-style step sharing a line is judged, not only the first" do + wf = """ + jobs: + check: + steps: [ { uses: dtolnay/rust-toolchain@#{@toolchain_sha}, with: { toolchain: stable } }, { uses: dtolnay/rust-toolchain@#{@toolchain_sha} } ] + other: + steps: [ { uses: dtolnay/rust-toolchain@#{@toolchain_sha} }, { uses: dtolnay/rust-toolchain@#{@toolchain_sha}, with: { components: clippy } } ] + """ + + assert [%{line: 3, auto_fixable: false}, %{line: 5}, %{line: 5}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "upper-case hex is the same commit: it fires like lower-case" do + sha = String.upcase(@toolchain_sha) + + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{sha} + - { uses: dtolnay/rust-toolchain@#{sha} } + """ + + assert [%{line: 4, current_sha: ^sha}, %{line: 5}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + test "a tag that merely begins with a SHA is a tag: it carries its channel" do wf = """ jobs: @@ -565,6 +639,52 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => wf}) end + test "a tag that merely begins with a broken SHA is a tag, not the pin" do + wf = """ + jobs: + rust-ci: + uses: #{@rust_ci}@#{@broken_standards}-patched + other: + uses: #{@rust_ci}@#{@broken_standards} + """ + + assert [%{rule: "WF027", line: 5}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => wf}) + end + + test "upper-case hex is the same revision: it fires, reported as written" do + sha = String.upcase(@broken_standards) + wf = "jobs:\n rust-ci:\n uses: #{@rust_ci}@#{sha}\n" + + assert [%{rule: "WF027", current_sha: ^sha, replacement_sha: @good_standards}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => wf}) + end + + test "a job written as a flow mapping is a call: on its own line or inside jobs: { }" do + wf = """ + jobs: + rust-ci: { uses: #{@rust_ci}@#{@broken_standards}, secrets: inherit } + note: + steps: + - run: echo "{ uses: #{@rust_ci}@#{@broken_standards} }" + """ + + assert [%{rule: "WF027", line: 2, current_sha: @broken_standards}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => wf}) + + one_line = + "jobs: { a: { uses: #{@rust_ci}@#{@good_standards} }, b: { uses: #{@rust_ci}@#{@broken_standards} } }\n" + + assert [%{rule: "WF027", line: 1, current_sha: @broken_standards}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => one_line}) + + nested_under_env = + "jobs:\n fixture:\n env: { uses: #{@rust_ci}@#{@broken_standards} }\n" + + assert [] = + WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => nested_under_env}) + end + test "the audit carries WF027 findings and a count" do wf = "jobs:\n rust-ci:\n uses: #{@rust_ci}@#{@broken_standards}\n" result = WorkflowAudit.audit(["rust-ci.yml"], %{"rust-ci.yml" => wf}) From 6e895b11ec5f7699640dbe7abaab8f7032317c85 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 22:08:47 +0000 Subject: [PATCH 23/47] test(rules): WF027 reads a KYAML caller as written MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The estate is moving to KYAML (standards YAML-POLICY Y-3). WF026 already judges a KYAML document as its block equivalent; this pins the same for WF027 without a rewrite: the quoted job-level `uses: "…@",` under `jobs: {` fires at its own line, and the same text in an `env:` string or a `run:` value inside the document does not. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- test/rules/workflow_audit_pins_test.exs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs index 4ba1b00f..14ee22ee 100644 --- a/test/rules/workflow_audit_pins_test.exs +++ b/test/rules/workflow_audit_pins_test.exs @@ -685,6 +685,31 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => nested_under_env}) end + test "a KYAML caller is read as written: the quoted job-level uses: fires, fixtures do not" do + wf = """ + { + name: "Rust", + on: { push: { branches: ["main"] } }, + jobs: { + note: { + "runs-on": "ubuntu-latest", + env: { FIXTURE: "uses: #{@rust_ci}@#{@broken_standards}" }, + steps: [{ + run: "echo uses: #{@rust_ci}@#{@broken_standards}", + }], + }, + "rust-ci": { + uses: "#{@rust_ci}@#{@broken_standards}", + secrets: "inherit", + }, + }, + } + """ + + assert [%{rule: "WF027", line: 13, current_sha: @broken_standards, auto_fixable: true}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => wf}) + end + test "the audit carries WF027 findings and a count" do wf = "jobs:\n rust-ci:\n uses: #{@rust_ci}@#{@broken_standards}\n" result = WorkflowAudit.audit(["rust-ci.yml"], %{"rust-ci.yml" => wf}) From 2a7a351496768cc69bd247c708f2b00171d7ec82 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 22:23:58 +0000 Subject: [PATCH 24/47] fix(rules,sweeps): case-fold SHA refs, quote-aware flow-job detection, and a sweep that confirms quoted pins and partial searches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review round 8 on #931 (Codex): - PI001 and PA001 compare refs in lower case when they are 40 hex characters: an upper-case spelling of a denylisted commit is the same object, so the repository scan and the automerge poison check both see it (pin_integrity.ex, pr_automerge.ex; tests in both suites). - WF027's flow-style job detection reads the openers still unclosed before `uses:` with the quote-aware walker and takes the opening key from the last comma-split entry, so a sibling's `name: "CI, build"` or a quoted brace separates nothing. The comma regex is gone. - estate-pin-integrity.sh: a search page marked `incomplete_results` (GitHub timed the search out and answered with a partial page, HTTP 200) is INCOMPLETE on any page, never a clean verdict; a denylisted pin is confirmed, counted and rewritten whatever follows the SHA — a closing quote, a flow delimiter, whitespace, a comment or the end of the line (REF_END, one definition for every grep and the sed); the action before the `@` is read through quotes and flow braces, and the pin-site counts before and after the rewrite use the same key test. Verified offline with a fake `gh`: quoted and flow-style poisoned pins → confirmed 1, repairable 1, both rewritten in place; a partial page → exit 2 INCOMPLETE; a healthy org → exit 0. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- lib/rules/pin_integrity.ex | 13 ++-- lib/rules/pr_automerge.ex | 5 +- lib/rules/workflow_audit.ex | 82 ++++++++++++++++++++----- scripts/sweeps/estate-pin-integrity.sh | 35 ++++++++--- test/rules/pin_integrity_test.exs | 4 ++ test/rules/pr_automerge_test.exs | 13 ++++ test/rules/workflow_audit_pins_test.exs | 19 ++++++ 7 files changed, 137 insertions(+), 34 deletions(-) diff --git a/lib/rules/pin_integrity.ex b/lib/rules/pin_integrity.ex index cd057d66..26fd9328 100644 --- a/lib/rules/pin_integrity.ex +++ b/lib/rules/pin_integrity.ex @@ -140,13 +140,12 @@ defmodule Hypatia.Rules.PinIntegrity do defp normalise_all(list), do: Enum.map(list, &normalise_ref/1) defp normalise_ref(ref) do - ref - |> String.trim() - |> String.trim_leading("v") - |> case do - # `v4.38.1` and `4.38.1` are the same claim about the same artefact. - other -> other - end + trimmed = ref |> String.trim() |> String.trim_leading("v") + + # `v4.38.1` and `4.38.1` are the same claim about the same artefact; a + # commit ID in upper-case hex is the same object as the lower-case one + # the table carries, so both sides compare in lower case. + if Regex.match?(~r/^[0-9a-fA-F]{40}$/, trimmed), do: String.downcase(trimmed), else: trimmed end @doc """ diff --git a/lib/rules/pr_automerge.ex b/lib/rules/pr_automerge.ex index bb287308..c2567cd7 100644 --- a/lib/rules/pr_automerge.ex +++ b/lib/rules/pr_automerge.ex @@ -166,7 +166,10 @@ defmodule Hypatia.Rules.PrAutomerge do end defp normalise_ref(ref) when is_binary(ref) do - ref |> String.trim() |> String.replace_prefix("v", "") + trimmed = ref |> String.trim() |> String.replace_prefix("v", "") + # A commit ID in upper-case hex is the same object the table carries in + # lower case; git resolves either spelling. + if Regex.match?(~r/^[0-9a-fA-F]{40}$/, trimmed), do: String.downcase(trimmed), else: trimmed end defp normalise_ref(_), do: "" diff --git a/lib/rules/workflow_audit.ex b/lib/rules/workflow_audit.ex index 7ccf0bfd..c0ea8ad9 100644 --- a/lib/rules/workflow_audit.ex +++ b/lib/rules/workflow_audit.ex @@ -1683,6 +1683,39 @@ defmodule Hypatia.Rules.WorkflowAudit do # State: nil (plain text), ?" or ?' (inside that quote), :comment. The # stack holds the byte column of each open `{` (`:seq` for a `[`); the # innermost must be a mapping for `at` to be a key position. + # Every `{`/`[` still open before byte `at` of `line`, innermost first: + # the column of each `{`, `:seq` for a `[`. Quoted scalars and comments + # are skipped, as in wf_flow_opener_before/2; :error inside one. + defp wf_flow_opener_stack(line, at) do + line + |> binary_part(0, at) + |> :binary.bin_to_list() + |> wf_flow_stack(0, nil, [], ?\s) + end + + defp wf_flow_stack([], _pos, nil, stack, _prev), do: {:ok, stack} + defp wf_flow_stack([], _pos, _state, _stack, _prev), do: :error + + defp wf_flow_stack([c | rest], pos, nil, stack, prev) do + cond do + c == ?# and prev in [?\s, ?\t] -> wf_flow_stack(rest, pos + 1, :comment, stack, c) + c in [?", ?'] -> wf_flow_stack(rest, pos + 1, c, stack, c) + c == ?{ -> wf_flow_stack(rest, pos + 1, nil, [pos | stack], c) + c == ?[ -> wf_flow_stack(rest, pos + 1, nil, [:seq | stack], c) + c in [?}, ?]] -> wf_flow_stack(rest, pos + 1, nil, Enum.drop(stack, 1), c) + true -> wf_flow_stack(rest, pos + 1, nil, stack, c) + end + end + + defp wf_flow_stack([?\\, c | rest], pos, ?", stack, _prev), + do: wf_flow_stack(rest, pos + 2, ?", stack, c) + + defp wf_flow_stack([q | rest], pos, q, stack, _prev), + do: wf_flow_stack(rest, pos + 1, nil, stack, q) + + defp wf_flow_stack([c | rest], pos, state, stack, _prev), + do: wf_flow_stack(rest, pos + 1, state, stack, c) + defp wf_flow_prefix([], _pos, nil, [col | _], _prev) when is_integer(col), do: {:ok, col} defp wf_flow_prefix([], _pos, _state, _stack, _prev), do: :error @@ -2039,24 +2072,39 @@ defmodule Hypatia.Rules.WorkflowAudit do # Is the `uses` at `uses_at` the entry of a flow mapping a job key opens? # Either the job's own line under `jobs:` (`rust: { uses: ... }`) or the # whole `jobs:` mapping written on one line (`jobs: { rust: { uses: ... } }`). - # Earlier siblings on the line (`a: { ... }, `) are closed entries before - # this one; they are removed before the key opening this mapping is read. + # The openers still unclosed before `uses` say which: one, opened by a key + # on a line under `jobs:`; two, the outer one opened by `jobs:` at the top + # of the document. The opening key is the last entry before its brace, + # split quote-aware, so a sibling's `name: "CI, build"` separates nothing. defp wf027_inline_job?(lines, line, number, uses_at) do - with {:ok, open_col} <- wf_flow_opener_before(line, uses_at), - prefix = line |> binary_part(0, open_col) |> wf_drop_closed_mappings(), - prefix = Regex.replace(~r/["']?[\w.-]+["']?\s*:\s*[^,{}]*,\s*/, prefix, ""), - [_, lead, _job] <- - Regex.run( - ~r/^(\s*(?:["']?jobs["']?\s*:\s*\{\s*,?\s*)?)["']?([\w.-]+)["']?\s*:\s*$/, - prefix - ) do - if String.contains?(lead, "jobs") do - wf_ancestor_keys(lines, number) == [] - else - match?(["jobs" | _], wf_ancestor_keys(lines, number)) - end - else - _ -> false + case wf_flow_opener_stack(line, uses_at) do + {:ok, [open]} when is_integer(open) -> + wf027_key_before(line, 0, open) != nil and + match?(["jobs" | _], wf_ancestor_keys(lines, number)) + + {:ok, [open, outer]} when is_integer(open) and is_integer(outer) -> + wf027_key_before(line, 0, outer) == "jobs" and + wf027_key_before(line, outer + 1, open) != nil and + wf_ancestor_keys(lines, number) == [] + + _ -> + false + end + end + + # The key whose value is the flow mapping opening at byte `open`: the last + # top-level entry of `line[from, open)` must read `key:` and nothing else. + defp wf027_key_before(line, from, open) do + entry = + line + |> binary_part(from, open - from) + |> wf_flow_split(?,) + |> List.last("") + |> String.trim() + + case Regex.run(~r/^["']?([\w.-]+)["']?\s*:$/, entry) do + [_, key] -> key + _ -> nil end end diff --git a/scripts/sweeps/estate-pin-integrity.sh b/scripts/sweeps/estate-pin-integrity.sh index f8bb64cc..5b303ee3 100755 --- a/scripts/sweeps/estate-pin-integrity.sh +++ b/scripts/sweeps/estate-pin-integrity.sh @@ -101,6 +101,14 @@ REF_ALTERNATION=$( } | sed 's/\./\\./g' | paste -sd'|' - ) +# What may follow a ref token: a closing quote (`uses: "…@sha"`), a flow +# delimiter (`{ uses: …@sha }`, `…@sha,`), whitespace, a comment, or the end +# of the line. A pin written in any of these forms is the same pin. +REF_END='([]"'"'"'[:space:]#,}]|$)' +# The `uses` key, quoted or bare, with the quote a quoted value opens with: +# `uses: x@…`, `"uses": "x@…"`, `{ uses: x@… }` all name an action. +USES_KEY='["'"'"']?uses["'"'"']?[[:space:]]*:[[:space:]]*["'"'"']?' + known_good_sha_for() { jq -r --arg a "$1" '.pin_denylist[] | select(.action == $a) | .known_good_sha' "$POLICY" | head -n1 } @@ -177,6 +185,12 @@ if [ "$VIA" = "search" ]; then fi n=$(jq -r '.items[]? | "'"${org}"'\t\(.repository.name)\t\(.path)"' "$OUT_DIR/.search.json" 2>/dev/null \ | tee -a "$HITS" | wc -l) + # GitHub answers a timed-out search with HTTP 200, the matches found + # so far and `incomplete_results: true` — a partial page, on any page. + if [ "$(jq -r '.incomplete_results // false' "$OUT_DIR/.search.json" 2>/dev/null)" = "true" ]; then + log " ! ${org}: '${token}' (page ${page}) is marked incomplete_results: GitHub timed the search out and answered with a partial page — phase 1 is INCOMPLETE for this org, use --via clone" + incomplete=$((incomplete + 1)) + fi if [ "$page" -eq 1 ]; then total=$(jq -r '.total_count // 0' "$OUT_DIR/.search.json" 2>/dev/null || echo 0) if [ "${total:-0}" -gt 1000 ] 2>/dev/null; then @@ -219,7 +233,7 @@ log "phase 1: ${total_hits} candidate file(s)" # is clean; one that could not see everything never is. unseen() { local parts=() - [ "$incomplete" -gt 0 ] && parts+=("${incomplete} search(es) overflowed GitHub's 1,000-result cap") + [ "$incomplete" -gt 0 ] && parts+=("${incomplete} search page(s) overflowed GitHub's 1,000-result cap or came back marked incomplete") [ "$failed" -gt 0 ] && parts+=("${failed} search(es) failed") [ "${unread:-0}" -gt 0 ] && parts+=("${unread} candidate file(s) could not be read") local IFS=';' @@ -257,8 +271,8 @@ while IFS=$'\t' read -r org repo path; do printf '%s\n' "$content" > "$OUT_DIR/.current" - sites_before=$(grep -cE '^[[:space:]]*-?[[:space:]]*uses:' "$OUT_DIR/.current" || true) - denied_before=$(grep -cE "@(${REF_ALTERNATION})([[:space:]#]|$)" "$OUT_DIR/.current" || true) + sites_before=$(grep -cE "(^|[{,[:space:]])${USES_KEY}" "$OUT_DIR/.current" || true) + denied_before=$(grep -cE "@(${REF_ALTERNATION})(${REF_END})" "$OUT_DIR/.current" || true) [ "$denied_before" -eq 0 ] && continue confirmed=$((confirmed + 1)) @@ -279,8 +293,11 @@ while IFS=$'\t' read -r org repo path; do [ "$in_scope" -eq 0 ] && fixable=0 : > "$OUT_DIR/.rewritten" while IFS= read -r line || [ -n "$line" ]; do - if printf '%s\n' "$line" | grep -qE "@(${REF_ALTERNATION})([[:space:]#]|$)"; then - action=$(printf '%s\n' "$line" | sed -E 's/^[[:space:]]*-?[[:space:]]*uses:[[:space:]]*([^@]+)@.*/\1/') + if printf '%s\n' "$line" | grep -qE "@(${REF_ALTERNATION})(${REF_END})"; then + # The action named before the `@`, whatever quotes or flow braces + # surround the key and the value. + action=$(printf '%s\n' "$line" | grep -oE "${USES_KEY}[^@\"'[:space:]]+@" | head -n1 \ + | sed -E "s/^${USES_KEY}//; s/@\$//") base=$(printf '%s\n' "$action" | awk -F/ '{print $1"/"$2}') good=$(known_good_sha_for "$base") actions="${actions}${base} " @@ -289,7 +306,7 @@ while IFS=$'\t' read -r org repo path; do printf '%s\n' "$line" >> "$OUT_DIR/.rewritten" else good_version=$(known_good_version_for "$base") - swapped=$(printf '%s\n' "$line" | sed -E "s%@(${REF_ALTERNATION})([[:space:]#]|$)%@${good}\2%g") + swapped=$(printf '%s\n' "$line" | sed -E "s%@(${REF_ALTERNATION})(${REF_END})%@${good}\2%g") head="${swapped%%#*}" if [ "$head" != "$swapped" ]; then tail_comment="#${swapped#*#}" @@ -303,8 +320,8 @@ while IFS=$'\t' read -r org repo path; do fi done < "$OUT_DIR/.current" - sites_after=$(grep -cE '^[[:space:]]*-?[[:space:]]*uses:' "$OUT_DIR/.rewritten" || true) - denied_after=$(grep -cE "@(${REF_ALTERNATION})([[:space:]#]|$)" "$OUT_DIR/.rewritten" || true) + sites_after=$(grep -cE "(^|[{,[:space:]])${USES_KEY}" "$OUT_DIR/.rewritten" || true) + denied_after=$(grep -cE "@(${REF_ALTERNATION})(${REF_END})" "$OUT_DIR/.rewritten" || true) status="flag" reason="owner_review" @@ -322,7 +339,7 @@ while IFS=$'\t' read -r org repo path; do jq -cn \ --arg repo "${org}/${repo}" --arg path "$path" --arg status "$status" \ - --arg reason "$reason" --arg actions "$(printf '%s' "$actions" | xargs || true)" \ + --arg reason "$reason" --arg actions "$(printf '%s' "$actions" | tr -s '[:space:]' ' ' | sed -E 's/^ //; s/ $//')" \ --argjson denied "$denied_before" --argjson sites "$sites_before" \ '{repo:$repo, path:$path, rule:"PI001", status:$status, repair:$reason, actions:($actions|split(" ")|map(select(.!=""))), diff --git a/test/rules/pin_integrity_test.exs b/test/rules/pin_integrity_test.exs index 69ad371d..b3aa5a81 100644 --- a/test/rules/pin_integrity_test.exs +++ b/test/rules/pin_integrity_test.exs @@ -96,6 +96,10 @@ defmodule Hypatia.Rules.PinIntegrityTest do PI.denylisted?(@policy, "github/codeql-action/analyze", "v4.38.1") assert %{"id" => "PIN-001"} = PI.denylisted?(@policy, "github/codeql-action/init", "4.38.1") + + # Upper-case hex is the same commit; the table is written in lower case. + assert %{"id" => "PIN-001"} = + PI.denylisted?(@policy, "github/codeql-action/init", String.upcase(@poison)) end test "does not match a good ref, another action, or another repo's entry" do diff --git a/test/rules/pr_automerge_test.exs b/test/rules/pr_automerge_test.exs index cd7dc93b..4257867b 100644 --- a/test/rules/pr_automerge_test.exs +++ b/test/rules/pr_automerge_test.exs @@ -162,6 +162,19 @@ defmodule Hypatia.Rules.PrAutomergeTest do assert decision.pool == "P1" end + test "an upper-case spelling of a denylisted SHA is the same poison" do + decision = + Hypatia.Rules.PrAutomerge.classify( + pr([ + file(".github/workflows/codeql.yml", codeql_patch(@good, String.upcase(@poison))) + ]), + @policy + ) + + assert decision.disposition == :close_poison_only + assert decision.blocked_by == "introduces_denylisted_pin" + end + test "a quoted uses: key and a quoted value are still a pin PA001 sees" do patch = """ @@ -1,1 +1,1 @@ diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs index 14ee22ee..ca800b50 100644 --- a/test/rules/workflow_audit_pins_test.exs +++ b/test/rules/workflow_audit_pins_test.exs @@ -685,6 +685,25 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => nested_under_env}) end + test "a quoted comma or brace in a sibling entry does not hide a flow-style job" do + one_line = + "jobs: { a: { name: \"x, y }\", uses: #{@rust_ci}@#{@good_standards} }, " <> + "b: { uses: #{@rust_ci}@#{@broken_standards} } }\n" + + assert [%{rule: "WF027", line: 1, current_sha: @broken_standards}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => one_line}) + + own_line = """ + jobs: + rust-ci: { name: "CI, build", uses: #{@rust_ci}@#{@broken_standards} } + note: + env: { name: "x, y", uses: #{@rust_ci}@#{@broken_standards} } + """ + + assert [%{rule: "WF027", line: 2}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => own_line}) + end + test "a KYAML caller is read as written: the quoted job-level uses: fires, fixtures do not" do wf = """ { From caab1dfb49bbf024d4887bdfe65bfd4cbbfd2e79 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 22:41:16 +0000 Subject: [PATCH 25/47] fix(sweeps,rules): judge pins per action/ref site, case-insensitively; flow delimiters end a PA001 ref MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review round 9 on #931 (Codex), all in estate-pin-integrity.sh unless noted: - A token is denied for ITS action: a blocked version is confirmed only on a `uses:` site whose action the policy entry names (`some/action@ v4.38.1` is no finding when only codeql-action's 4.38.1 is blocked); a denylisted SHA is one object and is confirmed in any position. - Matching is case-insensitive end to end (confirmation, the clone-mode grep, each site's rewrite): an upper-case SHA is the same commit. - Each `uses:` site on a line is rewritten with the known-good SHA for its own action — two actions on one flow line get two replacements — and the site count is unchanged by construction. - A denylisted token with no `uses:` key on its line (a lock record, a fixture) is confirmed and flagged `denylisted_token_outside_uses_key`; it no longer aborts the whole sweep under `set -e`. - The comment relabelled after a repair is found outside quoted scalars: `name: "C# v4.38.1 build"` is not a comment. - `--via clone`: a repository listing or a clone that fails counts as `failed` (INCOMPLETE, never clean), and clones live under the organisation so two organisations' same-named repositories never share a checkout. - `--sample N`, advertised in the usage, now exists: phase 2 reads the first N candidates and the sweep is INCOMPLETE by construction. - pr_automerge.ex: PA001's ref ends at a flow delimiter as well as a quote, space or comment, and a `- { uses: … }` step is a pin; test added. Verified offline with the fake `gh`: another action at a blocked version → clean; a lock record carrying the SHA → confirmed, flagged, sweep completes; one flow line with two actions → each replaced with its own known-good; upper-case SHA → confirmed and repaired; a quoted `C#` name beside a pin → name untouched, real comment relabelled; `--sample 1` of 2 → INCOMPLETE. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- lib/rules/pr_automerge.ex | 6 +- scripts/sweeps/estate-pin-integrity.sh | 168 ++++++++++++++++++++----- test/rules/pr_automerge_test.exs | 13 ++ 3 files changed, 151 insertions(+), 36 deletions(-) diff --git a/lib/rules/pr_automerge.ex b/lib/rules/pr_automerge.ex index c2567cd7..cf7c8b8f 100644 --- a/lib/rules/pr_automerge.ex +++ b/lib/rules/pr_automerge.ex @@ -459,8 +459,10 @@ defmodule Hypatia.Rules.PrAutomerge do # - - uses: actions/checkout@v4.1.7 # so the marker is part of the match. (Without this, every real diff looks # like a non-pin change and every PR falls through to `flag`.) - # The key and the value may each be quoted (`"uses": "owner/repo@sha"`). - @pin_re ~r/^[-+]?\s*-?\s*["']?uses["']?\s*:\s*["']?(?[A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@(?[^\s#"']+)/ + # The key and the value may each be quoted (`"uses": "owner/repo@sha"`), the + # step may be a flow mapping (`- { uses: owner/repo@sha, with: ... }`), and + # the ref ends at a flow delimiter as well as at a quote, space or comment. + @pin_re ~r/^[-+]?\s*-?\s*(?:\{\s*)?["']?uses["']?\s*:\s*["']?(?[A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@(?[^\s#"',}\]]+)/ @doc "Parse a `uses:` line into `%{action, base, ref}`; `nil` when it is not one." def parse_pin(line) do diff --git a/scripts/sweeps/estate-pin-integrity.sh b/scripts/sweeps/estate-pin-integrity.sh index 5b303ee3..754d0d6f 100755 --- a/scripts/sweeps/estate-pin-integrity.sh +++ b/scripts/sweeps/estate-pin-integrity.sh @@ -47,6 +47,7 @@ ORGS=() VIA="search" REWRITE=0 MAX_REPOS=0 +SAMPLE=0 while [ $# -gt 0 ]; do case "$1" in @@ -56,6 +57,7 @@ while [ $# -gt 0 ]; do --via) VIA="$2"; shift 2 ;; --rewrite) REWRITE=1; shift ;; --max-repos) MAX_REPOS="$2"; shift 2 ;; + --sample) SAMPLE="$2"; shift 2 ;; -h|--help) sed -n '2,40p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; *) echo "unknown argument: $1" >&2; exit 1 ;; esac @@ -108,6 +110,80 @@ REF_END='([]"'"'"'[:space:]#,}]|$)' # The `uses` key, quoted or bare, with the quote a quoted value opens with: # `uses: x@…`, `"uses": "x@…"`, `{ uses: x@… }` all name an action. USES_KEY='["'"'"']?uses["'"'"']?[[:space:]]*:[[:space:]]*["'"'"']?' +REF_END_CLASS='[]"'"'"'[:space:]#,}]' +# One `uses:` site: the action and a denylisted token, bounded. Matched in +# either case (-i): a commit ID in upper-case hex is the same object. +SITE_RE="${USES_KEY}[^@\"'[:space:],}]+@(${REF_ALTERNATION})(${REF_END})" +SHA_ALTERNATION=$(printf '%s\n' "${DENY_SHAS[@]}" | paste -sd'|' -) + +lc() { printf '%s' "$1" | tr '[:upper:]' '[:lower:]'; } + +# Does the policy deny at ? A blocked version is blocked +# for ITS action only (`some/action@v4.38.1` is no finding when only +# codeql-action's 4.38.1 is denied); a SHA is one object and meets its entry +# in either spelling. +denylisted_pair() { + jq -e --arg a "$1" --arg r "$(lc "$2")" ' + [.pin_denylist[] | select(.action == $a) + | (.blocked_shas // []), (.blocked_versions // []), ((.blocked_versions // []) | map("v" + .))] + | add // [] | map(ascii_downcase) | index($r) != null' "$POLICY" >/dev/null 2>&1 +} + +# The action and ref of one site as `grep -o` printed it: the key and the +# trailing boundary stripped, into site_action and site_ref. +site_parts() { + local t + t=$(printf '%s' "$1" | sed -E "s/^${USES_KEY}//; s/${REF_END_CLASS}\$//") + site_action="${t%@*}"; site_ref="${t##*@}" +} + +# Replace ONE site on a line — the action and ref exactly as written — with +# the known-good SHA for that action; the other sites on the line keep theirs. +replace_site() { # + local esc_a esc_r + esc_a=$(printf '%s' "$2" | sed -E 's#[][/.^$*+?(){}|\\]#\\&#g') + esc_r=$(printf '%s' "$3" | sed -E 's#[][/.^$*+?(){}|\\]#\\&#g') + printf '%s\n' "$1" | sed -E "s%(${USES_KEY}${esc_a})@${esc_r}(${REF_END})%\1@${4}\2%" +} + +# A denylisted SHA token in any position (a lock record key, a fixture): a +# finding in either spelling, but not a `uses:` site this sweep rewrites. +line_has_denied_sha() { + [ -n "$SHA_ALTERNATION" ] && printf '%s\n' "$1" | grep -qiE "@(${SHA_ALTERNATION})(${REF_END})" +} + +# Lines carrying a denied pin: a `uses:` site whose action/ref pair the +# policy denies, or a denylisted SHA token anywhere else. +count_denied_lines() { + local n=0 line site site_action site_ref + while IFS= read -r line || [ -n "$line" ]; do + if line_has_denied_sha "$line"; then n=$((n + 1)); continue; fi + while IFS= read -r site; do + [ -n "$site" ] || continue + site_parts "$site" + if denylisted_pair "$(printf '%s' "$site_action" | awk -F/ '{print $1"/"$2}')" "$site_ref"; then + n=$((n + 1)); break + fi + done < <(printf '%s\n' "$line" | grep -oiE "$SITE_RE" || true) + done < "$1" + printf '%s' "$n" +} + +# The column of a line's comment, found outside quoted scalars: a `#` at +# column 1 or after whitespace. `name: "C# build"` carries no comment; 0. +comment_column() { + printf '%s\n' "$1" | awk -v sq="'" '{ + q = "" + for (i = 1; i <= length($0); i++) { + c = substr($0, i, 1) + if (q == "") { + if (c == "\"" || c == sq) q = c + else if (c == "#" && (i == 1 || substr($0, i - 1, 1) ~ /[[:space:]]/)) { print i; exit } + } else if (c == q) q = "" + } + print 0 + }' +} known_good_sha_for() { jq -r --arg a "$1" '.pin_denylist[] | select(.action == $a) | .known_good_sha' "$POLICY" | head -n1 @@ -211,16 +287,26 @@ else WORK="$OUT_DIR/clones" mkdir -p "$WORK" for org in "${ORGS[@]}"; do - mapfile -t reponames < <(list_repos "$org") + # A listing or a clone that fails is a repository the authoritative mode + # never inspected: INCOMPLETE, never a clean verdict — as a failed search is. + if ! names=$(list_repos "$org"); then + log " ! ${org}: listing repositories failed — phase 1 is INCOMPLETE for this org" + failed=$((failed + 1)); continue + fi + reponames=() + [ -n "$names" ] && mapfile -t reponames <<<"$names" [ "$MAX_REPOS" -gt 0 ] && reponames=("${reponames[@]:0:$MAX_REPOS}") log " cloning ${#reponames[@]} repo(s) in ${org}" + # Clones live under the organisation: two organisations may each own a + # repository of the same name, and the second must not reuse the first's. + mkdir -p "$WORK/$org" for name in "${reponames[@]}"; do - d="$WORK/$name" + d="$WORK/$org/$name" [ -d "$d/.git" ] || git clone --depth 1 --quiet "https://github.com/${org}/${name}.git" "$d" 2>/dev/null || { - log " ! clone failed: ${org}/${name}"; continue; } + log " ! clone failed: ${org}/${name} — phase 1 is INCOMPLETE for this org"; failed=$((failed + 1)); continue; } while IFS= read -r f; do printf '%s\t%s\t%s\n' "$org" "$name" "${f#"$d"/}" >> "$HITS" - done < <(grep -rlE "$REF_ALTERNATION" "$d" --include='*.yml' --include='*.yaml' --include='*.lock' --include='*.toml' 2>/dev/null || true) + done < <(grep -rliE "$REF_ALTERNATION" "$d" --include='*.yml' --include='*.yaml' --include='*.lock' --include='*.toml' 2>/dev/null || true) done done sort -u "$HITS" -o "$HITS" @@ -228,20 +314,28 @@ fi total_hits=$(wc -l < "$HITS") log "phase 1: ${total_hits} candidate file(s)" +sampled=0 +if [ "$SAMPLE" -gt 0 ] && [ "$total_hits" -gt "$SAMPLE" ]; then + # A bounded run reads the first N candidates only: INCOMPLETE by construction. + sampled=$((total_hits - SAMPLE)) + head -n "$SAMPLE" "$HITS" > "$HITS.sample" && mv "$HITS.sample" "$HITS" + log " --sample ${SAMPLE}: ${sampled} candidate file(s) left unread; the sweep cannot report clean" +fi # What phase 1 could not see. A sweep that saw everything and found nothing # is clean; one that could not see everything never is. unseen() { local parts=() [ "$incomplete" -gt 0 ] && parts+=("${incomplete} search page(s) overflowed GitHub's 1,000-result cap or came back marked incomplete") - [ "$failed" -gt 0 ] && parts+=("${failed} search(es) failed") + [ "$failed" -gt 0 ] && parts+=("${failed} search(es), listing(s) or clone(s) failed") [ "${unread:-0}" -gt 0 ] && parts+=("${unread} candidate file(s) could not be read") + [ "${sampled:-0}" -gt 0 ] && parts+=("${sampled} candidate file(s) left unread by --sample") local IFS=';' printf '%s' "${parts[*]}" } if [ "$total_hits" -eq 0 ]; then - if [ "$incomplete" -gt 0 ] || [ "$failed" -gt 0 ]; then + if [ "$incomplete" -gt 0 ] || [ "$failed" -gt 0 ] || [ "$sampled" -gt 0 ]; then echo "INCOMPLETE: no denylisted pin tokens in the visible search results for ${ORGS[*]}, but $(unseen); a clean verdict needs --via clone" exit 2 fi @@ -272,7 +366,7 @@ while IFS=$'\t' read -r org repo path; do printf '%s\n' "$content" > "$OUT_DIR/.current" sites_before=$(grep -cE "(^|[{,[:space:]])${USES_KEY}" "$OUT_DIR/.current" || true) - denied_before=$(grep -cE "@(${REF_ALTERNATION})(${REF_END})" "$OUT_DIR/.current" || true) + denied_before=$(count_denied_lines "$OUT_DIR/.current") [ "$denied_before" -eq 0 ] && continue confirmed=$((confirmed + 1)) @@ -286,42 +380,46 @@ while IFS=$'\t' read -r org repo path; do *) in_scope=0 ;; esac - # Rewrite one line at a time so each pin gets the known-good SHA for ITS - # action, and so an action with no known-good pin blocks the whole file. + # Rewrite one pin SITE at a time: each `uses:` site gets the known-good + # SHA for ITS action (two actions on one flow line get two replacements); + # an action with no known-good pin blocks the whole file; a denylisted + # token outside a `uses:` key (a lock record, a fixture) is a finding the + # owner reviews, never a rewrite. Only the comment after the line's one + # replaced pin is relabelled, found outside quoted scalars. fixable=1 actions="" + outside_uses=0 [ "$in_scope" -eq 0 ] && fixable=0 : > "$OUT_DIR/.rewritten" while IFS= read -r line || [ -n "$line" ]; do - if printf '%s\n' "$line" | grep -qE "@(${REF_ALTERNATION})(${REF_END})"; then - # The action named before the `@`, whatever quotes or flow braces - # surround the key and the value. - action=$(printf '%s\n' "$line" | grep -oE "${USES_KEY}[^@\"'[:space:]]+@" | head -n1 \ - | sed -E "s/^${USES_KEY}//; s/@\$//") - base=$(printf '%s\n' "$action" | awk -F/ '{print $1"/"$2}') - good=$(known_good_sha_for "$base") + out="$line"; replaced=0; site_seen=0; good_version="" + while IFS= read -r site; do + [ -n "$site" ] || continue + site_parts "$site" + base=$(printf '%s' "$site_action" | awk -F/ '{print $1"/"$2}') + denylisted_pair "$base" "$site_ref" || continue + site_seen=1 actions="${actions}${base} " - if [ -z "$good" ] || [ "$good" = "null" ]; then - fixable=0 - printf '%s\n' "$line" >> "$OUT_DIR/.rewritten" - else - good_version=$(known_good_version_for "$base") - swapped=$(printf '%s\n' "$line" | sed -E "s%@(${REF_ALTERNATION})(${REF_END})%@${good}\2%g") - head="${swapped%%#*}" - if [ "$head" != "$swapped" ]; then - tail_comment="#${swapped#*#}" - printf '%s%s\n' "$head" "$(relabel_comment "$tail_comment" "$good_version")" >> "$OUT_DIR/.rewritten" - else - printf '%s\n' "$swapped" >> "$OUT_DIR/.rewritten" - fi + good=$(known_good_sha_for "$base") + if [ -z "$good" ] || [ "$good" = "null" ]; then fixable=0; continue; fi + out=$(replace_site "$out" "$site_action" "$site_ref" "$good") + replaced=$((replaced + 1)) + good_version=$(known_good_version_for "$base") + done < <(printf '%s\n' "$line" | grep -oiE "$SITE_RE" || true) + if [ "$site_seen" -eq 0 ] && line_has_denied_sha "$line"; then + outside_uses=$((outside_uses + 1)); fixable=0 + fi + if [ "$replaced" -eq 1 ] && [ -n "$good_version" ] && [ "$good_version" != "null" ]; then + at=$(comment_column "$out") + if [ "$at" -gt 0 ]; then + out="${out:0:$((at - 1))}$(relabel_comment "${out:$((at - 1))}" "$good_version")" fi - else - printf '%s\n' "$line" >> "$OUT_DIR/.rewritten" fi + printf '%s\n' "$out" >> "$OUT_DIR/.rewritten" done < "$OUT_DIR/.current" sites_after=$(grep -cE "(^|[{,[:space:]])${USES_KEY}" "$OUT_DIR/.rewritten" || true) - denied_after=$(grep -cE "@(${REF_ALTERNATION})(${REF_END})" "$OUT_DIR/.rewritten" || true) + denied_after=$(count_denied_lines "$OUT_DIR/.rewritten") status="flag" reason="owner_review" @@ -331,6 +429,8 @@ while IFS=$'\t' read -r org repo path; do reason="substitution" elif [ "$in_scope" -eq 0 ]; then reason="out_of_scope_file" + elif [ "$outside_uses" -gt 0 ]; then + reason="denylisted_token_outside_uses_key" elif [ "$fixable" -eq 0 ]; then reason="no_known_good_pin_for_action" elif [ "$sites_after" -ne "$sites_before" ]; then @@ -358,7 +458,7 @@ while IFS=$'\t' read -r org repo path; do done < "$HITS" if [ "$confirmed" -eq 0 ]; then - if [ "$incomplete" -gt 0 ] || [ "$failed" -gt 0 ] || [ "$unread" -gt 0 ]; then + if [ "$incomplete" -gt 0 ] || [ "$failed" -gt 0 ] || [ "$unread" -gt 0 ] || [ "$sampled" -gt 0 ]; then echo "INCOMPLETE: none of the ${total_hits} candidate file(s) carries a denylisted pin, but $(unseen); a clean verdict needs --via clone" exit 2 fi @@ -396,7 +496,7 @@ log " repair plan : ${PLAN}" [ "$REWRITE" -eq 1 ] && log " repaired trees : ${OUT_DIR}/patched" log " NOTE: this sweep never writes to a remote. Branch + PR creation is" log " the token-bearing actuator's job — docs/operations/estate-automerge.adoc" -if [ "$incomplete" -gt 0 ] || [ "$failed" -gt 0 ] || [ "$unread" -gt 0 ]; then +if [ "$incomplete" -gt 0 ] || [ "$failed" -gt 0 ] || [ "$unread" -gt 0 ] || [ "$sampled" -gt 0 ]; then log " INCOMPLETE: $(unseen); what they hid was not seen — rerun with --via clone" fi diff --git a/test/rules/pr_automerge_test.exs b/test/rules/pr_automerge_test.exs index 4257867b..022737dc 100644 --- a/test/rules/pr_automerge_test.exs +++ b/test/rules/pr_automerge_test.exs @@ -175,6 +175,19 @@ defmodule Hypatia.Rules.PrAutomergeTest do assert decision.blocked_by == "introduces_denylisted_pin" end + test "an unquoted flow-style pin ends at its delimiter: PA001 sees the poison" do + patch = """ + @@ -1,1 +1,1 @@ + - - { uses: github/codeql-action/init@#{@good}, with: { languages: rust } } + + - { uses: github/codeql-action/init@#{@poison}, with: { languages: rust } } + """ + + decision = PA.classify(pr([file(".github/workflows/codeql.yml", patch)]), @policy) + + assert decision.disposition == :close_poison_only + assert decision.blocked_by == "introduces_denylisted_pin" + end + test "a quoted uses: key and a quoted value are still a pin PA001 sees" do patch = """ @@ -1,1 +1,1 @@ From cf0866298a956e8a5d3a482bfdd058fa49d07755 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 23:20:19 +0000 Subject: [PATCH 26/47] fix(rules,sweeps): every uses: site per line, structural sites only, case-folded actions, one-line flow workflows Codex round on caab1df (hypatia#931), four findings: * pr_automerge (P1): poison_sites/2 and pin_deltas/3 read every `uses:` site on a changed line (parse_pins/1), not only the first. A flow sequence carrying an ordinary bump and a denylisted pin on one line was classified pin-only with no poison and armed automerge; it is now `close_poison_only`. Sites are read from a masked line (quoted scalars that are neither keys nor `uses` values blanked, comments dropped), so a pin quoted inside a `name:` string is not a site. Action identities are compared case-folded. * estate-pin-integrity.sh (P1): the policy lookups (denylisted_pair, known_good_sha_for, known_good_version_for) compare the action case-folded, so `GitHub/CodeQL-Action/init@v4.38.1` is a finding and is repaired with its spelling kept. * estate-pin-integrity.sh (P2): sites are found on a masked copy of the file (mask_file: block-scalar content, non-key quoted scalars and comments blanked in place, lengths kept) and replaced by position, so `- run: "echo uses: a/b@"` and a `run: |` script are never rewritten. A denylisted SHA left there is flagged `denylisted_token_outside_uses_key`, never `repairable`. * workflow_audit WF026 (P2): a workflow written in flow style on one line (`jobs: { build: { steps: [ { uses: ... } ] } }`, with or without a root `{ }`, or a `build: { steps: [...] }` job line) is read through the keys that open the brackets around the step: `steps` in a job in `jobs`. Tests: pr_automerge 27/27 (4 new, all failing before); workflow_audit_pins 47 with the 2 known harness-only failures (2 new; the positive one fails before). Sweep: 14 offline fake-gh modes, identical under gawk and mawk, 4 of them new. shellcheck -S warning clean; mix-format layout applied. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- lib/rules/pr_automerge.ex | 120 ++++++++++++++++++------ lib/rules/workflow_audit.ex | 56 +++++++++-- scripts/sweeps/estate-pin-integrity.sh | 118 ++++++++++++++++++----- test/rules/pr_automerge_test.exs | 56 +++++++++++ test/rules/workflow_audit_pins_test.exs | 39 ++++++++ 5 files changed, 334 insertions(+), 55 deletions(-) diff --git a/lib/rules/pr_automerge.ex b/lib/rules/pr_automerge.ex index cf7c8b8f..a45aed5e 100644 --- a/lib/rules/pr_automerge.ex +++ b/lib/rules/pr_automerge.ex @@ -127,25 +127,25 @@ defmodule Hypatia.Rules.PrAutomerge do file |> added_lines() |> Enum.flat_map(fn line -> - case parse_pin(line) do - %{action: action, ref: ref} -> - if action_base(action) == Map.get(entry, "action") and - denylisted_ref?(entry, ref) do - [ - %{ - file: filename_of(file), - line: String.trim(line), - denylist_id: Map.get(entry, "id"), - severity: Map.get(entry, "severity", "critical") - } - ] - else - [] - end - - _ -> - [] - end + # Every site on the line: a flow sequence can put several steps on + # one changed line, and a pin judged by its first site alone lets a + # denylisted second one through to `arm_auto`. The action is + # compared case-folded — GitHub resolves `GitHub/CodeQL-Action` and + # `github/codeql-action` to one repository. + line + |> parse_pins() + |> Enum.filter(fn %{base: base, ref: ref} -> + String.downcase(base) == String.downcase(Map.get(entry, "action", "")) and + denylisted_ref?(entry, ref) + end) + |> Enum.map(fn _pin -> + %{ + file: filename_of(file), + line: String.trim(line), + denylist_id: Map.get(entry, "id"), + severity: Map.get(entry, "severity", "critical") + } + end) end) end) end) @@ -213,20 +213,20 @@ defmodule Hypatia.Rules.PrAutomerge do def pin_deltas(files, resolution, claims) do files |> Enum.flat_map(fn file -> - removed = file |> removed_lines() |> Enum.filter(&pin_line?/1) - added = file |> added_lines() |> Enum.filter(&pin_line?/1) + # Every site of every pin line: a flow sequence carries several steps + # on one line, and each of them is a delta of its own. + removed = + file |> removed_lines() |> Enum.filter(&pin_line?/1) |> Enum.flat_map(&parse_pins/1) - removed - |> Enum.flat_map(fn line -> - old = parse_pin(line) + added = file |> added_lines() |> Enum.filter(&pin_line?/1) |> Enum.flat_map(&parse_pins/1) - case Enum.find(added, &(parse_pin(&1).action == old.action)) do + removed + |> Enum.flat_map(fn old -> + case Enum.find(added, &(String.downcase(&1.action) == String.downcase(old.action))) do nil -> [] - new_line -> - new = parse_pin(new_line) - + new -> if new.ref == old.ref do [] else @@ -485,6 +485,70 @@ defmodule Hypatia.Rules.PrAutomerge do defp pin_line?(line), do: parse_pin(line) != nil + # One `uses:` site anywhere on a line, bounded on the left so `reuses:` is + # not one. Matched against the MASKED line (see mask_scalars/1). + @pin_site_re ~r/(?:^|[^A-Za-z0-9_.\/-])["']?uses["']?\s*:\s*["']?(?[A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@(?[^\s#"',}\]]+)/ + + @doc """ + Every `uses:` site on a line, in order, as `%{action, base, ref}`. + + A flow sequence may put several steps on one line + (`+ { uses: a/b@x }, { uses: c/d@y },`); `parse_pin/1` reads only the + first. Text that merely mentions a pin — inside a quoted `name:` or + `run:` scalar, or after `#` — is not a site. + """ + @spec parse_pins(String.t()) :: [map()] + def parse_pins(line) do + @pin_site_re + |> Regex.scan(mask_scalars(line), capture: :all_names) + |> Enum.map(fn [action, ref] -> %{action: action, base: action_base(action), ref: ref} end) + end + + # The line with its comment dropped and every quoted scalar that is + # neither a key nor the value of a `uses` key blanked to `x`. A `uses` + # value and a quoted key are kept as written, so the captures read from + # the masked line are the pin's own text. + @uses_tail_re ~r/(?:^|[-{,\s])["']?uses["']?\s*:\s*$/ + + defp mask_scalars(line), do: do_mask(line, "") + + defp do_mask("", out), do: out + + defp do_mask(<>, out) when q in [?", ?'] do + {body, close, tail} = take_quoted(rest, q, "") + + kept = + if Regex.match?(~r/^\s*:/, tail) or Regex.match?(@uses_tail_re, out), + do: body, + else: String.duplicate("x", String.length(body)) + + do_mask(tail, out <> <> <> kept <> close) + end + + defp do_mask(<>, out) do + if out == "" or String.ends_with?(out, [" ", "\t"]), + do: out, + else: do_mask(rest, out <> "#") + end + + defp do_mask(<>, out), do: do_mask(rest, out <> <>) + defp do_mask(<>, out), do: do_mask(rest, out <> <>) + + # The body of a quoted scalar up to its closing quote; a backslash escapes + # the next character inside double quotes. An unterminated scalar runs to + # the end of the line. + defp take_quoted("", _q, acc), do: {acc, "", ""} + + defp take_quoted(<>, ?", acc), + do: take_quoted(rest, ?", acc <> <>) + + defp take_quoted(<>, q, acc) when c == q, do: {acc, <>, rest} + + defp take_quoted(<>, q, acc), + do: take_quoted(rest, q, acc <> <>) + + defp take_quoted(<>, q, acc), do: take_quoted(rest, q, acc <> <>) + defp content_lines(patch) do patch |> String.split("\n") diff --git a/lib/rules/workflow_audit.ex b/lib/rules/workflow_audit.ex index c0ea8ad9..b03c5401 100644 --- a/lib/rules/workflow_audit.ex +++ b/lib/rules/workflow_audit.ex @@ -1162,7 +1162,11 @@ defmodule Hypatia.Rules.WorkflowAudit do (`- { uses: ..., with: { toolchain: stable } }`, `steps: [ { uses: ... } ]`, or a `- {` item spread over lines) is read through its braces: its `with` entry decides, quotes and nested - braces respected, and a `uses:` inside a quoted string is text. + braces respected, and a `uses:` inside a quoted string is text. A + workflow written in flow style on one line (`jobs: { build: { steps: + [ { uses: ... } ] } }`, with or without a root `{ }`) is read the same + way: the keys opening the brackets around the step must be `steps` + in a job in `jobs`. * `with:` may be an anchor (`with: &inputs`, block or flow), an alias (`with: *inputs`) or carry a merge key (`<<: *inputs`): the anchored mapping is resolved from the document and its direct entries decide. @@ -1238,8 +1242,7 @@ defmodule Hypatia.Rules.WorkflowAudit do # lines) and its `with` entry decides. defp wf026_inline_step_finding(filename, lines, line, number, uses_at, sha, kyaml?) do with {:ok, open_col} <- wf_flow_opener_before(line, uses_at), - true <- wf026_inline_item?(line, open_col), - true <- wf026_inline_under_steps?(lines, line, number, open_col), + true <- wf026_inline_step_at?(lines, line, number, open_col), {:ok, interior, _} <- wf_flow_interior(lines, number, open_col) do case wf026_with_entry(wf_flow_entries(interior), lines) do {:toolchain, _} -> [] @@ -1250,6 +1253,47 @@ defmodule Hypatia.Rules.WorkflowAudit do end end + # Is the flow mapping opening at `open_col` a step? Either it sits in a + # sequence position of a block document (below), or the line itself + # writes the path to it in flow style. + defp wf026_inline_step_at?(lines, line, number, open_col) do + (wf026_inline_item?(line, open_col) and + wf026_inline_under_steps?(lines, line, number, open_col)) or + wf026_flow_path_step?(lines, line, number, open_col) + end + + # A workflow written in flow style on one line — `jobs: { build: { steps: + # [ { uses: ... } ] } }`, the same under a root `{ ... }`, or a job line + # `build: { steps: [ ... ] }` under a block `jobs:`. The mapping must sit + # directly in a `[` that `steps:` opens, in a mapping a job key opens, in + # the one `jobs:` opens: the keys that open the brackets still unclosed on + # the line, nearest first, then the block ancestors of the line. Each key + # is read quote-aware, so a `name: "CI, build"` sibling separates nothing. + defp wf026_flow_path_step?(lines, line, number, open_col) do + case wf_flow_opener_stack(line, open_col) do + {:ok, [{:seq, _} | _] = stack} -> + keys = wf_flow_opener_keys(line, stack) ++ (wf_ancestor_keys(lines, number) || []) + match?(["steps", job, "jobs" | _] when is_binary(job), keys) + + _ -> + false + end + end + + # The key that opens each bracket of an opener stack (innermost first), or + # nil for a bracket no key opens (a root `{`, a `{` item in a sequence). + defp wf_flow_opener_keys(line, stack) do + cols = + Enum.map(stack, fn + {:seq, col} -> col + col -> col + end) + + froms = cols |> Enum.drop(1) |> Enum.map(&(&1 + 1)) |> Kernel.++([0]) + + Enum.zip_with(cols, froms, fn col, from -> wf027_key_before(line, from, col) end) + end + # The text before the mapping's `{` must be a sequence position: a list # dash, the `steps:` key opening its sequence, a `[`, a comma, or nothing. # Anything else (`run: "echo { uses: ... }"`) is a scalar carrying braces. @@ -1684,8 +1728,8 @@ defmodule Hypatia.Rules.WorkflowAudit do # stack holds the byte column of each open `{` (`:seq` for a `[`); the # innermost must be a mapping for `at` to be a key position. # Every `{`/`[` still open before byte `at` of `line`, innermost first: - # the column of each `{`, `:seq` for a `[`. Quoted scalars and comments - # are skipped, as in wf_flow_opener_before/2; :error inside one. + # the column of each `{`, `{:seq, column}` for a `[`. Quoted scalars and + # comments are skipped, as in wf_flow_opener_before/2; :error inside one. defp wf_flow_opener_stack(line, at) do line |> binary_part(0, at) @@ -1701,7 +1745,7 @@ defmodule Hypatia.Rules.WorkflowAudit do c == ?# and prev in [?\s, ?\t] -> wf_flow_stack(rest, pos + 1, :comment, stack, c) c in [?", ?'] -> wf_flow_stack(rest, pos + 1, c, stack, c) c == ?{ -> wf_flow_stack(rest, pos + 1, nil, [pos | stack], c) - c == ?[ -> wf_flow_stack(rest, pos + 1, nil, [:seq | stack], c) + c == ?[ -> wf_flow_stack(rest, pos + 1, nil, [{:seq, pos} | stack], c) c in [?}, ?]] -> wf_flow_stack(rest, pos + 1, nil, Enum.drop(stack, 1), c) true -> wf_flow_stack(rest, pos + 1, nil, stack, c) end diff --git a/scripts/sweeps/estate-pin-integrity.sh b/scripts/sweeps/estate-pin-integrity.sh index 754d0d6f..098398b6 100755 --- a/scripts/sweeps/estate-pin-integrity.sh +++ b/scripts/sweeps/estate-pin-integrity.sh @@ -121,10 +121,12 @@ lc() { printf '%s' "$1" | tr '[:upper:]' '[:lower:]'; } # Does the policy deny at ? A blocked version is blocked # for ITS action only (`some/action@v4.38.1` is no finding when only # codeql-action's 4.38.1 is denied); a SHA is one object and meets its entry -# in either spelling. +# in either spelling. The action is compared case-folded too: GitHub resolves +# `GitHub/CodeQL-Action` and `github/codeql-action` to one repository, so a +# mixed-case caller meets the entry its lower-case spelling would. denylisted_pair() { - jq -e --arg a "$1" --arg r "$(lc "$2")" ' - [.pin_denylist[] | select(.action == $a) + jq -e --arg a "$(lc "$1")" --arg r "$(lc "$2")" ' + [.pin_denylist[] | select((.action | ascii_downcase) == $a) | (.blocked_shas // []), (.blocked_versions // []), ((.blocked_versions // []) | map("v" + .))] | add // [] | map(ascii_downcase) | index($r) != null' "$POLICY" >/dev/null 2>&1 } @@ -137,13 +139,79 @@ site_parts() { site_action="${t%@*}"; site_ref="${t##*@}" } +# Everything that is not YAML structure, blanked IN PLACE, one output line +# per input line: a quoted scalar that is neither a key nor the value of a +# `uses` key has its text replaced by `x` (its quotes kept), a comment by +# spaces, and the content lines of a block scalar (`run: |`, `script: >-`) +# entirely by `x` after their indentation. Lengths are kept, so a site found +# on a masked line sits at the same column of the real one. A `uses:` written +# inside `- run: "echo uses: a/b@"`, a `run: |` script or a comment is +# therefore no site this sweep rewrites; a denylisted SHA there is still +# seen by line_has_denied_sha on the raw line, and flagged for the owner. +# shellcheck disable=SC2016 # an awk program, not a shell expansion +MASK_AWK=' +function key_col(s, t) { t = s; sub(/[^[:space:]-].*$/, "", t); return length(t) } +function xs(n, t) { t = ""; while (n-- > 0) t = t "x"; return t } +function mask(s, out, n, i, c, j, body) { + out = ""; n = length(s); i = 1 + while (i <= n) { + c = substr(s, i, 1) + if (c == "#" && (i == 1 || substr(s, i - 1, 1) ~ /[[:space:]]/)) { while (i <= n) { out = out " "; i++ }; break } + if (c == "\"" || c == sq) { + j = i + 1 + if (c == "\"") { while (j <= n && substr(s, j, 1) != "\"") { if (substr(s, j, 1) == "\\") j++; j++ } } + else { while (j <= n && substr(s, j, 1) != sq) j++ } + if (j > n) j = n + body = substr(s, i + 1, j - i - 1) + if (substr(s, j + 1) ~ /^[[:space:]]*:/ || tolower(out) ~ USES_TAIL) out = out substr(s, i, j - i + 1) + else out = out c xs(length(body)) (j > i ? substr(s, j, 1) : "") + i = j + 1; continue + } + out = out c; i++ + } + return out +} +BEGIN { USES_TAIL = "(^|[-{,[:space:]])[\"" sq "]?uses[\"" sq "]?[[:space:]]*:[[:space:]]*$"; in_scalar = 0 } +{ + if (in_scalar) { + if ($0 ~ /^[[:space:]]*$/ || key_col($0) > scalar_col) { t = $0; sub(/[^[:space:]].*$/, "", t); print t xs(length($0) - length(t)); next } + in_scalar = 0 + } + m = mask($0); print m + if (m ~ /:[[:space:]]*[|>][-+0-9]*[[:space:]]*$/) { in_scalar = 1; scalar_col = key_col($0) } +}' + +# The masked form of one file, line for line (block scalars tracked). +mask_file() { awk -v sq="'" "$MASK_AWK" "$1"; } + +# The masked form of one line on its own: used after a site on that line has +# been rewritten, which a line inside a block scalar never is. +mask_line() { printf '%s\n' "$1" | awk -v sq="'" "$MASK_AWK"; } + +# The `uses:` sites of a masked line, as grep -o prints them. +masked_sites() { printf '%s\n' "$1" | grep -oiE "$SITE_RE" || true; } + # Replace ONE site on a line — the action and ref exactly as written — with # the known-good SHA for that action; the other sites on the line keep theirs. -replace_site() { # - local esc_a esc_r - esc_a=$(printf '%s' "$2" | sed -E 's#[][/.^$*+?(){}|\\]#\\&#g') - esc_r=$(printf '%s' "$3" | sed -E 's#[][/.^$*+?(){}|\\]#\\&#g') - printf '%s\n' "$1" | sed -E "s%(${USES_KEY}${esc_a})@${esc_r}(${REF_END})%\1@${4}\2%" +# The site is located on the masked line, so the same `action@ref` text +# inside a quoted scalar or a comment earlier on the line is never the one +# replaced. +replace_site() { # + awk -v line="$1" -v masked="$2" -v a="$3" -v r="$4" -v good="$5" -v sq="'" 'BEGIN { + tail = "(^|[-{,[:space:]])[\"" sq "]?uses[\"" sq "]?[[:space:]]*:[[:space:]]*[\"" sq "]?$" + ends = "]\"" sq " \t#,}" + lm = tolower(masked); needle = tolower(a "@" r); from = 1 + while ((p = index(substr(lm, from), needle)) > 0) { + p += from - 1 + nxt = substr(lm, p + length(needle), 1) + if ((nxt == "" || index(ends, nxt) > 0) && substr(lm, 1, p - 1) ~ tail) { + print substr(line, 1, p + length(a)) good substr(line, p + length(needle)) + exit + } + from = p + 1 + } + print line + }' } # A denylisted SHA token in any position (a lock record key, a fixture): a @@ -155,8 +223,10 @@ line_has_denied_sha() { # Lines carrying a denied pin: a `uses:` site whose action/ref pair the # policy denies, or a denylisted SHA token anywhere else. count_denied_lines() { - local n=0 line site site_action site_ref - while IFS= read -r line || [ -n "$line" ]; do + local n=0 line mline site site_action site_ref masked + masked=$(mktemp) + mask_file "$1" > "$masked" + while IFS= read -r line <&3 && IFS= read -r mline <&4; do if line_has_denied_sha "$line"; then n=$((n + 1)); continue; fi while IFS= read -r site; do [ -n "$site" ] || continue @@ -164,8 +234,9 @@ count_denied_lines() { if denylisted_pair "$(printf '%s' "$site_action" | awk -F/ '{print $1"/"$2}')" "$site_ref"; then n=$((n + 1)); break fi - done < <(printf '%s\n' "$line" | grep -oiE "$SITE_RE" || true) - done < "$1" + done < <(masked_sites "$mline") + done 3< "$1" 4< "$masked" + rm -f "$masked" printf '%s' "$n" } @@ -186,11 +257,11 @@ comment_column() { } known_good_sha_for() { - jq -r --arg a "$1" '.pin_denylist[] | select(.action == $a) | .known_good_sha' "$POLICY" | head -n1 + jq -r --arg a "$(lc "$1")" '.pin_denylist[] | select((.action | ascii_downcase) == $a) | .known_good_sha' "$POLICY" | head -n1 } known_good_version_for() { - jq -r --arg a "$1" '.pin_denylist[] | select(.action == $a) | .known_good_version' "$POLICY" | head -n1 + jq -r --arg a "$(lc "$1")" '.pin_denylist[] | select((.action | ascii_downcase) == $a) | .known_good_version' "$POLICY" | head -n1 } # Relabel a pin's inline comment ONLY when the comment leads with a version @@ -391,22 +462,27 @@ while IFS=$'\t' read -r org repo path; do outside_uses=0 [ "$in_scope" -eq 0 ] && fixable=0 : > "$OUT_DIR/.rewritten" - while IFS= read -r line || [ -n "$line" ]; do - out="$line"; replaced=0; site_seen=0; good_version="" + mask_file "$OUT_DIR/.current" > "$OUT_DIR/.masked" + while IFS= read -r line <&3 && IFS= read -r mline <&4; do + out="$line"; mout="$mline"; replaced=0; site_seen=0; good_version="" while IFS= read -r site; do [ -n "$site" ] || continue site_parts "$site" - base=$(printf '%s' "$site_action" | awk -F/ '{print $1"/"$2}') + base=$(lc "$(printf '%s' "$site_action" | awk -F/ '{print $1"/"$2}')") denylisted_pair "$base" "$site_ref" || continue site_seen=1 actions="${actions}${base} " good=$(known_good_sha_for "$base") if [ -z "$good" ] || [ "$good" = "null" ]; then fixable=0; continue; fi - out=$(replace_site "$out" "$site_action" "$site_ref" "$good") + out=$(replace_site "$out" "$mout" "$site_action" "$site_ref" "$good") + mout=$(mask_line "$out") replaced=$((replaced + 1)) good_version=$(known_good_version_for "$base") - done < <(printf '%s\n' "$line" | grep -oiE "$SITE_RE" || true) - if [ "$site_seen" -eq 0 ] && line_has_denied_sha "$line"; then + done < <(masked_sites "$mline") + # A denylisted SHA left on the line that is not at a `uses:` site — the + # line had none, or the SHA sits in a quoted scalar or a comment the + # mask blanked — is a finding for the owner, never part of a repair. + if line_has_denied_sha "$out" && { [ "$site_seen" -eq 0 ] || ! line_has_denied_sha "$mout"; }; then outside_uses=$((outside_uses + 1)); fixable=0 fi if [ "$replaced" -eq 1 ] && [ -n "$good_version" ] && [ "$good_version" != "null" ]; then @@ -416,7 +492,7 @@ while IFS=$'\t' read -r org repo path; do fi fi printf '%s\n' "$out" >> "$OUT_DIR/.rewritten" - done < "$OUT_DIR/.current" + done 3< "$OUT_DIR/.current" 4< "$OUT_DIR/.masked" sites_after=$(grep -cE "(^|[{,[:space:]])${USES_KEY}" "$OUT_DIR/.rewritten" || true) denied_after=$(count_denied_lines "$OUT_DIR/.rewritten") diff --git a/test/rules/pr_automerge_test.exs b/test/rules/pr_automerge_test.exs index 022737dc..90a9cfc3 100644 --- a/test/rules/pr_automerge_test.exs +++ b/test/rules/pr_automerge_test.exs @@ -336,6 +336,62 @@ defmodule Hypatia.Rules.PrAutomergeTest do assert PA.poison_sites([file(".github/workflows/codeql.yml", patch)], @policy) == [] end + + test "every site of a flow-sequence line is inspected, not only the first" do + # The first step is an ordinary resolvable bump; the second is the + # poison. Judging the line by its first pin armed automerge. + patch = """ + @@ -1,1 +1,1 @@ + - { uses: actions/checkout@v4.1.6 }, { uses: github/codeql-action/init@#{@good} }, + + { uses: actions/checkout@v4.1.7 }, { uses: github/codeql-action/init@#{@poison} }, + """ + + f = file(".github/workflows/codeql.yml", patch) + assert [site] = PA.poison_sites([f], @policy) + assert site.denylist_id == "PIN-001" + + decision = + PA.classify( + pr([f], + resolution: %{ + {"actions/checkout", "v4.1.6"} => "4.1.6", + {"actions/checkout", "v4.1.7"} => "4.1.7" + } + ), + @policy + ) + + assert decision.pin_only + assert decision.disposition == :close_poison_only + assert decision.blocked_by == "introduces_denylisted_pin" + end + + test "every flow-sequence site is a delta of its own" do + patch = """ + @@ -1,1 +1,1 @@ + - - { uses: actions/checkout@v4.1.6 }, { uses: actions/cache@v4.0.0 } + + - { uses: actions/checkout@v4.1.7 }, { uses: actions/cache@v5.0.0 } + """ + + deltas = PA.pin_deltas([file(".github/workflows/ci.yml", patch)], %{}, []) + assert Enum.map(deltas, & &1.action) == ["actions/checkout", "actions/cache"] + end + + test "a mixed-case action spelling meets its lower-case denylist entry" do + patch = + codeql_patch(@good, @poison) + |> String.replace("github/codeql-action", "GitHub/CodeQL-Action") + + assert [site] = PA.poison_sites([file(".github/workflows/codeql.yml", patch)], @policy) + assert site.denylist_id == "PIN-001" + end + + test "a pin quoted inside another scalar or a comment is not a site" do + line = + ~s(+ - { name: "see uses: github/codeql-action/init@#{@poison}", uses: actions/checkout@v4.1.7 } # uses: x/y@#{@poison}) + + assert [%{action: "actions/checkout", ref: "v4.1.7"}] = PA.parse_pins(line) + end end describe "body_claims/1" do diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs index ca800b50..752a876c 100644 --- a/test/rules/workflow_audit_pins_test.exs +++ b/test/rules/workflow_audit_pins_test.exs @@ -348,6 +348,45 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) end + test "a workflow written in flow style on one line is read through its keys" do + sha2 = String.duplicate("a", 40) + + docs = %{ + "jobs.yml" => + "jobs: { build: { steps: [ { uses: dtolnay/rust-toolchain@#{@toolchain_sha} } ] } }\n", + "root.yml" => + "{ on: push, jobs: { build: { runs-on: ubuntu-latest, steps: [ " <> + "{ uses: dtolnay/rust-toolchain@#{@toolchain_sha}, with: { toolchain: stable } }, " <> + "{ name: \"CI, build\", uses: dtolnay/rust-toolchain@#{sha2} } ] } } }\n", + "jobline.yml" => + "jobs:\n build: { steps: [ { uses: dtolnay/rust-toolchain@#{@toolchain_sha} } ] }\n" + } + + findings = WorkflowAudit.check_sha_pinned_toolchain_without_input(docs) + + assert [%{line: 1, auto_fixable: false}] = Enum.filter(findings, &(&1.file == "jobs.yml")) + + assert [%{line: 1, current_sha: ^sha2, auto_fixable: false}] = + Enum.filter(findings, &(&1.file == "root.yml")) + + assert [%{line: 2, auto_fixable: false}] = + Enum.filter(findings, &(&1.file == "jobline.yml")) + end + + test "a one-line flow path that is not steps in a job in jobs is not a step" do + docs = %{ + "env.yml" => + "env: { build: { steps: [ { uses: dtolnay/rust-toolchain@#{@toolchain_sha} } ] } }\n", + "fixture.yml" => + "jobs: { build: { fixture: [ { uses: dtolnay/rust-toolchain@#{@toolchain_sha} } ] } }\n", + "quoted.yml" => + "jobs: { build: { steps: [ { run: \"jobs: { b: { steps: [ { uses: " <> + "dtolnay/rust-toolchain@#{@toolchain_sha} } ] } }\" } ] } }\n" + } + + assert WorkflowAudit.check_sha_pinned_toolchain_without_input(docs) == [] + end + test "a - { item spread over lines: its with entry decides, nested braces respected" do wf = """ jobs: From 1b98ef253315d377ca6cdbf94158981eee5e002f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 23:40:05 +0000 Subject: [PATCH 27/47] fix(rules,sweeps): structural pin sites only, deltas on every line, case-folded WF026 action, root-braced WF027, single-site relabel Codex round on cf08662 (hypatia#931): * pr_automerge (P1): parse_pins/1 counts only structural sites - the line's own leading `uses:` key, or an entry of an anonymous flow mapping at line level or in a `[` that `steps:` opens, or a job mapping inside `jobs: { ... }`. Free text before a bracket (`echo { uses: ... }`), an input's mapping (`with: { uses: ... }`), quoted scalars and comments are not sites. Changed lines the hunk shows inside a block scalar (`run: |`) are script text: no poison site, no delta, not a pin line. Before, script text could be reported as poison and armed through `excise_poison_then_merge`. * pr_automerge (P1): pin_deltas/3 reads every structural site of every changed line, not only lines that start with `uses:`, so a one-line `steps: [ ... ]` major bump next to a poison pin closes as `close_poison_and_majors`. * WF026 (P2): `dtolnay/rust-toolchain` is matched in either case; a spelling other than lower case routes to review (auto_fixable false), since the fleet fixer matches the lower-case name. * WF027 (P2): a root-braced one-line workflow (`{ jobs: { rust: { uses: ... } } }`) is read through the keys that open the brackets around the call: a job key, then `jobs`. * estate-pin-integrity.sh (P2): a pin comment is relabelled only on a line with one `uses:` site; on a multi-site flow line the comment cannot be tied to the replaced pin and is kept as written. Declined with a traced reason (thread reply): a `uses:` inside a PLAIN (unquoted) run scalar - `- run: echo uses: a/b@sha` - is not valid YAML (PyYAML, Psych and yq all reject it: "mapping values are not allowed here"), so no loadable workflow carries it; quoted and block scalars are already masked. Tests: pr_automerge 30/30 (3 new, failing on cf08662); workflow_audit_pins 49 with the 2 known harness-only failures (2 new, failing on cf08662). Sweep: 15 offline fake-gh modes identical under gawk and mawk, the new multi-site relabel mode reproducing on cf08662. shellcheck clean. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- lib/rules/pr_automerge.ex | 153 +++++++++++++++++++++--- lib/rules/workflow_audit.ex | 66 ++++++---- scripts/sweeps/estate-pin-integrity.sh | 7 +- test/rules/pr_automerge_test.exs | 50 ++++++++ test/rules/workflow_audit_pins_test.exs | 23 ++++ 5 files changed, 254 insertions(+), 45 deletions(-) diff --git a/lib/rules/pr_automerge.ex b/lib/rules/pr_automerge.ex index a45aed5e..44dc15a3 100644 --- a/lib/rules/pr_automerge.ex +++ b/lib/rules/pr_automerge.ex @@ -184,9 +184,11 @@ defmodule Hypatia.Rules.PrAutomerge do """ @spec pin_lines_only?(map()) :: boolean() def pin_lines_only?(file) do - lines = content_lines(patch_of(file)) + lines = changed_lines(file) - lines != [] and Enum.all?(lines, fn line -> pin_line?(line) end) + # A changed line inside a block scalar (`run: |`) is script text, never a + # pin line, so a script edit is not a pin-only change. + lines != [] and Enum.all?(lines, fn {line, scalar?} -> not scalar? and pin_line?(line) end) end # ─── PA002/PA003/PA004 · version deltas ─────────────────────────────── @@ -213,12 +215,12 @@ defmodule Hypatia.Rules.PrAutomerge do def pin_deltas(files, resolution, claims) do files |> Enum.flat_map(fn file -> - # Every site of every pin line: a flow sequence carries several steps - # on one line, and each of them is a delta of its own. - removed = - file |> removed_lines() |> Enum.filter(&pin_line?/1) |> Enum.flat_map(&parse_pins/1) - - added = file |> added_lines() |> Enum.filter(&pin_line?/1) |> Enum.flat_map(&parse_pins/1) + # Every structural site of every changed line: a flow sequence carries + # several steps on one line (`steps: [ { uses: a@x }, { uses: b@y } ]`), + # and each of them is a delta of its own — including a major bump on a + # line that does not start with `uses:`. + removed = file |> removed_lines() |> Enum.flat_map(&parse_pins/1) + added = file |> added_lines() |> Enum.flat_map(&parse_pins/1) removed |> Enum.flat_map(fn old -> @@ -490,18 +492,81 @@ defmodule Hypatia.Rules.PrAutomerge do @pin_site_re ~r/(?:^|[^A-Za-z0-9_.\/-])["']?uses["']?\s*:\s*["']?(?[A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@(?[^\s#"',}\]]+)/ @doc """ - Every `uses:` site on a line, in order, as `%{action, base, ref}`. + Every structural `uses:` site on a line, in order, as `%{action, base, ref}`. A flow sequence may put several steps on one line (`+ { uses: a/b@x }, { uses: c/d@y },`); `parse_pin/1` reads only the - first. Text that merely mentions a pin — inside a quoted `name:` or - `run:` scalar, or after `#` — is not a site. + first. Only a site the YAML structure makes a pin counts: + + * the line's own leading `uses:` key (`uses:`, `- uses:`, `- { uses:`); + * an entry of a flow mapping that is a step or a job — an anonymous + `{ ... }` at the line's level or in a `[` that `steps:` (or nothing on + the line) opens, or a mapping a job key opens inside `jobs: { ... }`. + + Text is not a site: a pin inside a quoted `name:` or `run:` scalar, after + `#`, inside an input's mapping (`with: { uses: ... }`), or after free text + (`echo { uses: ... }` in a script). """ @spec parse_pins(String.t()) :: [map()] def parse_pins(line) do + masked = line |> String.replace(~r/^[-+]/, "") |> mask_scalars() + @pin_site_re - |> Regex.scan(mask_scalars(line), capture: :all_names) - |> Enum.map(fn [action, ref] -> %{action: action, base: action_base(action), ref: ref} end) + |> Regex.scan(masked, return: :index, capture: :all) + |> Enum.filter(fn [_full, {action_at, _} | _] -> + structural_site?(binary_part(masked, 0, action_at)) + end) + |> Enum.map(fn [_full, {a_at, a_len}, {r_at, r_len}] -> + action = binary_part(masked, a_at, a_len) + %{action: action, base: action_base(action), ref: binary_part(masked, r_at, r_len)} + end) + end + + # `prefix` is the masked line up to the site's action. + defp structural_site?(prefix) do + cond do + Regex.match?(~r/^\s*(?:-\s+)?(?:\{\s*)?["']?uses["']?\s*:\s*["']?$/, prefix) -> + true + + Regex.match?(~r/[{,]\s*["']?uses["']?\s*:\s*["']?$/, prefix) -> + prefix |> flow_stack() |> structural_flow_stack?() + + true -> + false + end + end + + # The brackets still open in `text`, innermost first, each with what opens + # it: the key before it (`steps: [`), :none (nothing but indentation or a + # list dash), or :text (anything else — `echo {` is a script, not YAML). + defp flow_stack(text) do + text + |> String.graphemes() + |> Enum.reduce({[], ""}, fn + c, {stack, seg} when c in ["{", "["] -> {[{c, opener_key(seg)} | stack], ""} + c, {stack, _seg} when c in ["}", "]"] -> {Enum.drop(stack, 1), ""} + ",", {stack, _seg} -> {stack, ""} + c, {stack, seg} -> {stack, seg <> c} + end) + |> elem(0) + end + + defp opener_key(seg) do + cond do + Regex.match?(~r/^\s*(?:-\s*)?$/, seg) -> :none + m = Regex.run(~r/^\s*(?:-\s+)?["']?([\w.-]+)["']?\s*:\s*$/, seg) -> Enum.at(m, 1) + true -> :text + end + end + + defp structural_flow_stack?(stack) do + Enum.all?(stack, fn {_, key} -> key != :text end) and + case stack do + [{"{", :none}] -> true + [{"{", :none}, {"[", key} | _] when key in ["steps", :none] -> true + [{"{", job}, {"{", "jobs"} | _] when is_binary(job) -> true + _ -> false + end end # The line with its comment dropped and every quoted scalar that is @@ -549,18 +614,68 @@ defmodule Hypatia.Rules.PrAutomerge do defp take_quoted(<>, q, acc), do: take_quoted(rest, q, acc <> <>) - defp content_lines(patch) do - patch + # Every changed line of a patch, paired with whether it sits inside a + # block scalar (`run: |`, `script: >-`) on its own side of the diff, as far + # as the hunk shows: the context and changed lines of each side are read in + # order, and a `key: |` line opens a scalar that runs while lines are + # indented deeper than its key. A pin written inside a script is text. + defp changed_lines(file) do + file + |> patch_of() |> String.split("\n") - |> Enum.filter(&Regex.match?(~r/^[-+]/, &1)) - |> Enum.reject(&Regex.match?(~r/^[-+]{3}/, &1)) + |> Enum.reduce({[], nil, nil}, fn line, {acc, old, new} -> + cond do + String.starts_with?(line, "@@") -> + {acc, nil, nil} + + Regex.match?(~r/^[-+]{3}/, line) -> + {acc, old, new} + + String.starts_with?(line, "+") -> + {inside?, new} = scalar_step(new, String.slice(line, 1..-1//1)) + {[{line, inside?} | acc], old, new} + + String.starts_with?(line, "-") -> + {inside?, old} = scalar_step(old, String.slice(line, 1..-1//1)) + {[{line, inside?} | acc], old, new} + + String.starts_with?(line, " ") or line == "" -> + text = String.slice(line, 1..-1//1) + {_, old} = scalar_step(old, text) + {_, new} = scalar_step(new, text) + {acc, old, new} + + true -> + {acc, old, new} + end + end) + |> elem(0) + |> Enum.reverse() + end + + # One line of one side: is it inside the open block scalar (`state` is the + # column of the key that opened it, or nil), and what is open after it? + defp scalar_step(state, text) do + indent = String.length(text) - String.length(String.trim_leading(text, " ")) + inside? = state != nil and (String.trim(text) == "" or indent > state) + + cond do + inside? -> + {true, state} + + Regex.match?(~r/:\s*[|>][-+0-9]*\s*$/, mask_scalars(text)) -> + {false, text |> String.replace(~r/[^\s-].*$/, "") |> String.length()} + + true -> + {false, nil} + end end defp added_lines(file), - do: file |> patch_of() |> content_lines() |> Enum.filter(&String.starts_with?(&1, "+")) + do: for({line, false} <- changed_lines(file), String.starts_with?(line, "+"), do: line) defp removed_lines(file), - do: file |> patch_of() |> content_lines() |> Enum.filter(&String.starts_with?(&1, "-")) + do: for({line, false} <- changed_lines(file), String.starts_with?(line, "-"), do: line) defp patch_of(file), do: Map.get(file, :patch) || Map.get(file, "patch") || "" diff --git a/lib/rules/workflow_audit.ex b/lib/rules/workflow_audit.ex index b03c5401..f02dd5ad 100644 --- a/lib/rules/workflow_audit.ex +++ b/lib/rules/workflow_audit.ex @@ -1115,11 +1115,13 @@ defmodule Hypatia.Rules.WorkflowAudit do # The 40 hex characters must end the ref: `@-patched` is a tag that # merely begins with one, and a tag carries its own channel. Git resolves # a commit ID in either case, so upper-case hex is the same pin. - @toolchain_sha_pin ~r/^\s*(?:-\s+)?["']?uses["']?\s*:\s*["']?dtolnay\/rust-toolchain@([0-9a-fA-F]{40})(?=["'\s#,}\]]|$)/ + # The owner/repository is matched in either case — GitHub resolves + # `Dtolnay/Rust-Toolchain` to the same action — and the SHA as written. + @toolchain_sha_pin ~r/^\s*(?:-\s+)?["']?uses["']?\s*:\s*["']?(?i:dtolnay\/rust-toolchain)@([0-9a-fA-F]{40})(?=["'\s#,}\]]|$)/ # The same key written inside a flow mapping on the line that opens it: # `- { uses: dtolnay/rust-toolchain@ }`, `steps: [ { uses: ... } ]`. - @toolchain_sha_pin_inline ~r/[{,]\s*["']?uses["']?\s*:\s*["']?dtolnay\/rust-toolchain@([0-9a-fA-F]{40})(?=["'\s#,}\]]|$)/ + @toolchain_sha_pin_inline ~r/[{,]\s*["']?uses["']?\s*:\s*["']?(?i:dtolnay\/rust-toolchain)@([0-9a-fA-F]{40})(?=["'\s#,}\]]|$)/ @doc """ WF026: a SHA-pinned `dtolnay/rust-toolchain` step that passes no @@ -1193,7 +1195,16 @@ defmodule Hypatia.Rules.WorkflowAudit do |> Enum.flat_map(fn {line, number} -> case Regex.run(@toolchain_sha_pin, line) do [_full, sha] -> - wf026_block_step_finding(filename, lines, line, number, sha, kyaml?) + # The fleet fixer matches the identity in lower case only; any + # other spelling is reported for review rather than handed to it. + review = + cond do + kyaml? -> :kyaml + not String.contains?(line, "dtolnay/rust-toolchain@") -> :spelling + true -> nil + end + + wf026_block_step_finding(filename, lines, line, number, sha, review) _ -> # Several flow-style steps may share one physical line @@ -1202,7 +1213,8 @@ defmodule Hypatia.Rules.WorkflowAudit do |> Regex.scan(line, return: :index) |> Enum.flat_map(fn [{at, _}, {sha_at, 40}] -> sha = binary_part(line, sha_at, 40) - wf026_inline_step_finding(filename, lines, line, number, at + 1, sha, kyaml?) + review = if kyaml?, do: :kyaml + wf026_inline_step_finding(filename, lines, line, number, at + 1, sha, review) end) end end) @@ -1210,7 +1222,7 @@ defmodule Hypatia.Rules.WorkflowAudit do end # The block-style path: `uses:` is a key on its own line. - defp wf026_block_step_finding(filename, lines, line, number, sha, kyaml?) do + defp wf026_block_step_finding(filename, lines, line, number, sha, review) do {step, key_indent, opener} = wf026_step(lines, number) flow_item? = Regex.match?(~r/^\s*-\s+\{/, Enum.at(lines, opener - 1)) @@ -1232,7 +1244,7 @@ defmodule Hypatia.Rules.WorkflowAudit do verdict -> form = if flow_item?, do: :flow_step, else: elem(verdict, 1) - [wf026_finding(filename, number, sha, verdict, form, kyaml?)] + [wf026_finding(filename, number, sha, verdict, form, review)] end end end @@ -1240,13 +1252,13 @@ defmodule Hypatia.Rules.WorkflowAudit do # The inline path: `uses:` sits inside a flow mapping on the line that # opens it. The mapping is read through its braces (quote-aware, across # lines) and its `with` entry decides. - defp wf026_inline_step_finding(filename, lines, line, number, uses_at, sha, kyaml?) do + defp wf026_inline_step_finding(filename, lines, line, number, uses_at, sha, review) do with {:ok, open_col} <- wf_flow_opener_before(line, uses_at), true <- wf026_inline_step_at?(lines, line, number, open_col), {:ok, interior, _} <- wf_flow_interior(lines, number, open_col) do case wf026_with_entry(wf_flow_entries(interior), lines) do {:toolchain, _} -> [] - verdict -> [wf026_finding(filename, number, sha, verdict, :flow_step, kyaml?)] + verdict -> [wf026_finding(filename, number, sha, verdict, :flow_step, review)] end else _ -> [] @@ -1325,8 +1337,11 @@ defmodule Hypatia.Rules.WorkflowAudit do # The finding. Only a plain block-style step is handed to the fixer: # the fleet script refuses flow-style steps, anchors and aliases (exit # 2, file untouched), and a KYAML document is a flow-style document. - defp wf026_finding(filename, number, sha, verdict, form, kyaml?) do - repairable? = not kyaml? and form in [:none, :block, :flow] + # `review` says why a finding must route to review whatever its form: + # :kyaml (a flow-style document) or :spelling (the action named in a case + # the fixer does not match); nil when only the form decides. + defp wf026_finding(filename, number, sha, verdict, form, review) do + repairable? = review == nil and form in [:none, :block, :flow] shape = case {verdict, form} do @@ -1342,10 +1357,15 @@ defmodule Hypatia.Rules.WorkflowAudit do " Its `with:` goes through a YAML anchor or alias; the repair script does not " <> "rewrite anchored mappings, so this one routes to review." - _ when kyaml? -> + _ when review == :kyaml -> " The workflow is KYAML; the repair script rewrites block-style steps only, " <> "so this one routes to review." + _ when review == :spelling -> + " The action is not spelled `dtolnay/rust-toolchain`; GitHub resolves the name in " <> + "either case, but the repair script matches it as written in lower case, so this " <> + "one routes to review." + _ -> "" end @@ -2114,22 +2134,18 @@ defmodule Hypatia.Rules.WorkflowAudit do end # Is the `uses` at `uses_at` the entry of a flow mapping a job key opens? - # Either the job's own line under `jobs:` (`rust: { uses: ... }`) or the - # whole `jobs:` mapping written on one line (`jobs: { rust: { uses: ... } }`). - # The openers still unclosed before `uses` say which: one, opened by a key - # on a line under `jobs:`; two, the outer one opened by `jobs:` at the top - # of the document. The opening key is the last entry before its brace, - # split quote-aware, so a sibling's `name: "CI, build"` separates nothing. + # The job's own line under `jobs:` (`rust: { uses: ... }`), the whole + # `jobs:` mapping written on one line (`jobs: { rust: { uses: ... } }`), or + # the same inside a root `{ ... }`: the keys that open the brackets still + # unclosed before `uses` (nearest first), then the line's block ancestors, + # must read a job key, then `jobs`. Each key is the last entry before its + # brace, split quote-aware, so a sibling's `name: "CI, build"` separates + # nothing. defp wf027_inline_job?(lines, line, number, uses_at) do case wf_flow_opener_stack(line, uses_at) do - {:ok, [open]} when is_integer(open) -> - wf027_key_before(line, 0, open) != nil and - match?(["jobs" | _], wf_ancestor_keys(lines, number)) - - {:ok, [open, outer]} when is_integer(open) and is_integer(outer) -> - wf027_key_before(line, 0, outer) == "jobs" and - wf027_key_before(line, outer + 1, open) != nil and - wf_ancestor_keys(lines, number) == [] + {:ok, [open | _] = stack} when is_integer(open) -> + keys = wf_flow_opener_keys(line, stack) ++ (wf_ancestor_keys(lines, number) || []) + match?([job, "jobs" | _] when is_binary(job), keys) _ -> false diff --git a/scripts/sweeps/estate-pin-integrity.sh b/scripts/sweeps/estate-pin-integrity.sh index 098398b6..318358ad 100755 --- a/scripts/sweeps/estate-pin-integrity.sh +++ b/scripts/sweeps/estate-pin-integrity.sh @@ -485,7 +485,12 @@ while IFS=$'\t' read -r org repo path; do if line_has_denied_sha "$out" && { [ "$site_seen" -eq 0 ] || ! line_has_denied_sha "$mout"; }; then outside_uses=$((outside_uses + 1)); fixable=0 fi - if [ "$replaced" -eq 1 ] && [ -n "$good_version" ] && [ "$good_version" != "null" ]; then + # The comment is relabelled only when the line has ONE `uses:` site: on a + # flow line with several, the trailing comment cannot be tied to the + # replaced one (`{ uses: }, { uses: actions/checkout@ } # v7` + # annotates checkout), so it is kept as written. + nsites=$(printf '%s\n' "$mline" | grep -oiE "(^|[-{,[:space:]])${USES_KEY}[^@\"'[:space:],}]+@" | wc -l || true) + if [ "$replaced" -eq 1 ] && [ "$nsites" -eq 1 ] && [ -n "$good_version" ] && [ "$good_version" != "null" ]; then at=$(comment_column "$out") if [ "$at" -gt 0 ]; then out="${out:0:$((at - 1))}$(relabel_comment "${out:$((at - 1))}" "$good_version")" diff --git a/test/rules/pr_automerge_test.exs b/test/rules/pr_automerge_test.exs index 90a9cfc3..d5bc9a5a 100644 --- a/test/rules/pr_automerge_test.exs +++ b/test/rules/pr_automerge_test.exs @@ -386,6 +386,56 @@ defmodule Hypatia.Rules.PrAutomergeTest do assert site.denylist_id == "PIN-001" end + test "a one-line flow sequence with a major bump and a poison pin closes both reasons" do + patch = """ + @@ -1,1 +1,1 @@ + - steps: [ { uses: actions/checkout@v4.1.7 }, { uses: github/codeql-action/init@#{@good} } ] + + steps: [ { uses: actions/checkout@v7.0.1 }, { uses: github/codeql-action/init@#{@poison} } ] + """ + + resolution = %{ + {"actions/checkout", "v4.1.7"} => "4.1.7", + {"actions/checkout", "v7.0.1"} => "7.0.1" + } + + decision = + PA.classify( + pr([file(".github/workflows/ci.yml", patch)], resolution: resolution), + @policy + ) + + assert decision.major_delta + assert decision.disposition == :close_poison_and_majors + end + + test "a pin written inside a run: | script is text, not a site" do + in_hunk = """ + @@ -1,4 +1,4 @@ + - run: | + cat > fixture.yml <<'EOF' + - - uses: github/codeql-action/init@#{@good} + + - uses: github/codeql-action/init@#{@poison} + EOF + """ + + f = file(".github/workflows/ci.yml", in_hunk) + assert PA.poison_sites([f], @policy) == [] + refute PA.pin_lines_only?(f) + + free_text = """ + @@ -9,1 +9,1 @@ + - echo done + + echo { uses: github/codeql-action/init@#{@poison} } + """ + + assert PA.poison_sites([file(".github/workflows/ci.yml", free_text)], @policy) == [] + end + + test "a pin inside an input's mapping is data, not a site" do + line = "+ with: { uses: github/codeql-action/init@#{@poison} }" + assert PA.parse_pins(line) == [] + end + test "a pin quoted inside another scalar or a comment is not a site" do line = ~s(+ - { name: "see uses: github/codeql-action/init@#{@poison}", uses: actions/checkout@v4.1.7 } # uses: x/y@#{@poison}) diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs index 752a876c..37d986cc 100644 --- a/test/rules/workflow_audit_pins_test.exs +++ b/test/rules/workflow_audit_pins_test.exs @@ -348,6 +348,20 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) end + test "the action is matched in either case; a spelling the fixer cannot match routes to review" do + wf = """ + jobs: + check: + steps: + - uses: Dtolnay/Rust-Toolchain@#{@toolchain_sha} + """ + + assert [%{rule: "WF026", line: 4, auto_fixable: false, fix_script: nil, reason: reason}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + + assert reason =~ "not spelled" + end + test "a workflow written in flow style on one line is read through its keys" do sha2 = String.duplicate("a", 40) @@ -743,6 +757,15 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => own_line}) end + test "a root-braced one-line workflow is read through its keys" do + root = + "{ on: push, jobs: { rust: { uses: #{@rust_ci}@#{@broken_standards} }, " <> + "build: { steps: [ { uses: #{@rust_ci}@#{@broken_standards} } ] } } }\n" + + assert [%{rule: "WF027", line: 1, current_sha: @broken_standards}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => root}) + end + test "a KYAML caller is read as written: the quoted job-level uses: fires, fixtures do not" do wf = """ { From 3788540e80a4bde70fe653896a354d58369cf47e Mon Sep 17 00:00:00 2001 From: "coderabbitai[bot]" <136622811+coderabbitai[bot]@users.noreply.github.com> Date: Tue, 20 Oct 2026 00:37:15 +0000 Subject: [PATCH 28/47] fix(pins): tighten diff checks, ignore heredocs and flag partial sweeps --- lib/rules/pr_automerge.ex | 34 +++++++++++++++---- lib/rules/workflow_audit.ex | 12 +++---- scripts/sweeps/estate-pin-integrity.sh | 24 +++++++++----- test/rules/pr_automerge_test.exs | 44 +++++++++++++++++++++++-- test/rules/workflow_audit_pins_test.exs | 32 ++++++++++++++++++ 5 files changed, 124 insertions(+), 22 deletions(-) diff --git a/lib/rules/pr_automerge.ex b/lib/rules/pr_automerge.ex index 44dc15a3..590f66dd 100644 --- a/lib/rules/pr_automerge.ex +++ b/lib/rules/pr_automerge.ex @@ -485,8 +485,6 @@ defmodule Hypatia.Rules.PrAutomerge do end end - defp pin_line?(line), do: parse_pin(line) != nil - # One `uses:` site anywhere on a line, bounded on the left so `reuses:` is # not one. Matched against the MASKED line (see mask_scalars/1). @pin_site_re ~r/(?:^|[^A-Za-z0-9_.\/-])["']?uses["']?\s*:\s*["']?(?[A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@(?[^\s#"',}\]]+)/ @@ -511,15 +509,39 @@ defmodule Hypatia.Rules.PrAutomerge do def parse_pins(line) do masked = line |> String.replace(~r/^[-+]/, "") |> mask_scalars() + masked + |> pin_sites() + |> Enum.map(fn [_full, {a_at, a_len}, {r_at, r_len}] -> + action = binary_part(masked, a_at, a_len) + %{action: action, base: action_base(action), ref: binary_part(masked, r_at, r_len)} + end) + end + + defp pin_sites(masked) do @pin_site_re |> Regex.scan(masked, return: :index, capture: :all) |> Enum.filter(fn [_full, {action_at, _} | _] -> structural_site?(binary_part(masked, 0, action_at)) end) - |> Enum.map(fn [_full, {a_at, a_len}, {r_at, r_len}] -> - action = binary_part(masked, a_at, a_len) - %{action: action, base: action_base(action), ref: binary_part(masked, r_at, r_len)} - end) + end + + defp pin_line?(line) do + masked = line |> String.replace(~r/^[-+]/, "") |> mask_scalars() + sites = pin_sites(masked) + + remaining = + sites + |> Enum.reverse() + |> Enum.reduce(masked, fn [{at, len}, {action_at, _}, _], rest -> + # The site match includes an opening value quote but ends at the ref. + # Remove its matching closing quote along with the pin. + quote = binary_part(masked, action_at - 1, 1) + closing = binary_part(rest, at + len, min(1, byte_size(rest) - at - len)) + len = if quote in ["\"", "'"] and closing == quote, do: len + 1, else: len + binary_part(rest, 0, at) <> binary_part(rest, at + len, byte_size(rest) - at - len) + end) + + sites != [] and Regex.match?(~r/^[\s\-\[\]{},]*$/, remaining) end # `prefix` is the masked line up to the site's action. diff --git a/lib/rules/workflow_audit.ex b/lib/rules/workflow_audit.ex index f02dd5ad..2fc1165b 100644 --- a/lib/rules/workflow_audit.ex +++ b/lib/rules/workflow_audit.ex @@ -1282,9 +1282,9 @@ defmodule Hypatia.Rules.WorkflowAudit do # the line, nearest first, then the block ancestors of the line. Each key # is read quote-aware, so a `name: "CI, build"` sibling separates nothing. defp wf026_flow_path_step?(lines, line, number, open_col) do - case wf_flow_opener_stack(line, open_col) do - {:ok, [{:seq, _} | _] = stack} -> - keys = wf_flow_opener_keys(line, stack) ++ (wf_ancestor_keys(lines, number) || []) + case {wf_flow_opener_stack(line, open_col), wf_ancestor_keys(lines, number)} do + {{:ok, [{:seq, _} | _] = stack}, ancestors} when is_list(ancestors) -> + keys = wf_flow_opener_keys(line, stack) ++ ancestors match?(["steps", job, "jobs" | _] when is_binary(job), keys) _ -> @@ -2142,9 +2142,9 @@ defmodule Hypatia.Rules.WorkflowAudit do # brace, split quote-aware, so a sibling's `name: "CI, build"` separates # nothing. defp wf027_inline_job?(lines, line, number, uses_at) do - case wf_flow_opener_stack(line, uses_at) do - {:ok, [open | _] = stack} when is_integer(open) -> - keys = wf_flow_opener_keys(line, stack) ++ (wf_ancestor_keys(lines, number) || []) + case {wf_flow_opener_stack(line, uses_at), wf_ancestor_keys(lines, number)} do + {{:ok, [open | _] = stack}, ancestors} when is_integer(open) and is_list(ancestors) -> + keys = wf_flow_opener_keys(line, stack) ++ ancestors match?([job, "jobs" | _] when is_binary(job), keys) _ -> diff --git a/scripts/sweeps/estate-pin-integrity.sh b/scripts/sweeps/estate-pin-integrity.sh index 318358ad..357715f7 100755 --- a/scripts/sweeps/estate-pin-integrity.sh +++ b/scripts/sweeps/estate-pin-integrity.sh @@ -28,9 +28,9 @@ # * `--via clone` is the authoritative mode (reads the files themselves); # `--via search` is the fast reconnaissance mode and can lag the index. # -# EXIT CODES: 0 clean, 1 error, 2 findings present (or the sweep could not -# see everything: a search that failed or overflowed, a -# candidate that could not be read — never reported clean), +# EXIT CODES: 0 clean, 1 error, 2 confirmed findings or an INCOMPLETE run +# (failed/partial searches, listings or clones, unread candidates, +# or limits imposed by --max-repos or --sample), # 3 nothing to do # # usage: estate-pin-integrity.sh [--org ORG]... [--policy FILE] [--out DIR] @@ -197,7 +197,10 @@ masked_sites() { printf '%s\n' "$1" | grep -oiE "$SITE_RE" || true; } # inside a quoted scalar or a comment earlier on the line is never the one # replaced. replace_site() { # - awk -v line="$1" -v masked="$2" -v a="$3" -v r="$4" -v good="$5" -v sq="'" 'BEGIN { + PIN_LINE="$1" PIN_MASKED="$2" PIN_ACTION="$3" PIN_REF="$4" PIN_GOOD="$5" \ + awk -v sq="'" 'BEGIN { + line = ENVIRON["PIN_LINE"]; masked = ENVIRON["PIN_MASKED"] + a = ENVIRON["PIN_ACTION"]; r = ENVIRON["PIN_REF"]; good = ENVIRON["PIN_GOOD"] tail = "(^|[-{,[:space:]])[\"" sq "]?uses[\"" sq "]?[[:space:]]*:[[:space:]]*[\"" sq "]?$" ends = "]\"" sq " \t#,}" lm = tolower(masked); needle = tolower(a "@" r); from = 1 @@ -319,6 +322,7 @@ mapfile -t SEARCH_TOKENS < <(jq -r ' # clean org. incomplete=0 failed=0 +skipped_repos=0 if [ "$VIA" = "search" ]; then for org in "${ORGS[@]}"; do for token in "${SEARCH_TOKENS[@]}"; do @@ -366,7 +370,10 @@ else fi reponames=() [ -n "$names" ] && mapfile -t reponames <<<"$names" - [ "$MAX_REPOS" -gt 0 ] && reponames=("${reponames[@]:0:$MAX_REPOS}") + if [ "$MAX_REPOS" -gt 0 ] && [ "${#reponames[@]}" -gt "$MAX_REPOS" ]; then + skipped_repos=$((skipped_repos + ${#reponames[@]} - MAX_REPOS)) + reponames=("${reponames[@]:0:$MAX_REPOS}") + fi log " cloning ${#reponames[@]} repo(s) in ${org}" # Clones live under the organisation: two organisations may each own a # repository of the same name, and the second must not reuse the first's. @@ -399,6 +406,7 @@ unseen() { local parts=() [ "$incomplete" -gt 0 ] && parts+=("${incomplete} search page(s) overflowed GitHub's 1,000-result cap or came back marked incomplete") [ "$failed" -gt 0 ] && parts+=("${failed} search(es), listing(s) or clone(s) failed") + [ "$skipped_repos" -gt 0 ] && parts+=("${skipped_repos} repo(s) skipped by --max-repos") [ "${unread:-0}" -gt 0 ] && parts+=("${unread} candidate file(s) could not be read") [ "${sampled:-0}" -gt 0 ] && parts+=("${sampled} candidate file(s) left unread by --sample") local IFS=';' @@ -406,7 +414,7 @@ unseen() { } if [ "$total_hits" -eq 0 ]; then - if [ "$incomplete" -gt 0 ] || [ "$failed" -gt 0 ] || [ "$sampled" -gt 0 ]; then + if [ "$incomplete" -gt 0 ] || [ "$failed" -gt 0 ] || [ "$skipped_repos" -gt 0 ] || [ "$sampled" -gt 0 ]; then echo "INCOMPLETE: no denylisted pin tokens in the visible search results for ${ORGS[*]}, but $(unseen); a clean verdict needs --via clone" exit 2 fi @@ -539,7 +547,7 @@ while IFS=$'\t' read -r org repo path; do done < "$HITS" if [ "$confirmed" -eq 0 ]; then - if [ "$incomplete" -gt 0 ] || [ "$failed" -gt 0 ] || [ "$unread" -gt 0 ] || [ "$sampled" -gt 0 ]; then + if [ "$incomplete" -gt 0 ] || [ "$failed" -gt 0 ] || [ "$skipped_repos" -gt 0 ] || [ "$unread" -gt 0 ] || [ "$sampled" -gt 0 ]; then echo "INCOMPLETE: none of the ${total_hits} candidate file(s) carries a denylisted pin, but $(unseen); a clean verdict needs --via clone" exit 2 fi @@ -577,7 +585,7 @@ log " repair plan : ${PLAN}" [ "$REWRITE" -eq 1 ] && log " repaired trees : ${OUT_DIR}/patched" log " NOTE: this sweep never writes to a remote. Branch + PR creation is" log " the token-bearing actuator's job — docs/operations/estate-automerge.adoc" -if [ "$incomplete" -gt 0 ] || [ "$failed" -gt 0 ] || [ "$unread" -gt 0 ] || [ "$sampled" -gt 0 ]; then +if [ "$incomplete" -gt 0 ] || [ "$failed" -gt 0 ] || [ "$skipped_repos" -gt 0 ] || [ "$unread" -gt 0 ] || [ "$sampled" -gt 0 ]; then log " INCOMPLETE: $(unseen); what they hid was not seen — rerun with --via clone" fi diff --git a/test/rules/pr_automerge_test.exs b/test/rules/pr_automerge_test.exs index d5bc9a5a..9cff8c34 100644 --- a/test/rules/pr_automerge_test.exs +++ b/test/rules/pr_automerge_test.exs @@ -184,8 +184,9 @@ defmodule Hypatia.Rules.PrAutomergeTest do decision = PA.classify(pr([file(".github/workflows/codeql.yml", patch)]), @policy) - assert decision.disposition == :close_poison_only - assert decision.blocked_by == "introduces_denylisted_pin" + assert decision.disposition == :excise_poison_then_merge + refute decision.pin_only + assert decision.blocked_by == "introduces_denylisted_pin_alongside_wanted_updates" end test "a quoted uses: key and a quoted value are still a pin PA001 sees" do @@ -308,6 +309,45 @@ defmodule Hypatia.Rules.PrAutomergeTest do end describe "poison_sites/2 and pin_lines_only?/1" do + test "a flow pin with neighbouring input, condition or environment edits is not pin-only" do + for {before, after_edit} <- [ + {"with: { languages: rust }", "with: { languages: javascript }"}, + {"if: false", "if: true"}, + {"env: { MODE: test }", "env: { MODE: deploy }"} + ] do + patch = """ + @@ -1,1 +1,1 @@ + - - { uses: actions/checkout@v4.1.7, #{before} } + + - { uses: actions/checkout@v4.2.0, #{after_edit} } + """ + + decision = + PA.classify( + pr([file(".github/workflows/ci.yml", patch)], + resolution: %{ + {"actions/checkout", "v4.1.7"} => "4.1.7", + {"actions/checkout", "v4.2.0"} => "4.2.0" + } + ), + @policy + ) + + refute decision.pin_only + assert decision.disposition == :flag + end + end + + test "quoted flow pins and their comments are pin-only, punctuation alone is not" do + patch = """ + @@ -1,1 +1,1 @@ + - - [{ "uses": "actions/checkout@v4.1.7" }, { 'uses': 'actions/cache@v4' }] # old + + - [{ "uses": "actions/checkout@v4.2.0" }, { 'uses': 'actions/cache@v5' }] # new + """ + + assert PA.pin_lines_only?(file(".github/workflows/ci.yml", patch)) + refute PA.pin_lines_only?(file(".github/workflows/ci.yml", "@@ -1 +1 @@\n- []\n+ {}\n")) + end + test "the poisoned SHA is found on the added side only" do patch = codeql_patch(@good, @poison) sites = PA.poison_sites([file(".github/workflows/codeql.yml", patch)], @policy) diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs index 37d986cc..44886644 100644 --- a/test/rules/workflow_audit_pins_test.exs +++ b/test/rules/workflow_audit_pins_test.exs @@ -387,6 +387,22 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do Enum.filter(findings, &(&1.file == "jobline.yml")) end + test "a one-line flow workflow inside a run heredoc is not a toolchain step" do + for opener <- ["", "{ "], closer = if(opener == "", do: "", else: " }") do + wf = """ + jobs: + build: + steps: + - run: | + cat <<'YAML' + #{opener}jobs: { fixture: { steps: [ { uses: dtolnay/rust-toolchain@#{@toolchain_sha} } ] } }#{closer} + YAML + """ + + assert WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"x.yml" => wf}) == [] + end + end + test "a one-line flow path that is not steps in a job in jobs is not a step" do docs = %{ "env.yml" => @@ -757,6 +773,22 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => own_line}) end + test "a one-line flow workflow inside a run heredoc is not a reusable call" do + for opener <- ["", "{ "], closer = if(opener == "", do: "", else: " }") do + wf = """ + jobs: + build: + steps: + - run: | + cat <<'YAML' + #{opener}jobs: { fixture: { uses: #{@rust_ci}@#{@broken_standards} } }#{closer} + YAML + """ + + assert WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => wf}) == [] + end + end + test "a root-braced one-line workflow is read through its keys" do root = "{ on: push, jobs: { rust: { uses: #{@rust_ci}@#{@broken_standards} }, " <> From 20c8657517876b1d8ff849e70ee5f2b6157e48b2 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 02:41:37 +0000 Subject: [PATCH 29/47] fix(pins): one site reader for PI001 and PA00x, 1:1 pairing, hunk certainty, per-repo veto Codex round on 1b98ef2. - lib/rules/pin_sites.ex (new): the structural `uses:` site reader that PrAutomerge carried privately, now shared. PI001/PI002/PI004 read files with it, so compact flow steps (`- { name: x, uses: a/b@sha }`) and multi-site lines are sites in the file scan too. - PinIntegrity: action identity is case-folded in denylisted?/known_good and PI004 (PIN-003 now meets `Hyperpolymath/Standards/...`). The repair splices each ref at its own byte offset, right to left, and relabels the comment only on a one-site line; the comment is found outside quoted scalars. pin_only_edit? asks PinSites.pin_line? of every changed line. - PrAutomerge: each removed site pairs with one added site of the same action, and each added site is used once, so a second checkout's major bump is no longer hidden behind an unchanged first one. A changed line is "certain" only when every line of its side at its indentation or shallower is YAML; a denylisted pin the hunk shows among script text flags the PR (denylisted_pin_context_unknown) instead of closing or excising, and is not a pin line for MGX-001. - WF027: owner/repo matched case-insensitively, workflow path as written. - WF026: an anchor is looked for on the line with quoted scalars and the comment masked, so `note: "see &x"` is not `&x`. - estate-pin-integrity.sh: a repair plan is emitted only for a repository whose every confirmed file is repairable and every candidate file was read; others are held and logged. Also fixes the doubled `0` in the "repos with a repair ready" line. - docs: --sample, exit 2 for findings or INCOMPLETE, the held-repo rule, and the new blocked_by value. CLAUDE.md module total re-measured (142). Tests: 7 new cases; each fails on 3788540 and passes here. Fake-gh sweep: 3 new modes (sibling, tworepos, sibunread); the 15 earlier modes are byte-identical to the previous round under gawk and mawk. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- .claude/CLAUDE.md | 2 +- docs/operations/estate-automerge.adoc | 17 +- lib/rules/pin_integrity.ex | 193 +++++++------ lib/rules/pin_sites.ex | 206 ++++++++++++++ lib/rules/pr_automerge.ex | 361 +++++++++++------------- lib/rules/workflow_audit.ex | 50 +++- scripts/sweeps/estate-pin-integrity.sh | 36 ++- test/rules/pin_integrity_test.exs | 46 +++ test/rules/pr_automerge_test.exs | 60 ++++ test/rules/workflow_audit_pins_test.exs | 38 +++ 10 files changed, 699 insertions(+), 310 deletions(-) create mode 100644 lib/rules/pin_sites.ex diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index 557e2780..eca7821a 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -13,7 +13,7 @@ Hypatia is the neurosymbolic CI/CD intelligence layer for the hyperpolymath ecos ## Architecture ``` -Hypatia # 141 Elixir modules total (`shopt -s globstar; ls lib/**/*.ex | wc -l`) +Hypatia # 142 Elixir modules total (`shopt -s globstar; ls lib/**/*.ex | wc -l`) ├── Elixir pipeline # core: pattern analysis, dispatch, learning ├── Merge orchestration # 12 modules — leases, ledger, ticker (largest subsystem) ├── Kin subsystem # 6 modules — contingency, arbiter, gate, watchdog diff --git a/docs/operations/estate-automerge.adoc b/docs/operations/estate-automerge.adoc index db17e4d0..f9f3d2e3 100644 --- a/docs/operations/estate-automerge.adoc +++ b/docs/operations/estate-automerge.adoc @@ -89,7 +89,10 @@ Every examined PR gets one line in `pr-decisions.jsonl`: | `blocked_by` | `awaiting_required_checks`, `introduces_denylisted_pin`, `introduces_denylisted_pin_and_major_bumps`, - `introduces_denylisted_pin_alongside_wanted_updates`, `major_version_delta`, + `introduces_denylisted_pin_alongside_wanted_updates`, + `denylisted_pin_context_unknown` (the hunk shows the pin among script text; + brain only — the shell actuator does not yet read hunk context), + `major_version_delta`, `version_claims_conflict`, `pin_delta_unresolvable`, `dependency_manifest_not_lockfile`, `touches_licence_requires_owner_review`, `repository_is_archived_cannot_merge`, `not_a_dependency_bot`, @@ -127,11 +130,17 @@ Find every denylisted pin in an estate; with `--rewrite`, produce the substituted file trees ready for a branch and a PR. * Flags: `--org` (repeatable), `--policy`, `--out DIR`, `--via search|clone`, - `--rewrite`, `--max-repos N`. + `--rewrite`, `--max-repos N`, `--sample N`. * Artifacts under `--out`: `pin-findings.jsonl`, `pin-repair-plan.jsonl`, `patched//` trees; one squash PR is planned per repository - (`hypatia/pin-integrity-known-good`). -* Exit: `0` clean, `1` error, `2` findings, `3` nothing to do. + (`hypatia/pin-integrity-known-good`), and only for a repository whose every + confirmed file is repairable and whose every candidate file was read. A + repository with a flagged sibling (an `actions.lock` entry, a fixture) or an + unread candidate is held for owner review: repairing the workflow alone + would arm a merge on a repository that still carries the pin. +* Exit: `0` clean, `1` error, `2` findings or an `INCOMPLETE` run (failed or + partial searches, listings or clones, unread candidates, or the limits + `--max-repos` and `--sample` impose), `3` nothing to do. * A repair that cannot be proved is a finding, not an action: the substitution refuses when the policy carries no `known_good_sha`, and it refuses when the rewrite would drop a `uses:` site. diff --git a/lib/rules/pin_integrity.ex b/lib/rules/pin_integrity.ex index 26fd9328..24827be4 100644 --- a/lib/rules/pin_integrity.ex +++ b/lib/rules/pin_integrity.ex @@ -53,49 +53,58 @@ defmodule Hypatia.Rules.PinIntegrity do or to the owner as `flag` when the repair is not a substitution. """ - # A `uses:` pin site, in block form (`- uses: a/b@ref # v1`) or KYAML form - # (`uses: "a/b@ref", # v1` and one-line `{ uses: "a/b@ref" },`). The key - # must open the line — after indentation and an optional `-` or `{` — so a - # `uses:` inside a `run:` string never counts. The value may be quoted; the - # quote, trailing comma and closing brace are not part of the ref. - # The key itself may be quoted too (`"uses": "a/b@ref"`): GitHub reads - # the same key, so the scanner and the repair must as well. - @uses_regex ~r/^\s*(?:-\s*|\{\s*)?(?["']?)uses\k\s*:\s*(?["']?)(?[A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@(?[^\s#"',}]+)\k\s*\}?\s*,?\s*(?:#\s*(?.*?))?\s*$/ + alias Hypatia.Rules.PinSites @doc """ Parse every `uses:` pin site in a workflow file. + A site is what `Hypatia.Rules.PinSites` reads as one — block form + (`- uses: a/b@ref # v1`), KYAML form (`uses: "a/b@ref", # v1`), and every + step of a compact flow sequence or mapping (`steps: [{ uses: a/b@x }]`, + `- { name: x, uses: a/b@x }`) — so the file scan sees the pins the diff + rules (PA001-PA004) see. A `uses:` inside a quoted `run:` string, a + comment or script text is not one. + Returns a list of maps with `:line`, `:action` (the bare action path as written, e.g. `github/codeql-action/init`), `:action_base` (the first - two path segments, which is what an allow/deny list keys on), - `:ref`, and `:comment`. + two path segments, which is what an allow/deny list keys on), `:ref`, + `:ref_at` (the ref's byte offset in its line) and `:comment` — the + line's comment, trimmed, when the site is the line's only one, else `""`: + `[ { uses: a/b@x }, { uses: c/d@y } ] # v4` labels neither pin. """ @spec pin_sites(String.t()) :: [map()] def pin_sites(content) when is_binary(content) do content |> String.split("\n") |> Enum.with_index(1) - |> Enum.flat_map(fn {line, number} -> - case Regex.named_captures(@uses_regex, line) do - %{"action" => action, "ref" => ref} = caps -> - [ - %{ - line: number, - action: action, - action_base: action_base(action), - ref: ref, - comment: Map.get(caps, "comment", "") || "" - } - ] - - _ -> - [] - end - end) + |> Enum.flat_map(fn {line, number} -> line_sites(line, number) end) end def pin_sites(_), do: [] + defp line_sites(line, number) do + sites = PinSites.sites(line) + comment = if length(sites) == 1, do: line |> PinSites.comment() |> String.trim(), else: "" + + for site <- sites do + %{ + line: number, + action: site.action, + action_base: action_base(site.action), + ref: site.ref, + ref_at: site.ref_at, + comment: comment + } + end + end + + # GitHub resolves `GitHub/CodeQL-Action` and `github/codeql-action` to one + # repository, so a table keyed on one spelling must match the other. + defp same_action?(a, b) when is_binary(a) and is_binary(b), + do: String.downcase(a) == String.downcase(b) + + defp same_action?(_, _), do: false + @doc """ The first two segments of an action path. @@ -127,7 +136,7 @@ defmodule Hypatia.Rules.PinIntegrity do policy |> Map.get("pin_denylist", []) |> Enum.find(fn entry -> - Map.get(entry, "action") == base and + same_action?(Map.get(entry, "action"), base) and (normalised in normalise_all(Map.get(entry, "blocked_shas", [])) or normalised in normalise_all(Map.get(entry, "blocked_versions", [])) or normalised in normalise_all(Map.get(entry, "blocked_refs", []))) @@ -155,7 +164,10 @@ defmodule Hypatia.Rules.PinIntegrity do def known_good(policy, action) do base = action_base(action) - case Enum.find(Map.get(policy, "pin_denylist", []), &(Map.get(&1, "action") == base)) do + case Enum.find( + Map.get(policy, "pin_denylist", []), + &same_action?(Map.get(&1, "action"), base) + ) do %{"known_good_sha" => sha} = entry when is_binary(sha) -> %{sha: sha, version: Map.get(entry, "known_good_version"), rule: Map.get(entry, "id")} @@ -415,7 +427,7 @@ defmodule Hypatia.Rules.PinIntegrity do |> Enum.flat_map(fn site -> locked_refs_for_action = locked - |> Enum.filter(fn {_ref, {action, _line}} -> action == site.action_base end) + |> Enum.filter(fn {_ref, {action, _line}} -> same_action?(action, site.action_base) end) |> Enum.map(fn {ref, _} -> ref end) cond do @@ -487,31 +499,18 @@ defmodule Hypatia.Rules.PinIntegrity do {:error, "no known_good_sha for a denylisted action in this file"} true -> - {new_content, excisions} = - Enum.reduce(hits, {content, []}, fn site, {acc, log} -> - original = Enum.at(lines, site.line - 1) || "" - %{sha: sha, version: version} = known_good(policy, site.action) - - rewritten = - original - |> String.replace("@" <> site.ref, "@" <> sha) - |> relabel_line(version) - - excision = %{ - line: site.line, - action: site.action, - from_ref: site.ref, - to_ref: sha, - to_version: version, - comment_before: site.comment, - comment_after: comment_of(rewritten) - } - - # Whole-line replacement, so two identical pin lines are treated - # identically and neither is touched twice. - {String.replace(acc, original, rewritten), log ++ [excision]} + {new_lines, excisions} = + lines + |> Enum.with_index(1) + |> Enum.map_reduce([], fn {original, number}, log -> + case Enum.filter(hits, &(&1.line == number)) do + [] -> {original, log} + line_hits -> rewrite_line(original, line_hits, sites, policy, log) + end end) + new_content = Enum.join(new_lines, "\n") + if length(pin_sites(new_content)) == length(sites) do {:ok, new_content, excisions} else @@ -520,20 +519,57 @@ defmodule Hypatia.Rules.PinIntegrity do end end - defp comment_of(line) do - case String.split(line, "#", parts: 2) do - [_head, comment] -> comment - _ -> "" - end + # Each denylisted ref is spliced at its own offset, right to left so the + # offsets still to come stay valid: a line that carries the same ref twice, + # or the ref's text again in a `name:`, changes only at its sites. The + # comment is relabelled only on a line whose one site was the hit — on a + # multi-site line it labels no pin in particular. + defp rewrite_line(original, line_hits, sites, policy, log) do + rewritten = + line_hits + |> Enum.sort_by(& &1.ref_at, :desc) + |> Enum.reduce(original, fn site, acc -> + %{sha: sha} = known_good(policy, site.action) + tail_at = site.ref_at + byte_size(site.ref) + + binary_part(acc, 0, site.ref_at) <> + sha <> binary_part(acc, tail_at, byte_size(acc) - tail_at) + end) + + rewritten = + case {line_hits, Enum.count(sites, &(&1.line == hd(line_hits).line))} do + {[site], 1} -> relabel_line(rewritten, known_good(policy, site.action).version) + _ -> rewritten + end + + excisions = + for site <- line_hits do + %{sha: sha, version: version} = known_good(policy, site.action) + + %{ + line: site.line, + action: site.action, + from_ref: site.ref, + to_ref: sha, + to_version: version, + comment_before: site.comment, + comment_after: PinSites.comment(rewritten) + } + end + + {rewritten, log ++ excisions} end + # The comment is the text after the `#` PinSites finds outside quoted + # scalars, so `name: "C# build"` is not cut in two. defp relabel_line(line, version) when is_binary(version) do - case String.split(line, "#", parts: 2) do - # Mirror estate-pin-integrity.sh: the head keeps no `#`, the comment - # is handed over WITH it, and relabel/2 returns it with it. - [head, comment] -> head <> relabel("#" <> comment, version) - _ -> line - end + head = PinSites.head(line) + + # Mirror estate-pin-integrity.sh: the head keeps no `#`, the comment + # is handed over WITH it, and relabel/2 returns it with it. + if byte_size(head) < byte_size(line), + do: head <> relabel("#" <> PinSites.comment(line), version), + else: line end defp relabel_line(line, _version), do: line @@ -611,8 +647,8 @@ defmodule Hypatia.Rules.PinIntegrity do """ @spec pin_only_edit?([String.t()], [String.t()], map()) :: boolean() def pin_only_edit?(removed, added, policy) do - removed_sites = lines_to_sites(removed) - added_sites = lines_to_sites(added) + removed_sites = Enum.flat_map(removed, &line_sites(&1, 0)) + added_sites = Enum.flat_map(added, &line_sites(&1, 0)) pinned_removed = removed_sites |> Enum.map(& &1.ref) |> Enum.sort() pinned_added = added_sites |> Enum.map(& &1.ref) |> Enum.sort() @@ -621,12 +657,12 @@ defmodule Hypatia.Rules.PinIntegrity do removed == [] or added == [] -> false - length(removed_sites) != length(removed) or length(added_sites) != length(added) -> + not Enum.all?(removed ++ added, &PinSites.pin_line?/1) -> # At least one changed line is not a `uses:` line. false - Enum.sort(Enum.map(removed_sites, & &1.action)) != - Enum.sort(Enum.map(added_sites, & &1.action)) -> + Enum.sort(Enum.map(removed_sites, &String.downcase(&1.action))) != + Enum.sort(Enum.map(added_sites, &String.downcase(&1.action))) -> # An action was swapped for a different action. false @@ -643,25 +679,6 @@ defmodule Hypatia.Rules.PinIntegrity do end end - defp lines_to_sites(lines) do - lines - |> Enum.map(&Regex.named_captures(@uses_regex, &1)) - |> Enum.flat_map(fn - %{"action" => action, "ref" => ref} = caps -> - [ - %{ - action: action, - action_base: action_base(action), - ref: ref, - comment: Map.get(caps, "comment", "") || "" - } - ] - - _ -> - [] - end) - end - # ─── PI005 · stale pin (advisory) ───────────────────────────────────── @doc """ diff --git a/lib/rules/pin_sites.ex b/lib/rules/pin_sites.ex new file mode 100644 index 00000000..2044c171 --- /dev/null +++ b/lib/rules/pin_sites.ex @@ -0,0 +1,206 @@ +# SPDX-License-Identifier: MPL-2.0 +# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) + +defmodule Hypatia.Rules.PinSites do + @moduledoc """ + Structural `uses:` pin sites on one line of workflow YAML. + + One reader for every rule that judges a pin by the line it sits on — + PI001/PI002 (`PinIntegrity`) over a file, PA001-PA004 (`PrAutomerge`) over + a diff — so a pin shape one of them learns, the others see too. + + A site is a `uses:` key the YAML structure makes a pin: + + * the line's own leading `uses:` key (`uses:`, `- uses:`, `- { uses:`, + quoted or not); + * an entry of a flow mapping that is a step or a job — an anonymous + `{ ... }` at the line's level or in a `[` that `steps:` (or nothing on + the line) opens, or a mapping a job key opens inside `jobs: { ... }`. + + Text is not a site: a pin inside a quoted `name:` or `run:` scalar, after + `#`, inside an input's mapping (`with: { uses: ... }`), or after free text + (`echo { uses: ... }` in a script). + + Every position this module returns is a byte offset into the line as + given: the mask keeps the line's bytes where they were. + """ + + # One `uses:` site anywhere on a line, bounded on the left so `reuses:` is + # not one. Matched against the MASKED line. + @site_re ~r/(?:^|[^A-Za-z0-9_.\/-])["']?uses["']?\s*:\s*["']?(?[A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@(?[^\s#"',}\]]+)/ + + @uses_tail_re ~r/(?:^|[-{,\s])["']?uses["']?\s*:\s*$/ + + @doc """ + The structural sites of `line`, in order, as + `%{action: a, ref: r, action_at: byte, ref_at: byte}`. + """ + @spec sites(String.t()) :: [map()] + def sites(line) when is_binary(line) do + masked = mask(line) + + masked + |> scan() + |> Enum.map(fn [_full, {a_at, a_len}, {r_at, r_len}] -> + %{ + action: binary_part(masked, a_at, a_len), + ref: binary_part(masked, r_at, r_len), + action_at: a_at, + ref_at: r_at + } + end) + end + + @doc """ + The structural site matches of an already masked line, each as + `[full, action, ref]` `{byte, length}` pairs (`Regex.scan/3` with + `return: :index`). + """ + @spec scan(String.t()) :: [[{non_neg_integer(), non_neg_integer()}]] + def scan(masked) when is_binary(masked) do + @site_re + |> Regex.scan(masked, return: :index, capture: :all) + |> Enum.filter(fn [_full, {action_at, _} | _] -> + structural?(binary_part(masked, 0, action_at)) + end) + end + + @doc """ + The line cut at its comment, with every quoted scalar that is neither a + key nor the value of a `uses` key blanked to `x` byte for byte. A `uses` + value and a quoted key are kept as written, so text read from the masked + line at a site is the pin's own text, and an offset into it is an offset + into the line. + """ + @spec mask(String.t()) :: String.t() + def mask(line) when is_binary(line), do: do_mask(line, "") + + @doc """ + Is the line nothing but pin sites? Every structural site removed (with its + value's closing quote), what is left of the masked line may only be + indentation, list dashes, flow brackets and commas — so `- uses: a/b@x`, + `{ uses: "a/b@x" },` and `[ { uses: a/b@x }, { uses: c/d@y } ]` are pin + lines, and `steps: [ { uses: a/b@x } ]`, `- { name: x, uses: a/b@x }` + and `uses: a/b@x extra` are not. The comment is never part of it. + """ + @spec pin_line?(String.t()) :: boolean() + def pin_line?(line) when is_binary(line) do + masked = mask(line) + sites = scan(masked) + + remaining = + sites + |> Enum.reverse() + |> Enum.reduce(masked, fn [{at, len}, {action_at, _}, _], rest -> + # The site match includes an opening value quote but ends at the ref. + # Remove its matching closing quote along with the pin. + quote = binary_part(masked, action_at - 1, 1) + closing = binary_part(rest, at + len, min(1, byte_size(rest) - at - len)) + len = if quote in ["\"", "'"] and closing == quote, do: len + 1, else: len + binary_part(rest, 0, at) <> binary_part(rest, at + len, byte_size(rest) - at - len) + end) + + sites != [] and Regex.match?(~r/^[\s\-\[\]{},]*$/, remaining) + end + + @doc """ + The text after the line's comment `#`, as written (untrimmed), or `""`. + The `#` is found outside quoted scalars, so `name: "C# build"` has none. + """ + @spec comment(String.t()) :: String.t() + def comment(line) when is_binary(line) do + cut = byte_size(mask(line)) + + if cut < byte_size(line), + do: binary_part(line, cut + 1, byte_size(line) - cut - 1), + else: "" + end + + @doc "The line up to (not including) its comment `#`." + @spec head(String.t()) :: String.t() + def head(line) when is_binary(line), do: binary_part(line, 0, byte_size(mask(line))) + + # `prefix` is the masked line up to the site's action. + defp structural?(prefix) do + cond do + Regex.match?(~r/^\s*(?:-\s+)?(?:\{\s*)?["']?uses["']?\s*:\s*["']?$/, prefix) -> + true + + Regex.match?(~r/[{,]\s*["']?uses["']?\s*:\s*["']?$/, prefix) -> + prefix |> flow_stack() |> structural_flow_stack?() + + true -> + false + end + end + + # The brackets still open in `text`, innermost first, each with what opens + # it: the key before it (`steps: [`), :none (nothing but indentation or a + # list dash), or :text (anything else — `echo {` is a script, not YAML). + defp flow_stack(text) do + text + |> String.graphemes() + |> Enum.reduce({[], ""}, fn + c, {stack, seg} when c in ["{", "["] -> {[{c, opener_key(seg)} | stack], ""} + c, {stack, _seg} when c in ["}", "]"] -> {Enum.drop(stack, 1), ""} + ",", {stack, _seg} -> {stack, ""} + c, {stack, seg} -> {stack, seg <> c} + end) + |> elem(0) + end + + defp opener_key(seg) do + cond do + Regex.match?(~r/^\s*(?:-\s*)?$/, seg) -> :none + m = Regex.run(~r/^\s*(?:-\s+)?["']?([\w.-]+)["']?\s*:\s*$/, seg) -> Enum.at(m, 1) + true -> :text + end + end + + defp structural_flow_stack?(stack) do + Enum.all?(stack, fn {_, key} -> key != :text end) and + case stack do + [{"{", :none}] -> true + [{"{", :none}, {"[", key} | _] when key in ["steps", :none] -> true + [{"{", job}, {"{", "jobs"} | _] when is_binary(job) -> true + _ -> false + end + end + + defp do_mask("", out), do: out + + defp do_mask(<>, out) when q in [?", ?'] do + {body, close, tail} = take_quoted(rest, q, "") + + kept = + if Regex.match?(~r/^\s*:/, tail) or Regex.match?(@uses_tail_re, out), + do: body, + else: String.duplicate("x", byte_size(body)) + + do_mask(tail, out <> <> <> kept <> close) + end + + defp do_mask(<>, out) do + if out == "" or String.ends_with?(out, [" ", "\t"]), + do: out, + else: do_mask(rest, out <> "#") + end + + defp do_mask(<>, out), do: do_mask(rest, out <> <>) + defp do_mask(<>, out), do: do_mask(rest, out <> <>) + + # The body of a quoted scalar up to its closing quote; a backslash escapes + # the next character inside double quotes. An unterminated scalar runs to + # the end of the line. + defp take_quoted("", _q, acc), do: {acc, "", ""} + + defp take_quoted(<>, ?", acc), + do: take_quoted(rest, ?", acc <> <>) + + defp take_quoted(<>, q, acc) when c == q, do: {acc, <>, rest} + + defp take_quoted(<>, q, acc), + do: take_quoted(rest, q, acc <> <>) + + defp take_quoted(<>, q, acc), do: take_quoted(rest, q, acc <> <>) +end diff --git a/lib/rules/pr_automerge.ex b/lib/rules/pr_automerge.ex index 590f66dd..d9860183 100644 --- a/lib/rules/pr_automerge.ex +++ b/lib/rules/pr_automerge.ex @@ -61,6 +61,8 @@ defmodule Hypatia.Rules.PrAutomerge do dependabot to re-raise it. """ + alias Hypatia.Rules.PinSites + @doc """ Classify a PR record against the policy. @@ -125,8 +127,8 @@ defmodule Hypatia.Rules.PrAutomerge do files |> Enum.flat_map(fn file -> file - |> added_lines() - |> Enum.flat_map(fn line -> + |> added_entries() + |> Enum.flat_map(fn {line, certain?} -> # Every site on the line: a flow sequence can put several steps on # one changed line, and a pin judged by its first site alone lets a # denylisted second one through to `arm_auto`. The action is @@ -143,7 +145,10 @@ defmodule Hypatia.Rules.PrAutomerge do file: filename_of(file), line: String.trim(line), denylist_id: Map.get(entry, "id"), - severity: Map.get(entry, "severity", "critical") + severity: Map.get(entry, "severity", "critical"), + # false when the hunk shows the line among script text: it may + # be a fixture inside a `run: |` whose header is outside the hunk. + certain: certain? } end) end) @@ -188,7 +193,11 @@ defmodule Hypatia.Rules.PrAutomerge do # A changed line inside a block scalar (`run: |`) is script text, never a # pin line, so a script edit is not a pin-only change. - lines != [] and Enum.all?(lines, fn {line, scalar?} -> not scalar? and pin_line?(line) end) + # A line the hunk shows among script text is not known to be a pin line. + lines != [] and + Enum.all?(lines, fn {line, scalar?, certain?} -> + certain? and not scalar? and pin_line?(line) + end) end # ─── PA002/PA003/PA004 · version deltas ─────────────────────────────── @@ -197,7 +206,8 @@ defmodule Hypatia.Rules.PrAutomerge do Version deltas for every action this PR re-pins. Each removed pin is paired with the first added pin for the same action - in the same file. Unpaired pins and unchanged refs are omitted. Returns a + in the same file that no earlier removal has claimed. Unpaired pins and + unchanged refs are omitted. Returns a flat list of maps with `:action`, `:file`, `:from`, `:to`, `:status`, `:source` and `:major?`. @@ -222,44 +232,54 @@ defmodule Hypatia.Rules.PrAutomerge do removed = file |> removed_lines() |> Enum.flat_map(&parse_pins/1) added = file |> added_lines() |> Enum.flat_map(&parse_pins/1) + # Each added site pairs with ONE removed site of the same action, in + # order: a line that keeps one `actions/checkout` at v4 and moves a + # second from v4 to v7 carries the major bump, which pairing every + # removal with the first addition would hide. removed - |> Enum.flat_map(fn old -> - case Enum.find(added, &(String.downcase(&1.action) == String.downcase(old.action))) do + |> Enum.flat_map_reduce(added, fn old, pool -> + case Enum.find_index(pool, &(String.downcase(&1.action) == String.downcase(old.action))) do nil -> - [] - - new -> - if new.ref == old.ref do - [] - else - from = resolve(resolution, old.action, old.ref) - to = resolve(resolution, new.action, new.ref) - claim = claim_for(claims, old.action) - - cond do - from != nil and to != nil and claim != nil and - (claim.from != from or claim.to != to) -> - delta(old, new, from, to, :conflict, "tags") - - from != nil and to != nil -> - delta(old, new, from, to, :ok, "tags") - - claim != nil -> - delta(old, new, claim.from, claim.to, :ok, "body-claim") - - true -> - delta(old, new, nil, nil, :unresolved, "none") - end - |> Map.put(:file, filename_of(file)) - # flat_map needs a list: a bare map would be flattened into - # its {key, value} pairs. - |> List.wrap() - end + {[], pool} + + i -> + {Enum.at(pool, i) |> pin_delta(old, file, resolution, claims), + List.delete_at(pool, i)} end end) + |> elem(0) end) end + defp pin_delta(new, old, file, resolution, claims) do + if new.ref == old.ref do + [] + else + from = resolve(resolution, old.action, old.ref) + to = resolve(resolution, new.action, new.ref) + claim = claim_for(claims, old.action) + + cond do + from != nil and to != nil and claim != nil and + (claim.from != from or claim.to != to) -> + delta(old, new, from, to, :conflict, "tags") + + from != nil and to != nil -> + delta(old, new, from, to, :ok, "tags") + + claim != nil -> + delta(old, new, claim.from, claim.to, :ok, "body-claim") + + true -> + delta(old, new, nil, nil, :unresolved, "none") + end + |> Map.put(:file, filename_of(file)) + # flat_map_reduce needs a list: a bare map would be flattened into its + # {key, value} pairs. + |> List.wrap() + end + end + defp delta(old, _new, from, to, status, source) do %{ action: old.action, @@ -334,6 +354,12 @@ defmodule Hypatia.Rules.PrAutomerge do not dependency_bot?(author, policy) -> reject(base, :flag, "not_a_dependency_bot", "P3") + Enum.any?(scan.poison_sites, &(not &1.certain)) -> + # The hunk shows a denylisted pin among script text: it may be a + # fixture inside a `run: |` whose header is outside the hunk. Closing + # the PR or excising the line would both act on a guess. + reject(base, :flag, "denylisted_pin_context_unknown", "P1") + scan.poison_sites != [] and scan.major_delta -> reject(base, :close_poison_and_majors, "introduces_denylisted_pin_and_major_bumps", "P1") @@ -485,207 +511,126 @@ defmodule Hypatia.Rules.PrAutomerge do end end - # One `uses:` site anywhere on a line, bounded on the left so `reuses:` is - # not one. Matched against the MASKED line (see mask_scalars/1). - @pin_site_re ~r/(?:^|[^A-Za-z0-9_.\/-])["']?uses["']?\s*:\s*["']?(?[A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@(?[^\s#"',}\]]+)/ - @doc """ - Every structural `uses:` site on a line, in order, as `%{action, base, ref}`. + Every structural `uses:` site on a diff line, in order, as + `%{action, base, ref}`. A flow sequence may put several steps on one line (`+ { uses: a/b@x }, { uses: c/d@y },`); `parse_pin/1` reads only the - first. Only a site the YAML structure makes a pin counts: - - * the line's own leading `uses:` key (`uses:`, `- uses:`, `- { uses:`); - * an entry of a flow mapping that is a step or a job — an anonymous - `{ ... }` at the line's level or in a `[` that `steps:` (or nothing on - the line) opens, or a mapping a job key opens inside `jobs: { ... }`. - - Text is not a site: a pin inside a quoted `name:` or `run:` scalar, after - `#`, inside an input's mapping (`with: { uses: ... }`), or after free text - (`echo { uses: ... }` in a script). + first. Which sites count is `Hypatia.Rules.PinSites`'s rule, the same one + PI001 reads a file with. """ @spec parse_pins(String.t()) :: [map()] def parse_pins(line) do - masked = line |> String.replace(~r/^[-+]/, "") |> mask_scalars() - - masked - |> pin_sites() - |> Enum.map(fn [_full, {a_at, a_len}, {r_at, r_len}] -> - action = binary_part(masked, a_at, a_len) - %{action: action, base: action_base(action), ref: binary_part(masked, r_at, r_len)} + line + |> String.replace(~r/^[-+]/, "") + |> PinSites.sites() + |> Enum.map(fn %{action: action, ref: ref} -> + %{action: action, base: action_base(action), ref: ref} end) end - defp pin_sites(masked) do - @pin_site_re - |> Regex.scan(masked, return: :index, capture: :all) - |> Enum.filter(fn [_full, {action_at, _} | _] -> - structural_site?(binary_part(masked, 0, action_at)) - end) - end - - defp pin_line?(line) do - masked = line |> String.replace(~r/^[-+]/, "") |> mask_scalars() - sites = pin_sites(masked) - - remaining = - sites - |> Enum.reverse() - |> Enum.reduce(masked, fn [{at, len}, {action_at, _}, _], rest -> - # The site match includes an opening value quote but ends at the ref. - # Remove its matching closing quote along with the pin. - quote = binary_part(masked, action_at - 1, 1) - closing = binary_part(rest, at + len, min(1, byte_size(rest) - at - len)) - len = if quote in ["\"", "'"] and closing == quote, do: len + 1, else: len - binary_part(rest, 0, at) <> binary_part(rest, at + len, byte_size(rest) - at - len) - end) - - sites != [] and Regex.match?(~r/^[\s\-\[\]{},]*$/, remaining) + defp pin_line?(line), do: line |> String.replace(~r/^[-+]/, "") |> PinSites.pin_line?() + + # Every changed line of a patch as `{line, scalar?, certain?}`. + # + # `scalar?`: the line sits inside a block scalar (`run: |`, `script: >-`) + # on its own side of the diff, as far as the hunk shows — the context and + # changed lines of each side are read in order, and a `key: |` line opens + # a scalar that runs while lines are indented deeper than its key. A pin + # written inside a script is text. + # + # `certain?`: the hunk does not show the line among script text. A hunk + # that starts inside a long `run: |` has no header to see, so the line's + # YAML ancestry is unknown; the context around it says which. Every other + # line of its side at its indentation or shallower — its siblings and + # ancestors as far as the hunk shows — must be YAML (a key, a list item, a + # flow bracket, a comment); one that is not (`cat > f <<'EOF'`, `EOF`, + # `make test`) leaves it uncertain. A hunk with no such line gives no + # evidence and leaves it certain. + defp changed_lines(file) do + file + |> patch_of() + |> String.split("\n") + |> Enum.chunk_while( + [], + fn line, hunk -> + if String.starts_with?(line, "@@"), + do: {:cont, Enum.reverse(hunk), []}, + else: {:cont, [line | hunk]} + end, + fn hunk -> {:cont, Enum.reverse(hunk), []} end + ) + |> Enum.flat_map(&hunk_lines/1) end - # `prefix` is the masked line up to the site's action. - defp structural_site?(prefix) do - cond do - Regex.match?(~r/^\s*(?:-\s+)?(?:\{\s*)?["']?uses["']?\s*:\s*["']?$/, prefix) -> - true - - Regex.match?(~r/[{,]\s*["']?uses["']?\s*:\s*["']?$/, prefix) -> - prefix |> flow_stack() |> structural_flow_stack?() + defp hunk_lines(hunk) do + hunk = Enum.reject(hunk, &Regex.match?(~r/^[-+]{3}/, &1)) + old = side_view(hunk, "-") + new = side_view(hunk, "+") - true -> - false - end + hunk + |> Enum.with_index() + |> Enum.flat_map(fn {line, i} -> + cond do + String.starts_with?(line, "+") -> [line_status(line, i, new)] + String.starts_with?(line, "-") -> [line_status(line, i, old)] + true -> [] + end + end) end - # The brackets still open in `text`, innermost first, each with what opens - # it: the key before it (`steps: [`), :none (nothing but indentation or a - # list dash), or :text (anything else — `echo {` is a script, not YAML). - defp flow_stack(text) do - text - |> String.graphemes() - |> Enum.reduce({[], ""}, fn - c, {stack, seg} when c in ["{", "["] -> {[{c, opener_key(seg)} | stack], ""} - c, {stack, _seg} when c in ["}", "]"] -> {Enum.drop(stack, 1), ""} - ",", {stack, _seg} -> {stack, ""} - c, {stack, seg} -> {stack, seg <> c} + # One side of a hunk (context lines plus the changed lines of `mark`), in + # order, as `{hunk_index, text, scalar?}`. + defp side_view(hunk, mark) do + hunk + |> Enum.with_index() + |> Enum.filter(fn {line, _} -> + String.starts_with?(line, mark) or String.starts_with?(line, " ") or line == "" + end) + |> Enum.map_reduce(nil, fn {line, i}, state -> + text = String.slice(line, 1..-1//1) + {inside?, state} = scalar_step(state, text) + {{i, text, inside?}, state} end) |> elem(0) end - defp opener_key(seg) do - cond do - Regex.match?(~r/^\s*(?:-\s*)?$/, seg) -> :none - m = Regex.run(~r/^\s*(?:-\s+)?["']?([\w.-]+)["']?\s*:\s*$/, seg) -> Enum.at(m, 1) - true -> :text - end - end - - defp structural_flow_stack?(stack) do - Enum.all?(stack, fn {_, key} -> key != :text end) and - case stack do - [{"{", :none}] -> true - [{"{", :none}, {"[", key} | _] when key in ["steps", :none] -> true - [{"{", job}, {"{", "jobs"} | _] when is_binary(job) -> true - _ -> false - end - end - - # The line with its comment dropped and every quoted scalar that is - # neither a key nor the value of a `uses` key blanked to `x`. A `uses` - # value and a quoted key are kept as written, so the captures read from - # the masked line are the pin's own text. - @uses_tail_re ~r/(?:^|[-{,\s])["']?uses["']?\s*:\s*$/ + defp line_status(line, i, view) do + {before, [{_, text, scalar?} | rest]} = Enum.split_while(view, fn {j, _, _} -> j != i end) + indent = indent_of(text) - defp mask_scalars(line), do: do_mask(line, "") + certain? = + (before ++ rest) + |> Enum.reject(fn {_, t, inside?} -> inside? or String.trim(t) == "" end) + |> Enum.filter(fn {_, t, _} -> indent_of(t) <= indent end) + |> Enum.all?(fn {_, t, _} -> yaml_like?(t) end) - defp do_mask("", out), do: out - - defp do_mask(<>, out) when q in [?", ?'] do - {body, close, tail} = take_quoted(rest, q, "") - - kept = - if Regex.match?(~r/^\s*:/, tail) or Regex.match?(@uses_tail_re, out), - do: body, - else: String.duplicate("x", String.length(body)) - - do_mask(tail, out <> <> <> kept <> close) - end - - defp do_mask(<>, out) do - if out == "" or String.ends_with?(out, [" ", "\t"]), - do: out, - else: do_mask(rest, out <> "#") + {line, scalar?, certain?} end - defp do_mask(<>, out), do: do_mask(rest, out <> <>) - defp do_mask(<>, out), do: do_mask(rest, out <> <>) - - # The body of a quoted scalar up to its closing quote; a backslash escapes - # the next character inside double quotes. An unterminated scalar runs to - # the end of the line. - defp take_quoted("", _q, acc), do: {acc, "", ""} - - defp take_quoted(<>, ?", acc), - do: take_quoted(rest, ?", acc <> <>) - - defp take_quoted(<>, q, acc) when c == q, do: {acc, <>, rest} + defp indent_of(text), do: String.length(text) - String.length(String.trim_leading(text, " ")) - defp take_quoted(<>, q, acc), - do: take_quoted(rest, q, acc <> <>) + # A line YAML structure could be: a key, a list item, a flow bracket, or + # nothing but a comment. Shell, prose and heredoc markers are not. + defp yaml_like?(text) do + t = text |> PinSites.mask() |> String.trim() - defp take_quoted(<>, q, acc), do: take_quoted(rest, q, acc <> <>) - - # Every changed line of a patch, paired with whether it sits inside a - # block scalar (`run: |`, `script: >-`) on its own side of the diff, as far - # as the hunk shows: the context and changed lines of each side are read in - # order, and a `key: |` line opens a scalar that runs while lines are - # indented deeper than its key. A pin written inside a script is text. - defp changed_lines(file) do - file - |> patch_of() - |> String.split("\n") - |> Enum.reduce({[], nil, nil}, fn line, {acc, old, new} -> - cond do - String.starts_with?(line, "@@") -> - {acc, nil, nil} - - Regex.match?(~r/^[-+]{3}/, line) -> - {acc, old, new} - - String.starts_with?(line, "+") -> - {inside?, new} = scalar_step(new, String.slice(line, 1..-1//1)) - {[{line, inside?} | acc], old, new} - - String.starts_with?(line, "-") -> - {inside?, old} = scalar_step(old, String.slice(line, 1..-1//1)) - {[{line, inside?} | acc], old, new} - - String.starts_with?(line, " ") or line == "" -> - text = String.slice(line, 1..-1//1) - {_, old} = scalar_step(old, text) - {_, new} = scalar_step(new, text) - {acc, old, new} - - true -> - {acc, old, new} - end - end) - |> elem(0) - |> Enum.reverse() + t == "" or Regex.match?(~r/^(?:-\s+)?["']?[\w.\/-][\w.\/ -]*["']?\s*:(?:\s|$)/, t) or + Regex.match?(~r/^-(?:\s|$)/, t) or Regex.match?(~r/^[{}\[\],]/, t) end # One line of one side: is it inside the open block scalar (`state` is the # column of the key that opened it, or nil), and what is open after it? defp scalar_step(state, text) do - indent = String.length(text) - String.length(String.trim_leading(text, " ")) + indent = indent_of(text) inside? = state != nil and (String.trim(text) == "" or indent > state) cond do inside? -> {true, state} - Regex.match?(~r/:\s*[|>][-+0-9]*\s*$/, mask_scalars(text)) -> + Regex.match?(~r/:\s*[|>][-+0-9]*\s*$/, PinSites.mask(text)) -> {false, text |> String.replace(~r/[^\s-].*$/, "") |> String.length()} true -> @@ -693,11 +638,19 @@ defmodule Hypatia.Rules.PrAutomerge do end end - defp added_lines(file), - do: for({line, false} <- changed_lines(file), String.starts_with?(line, "+"), do: line) + defp added_lines(file), do: for({line, _} <- added_entries(file), do: line) + + # Added lines outside block scalars, each with its certainty. + defp added_entries(file), + do: + for( + {line, false, certain?} <- changed_lines(file), + String.starts_with?(line, "+"), + do: {line, certain?} + ) defp removed_lines(file), - do: for({line, false} <- changed_lines(file), String.starts_with?(line, "-"), do: line) + do: for({line, false, _} <- changed_lines(file), String.starts_with?(line, "-"), do: line) defp patch_of(file), do: Map.get(file, :patch) || Map.get(file, "patch") || "" diff --git a/lib/rules/workflow_audit.ex b/lib/rules/workflow_audit.ex index 2fc1165b..d7d700b2 100644 --- a/lib/rules/workflow_audit.ex +++ b/lib/rules/workflow_audit.ex @@ -1669,23 +1669,23 @@ defmodule Hypatia.Rules.WorkflowAudit do end # The entries of the mapping anchored `&name`: the first `&name` in the - # document, read as a flow mapping when a `{` follows it, else as the - # block mapping under its line. + # document outside comments and quoted scalars, read as a flow mapping + # when a `{` follows it, else as the block mapping under its line. A + # `note: "see &name"` before the real anchor is text. defp wf_anchor_entries(lines, name) do re = ~r/&#{Regex.escape(name)}(?=[\s,{}\[\]]|$)/ lines |> Enum.with_index(1) - |> Enum.find(fn {l, _} -> wf026_plain?(l) and Regex.match?(re, wf_strip_comment(l)) end) + |> Enum.find(fn {l, _} -> wf026_plain?(l) and Regex.match?(re, wf_mask_quoted(l)) end) |> case do nil -> :error {line, number} -> - [{at, len}] = Regex.run(re, line, return: :index) - - after_anchor = - line |> binary_part(at + len, byte_size(line) - at - len) |> wf_strip_comment() + masked = wf_mask_quoted(line) + [{at, len}] = Regex.run(re, masked, return: :index) + after_anchor = binary_part(masked, at + len, byte_size(masked) - at - len) case Regex.run(~r/\{/, after_anchor, return: :index) do [{brace, _}] -> @@ -1935,6 +1935,28 @@ defmodule Hypatia.Rules.WorkflowAudit do defp wf_strip_comment([q | rest], q, acc), do: wf_strip_comment(rest, nil, [q | acc]) defp wf_strip_comment([c | rest], q, acc), do: wf_strip_comment(rest, q, [c | acc]) + # The line with its comment dropped and every quoted scalar's body blanked + # to `x` byte for byte, so `&name` or `{` written inside a string is not + # structure, and an offset into the result is an offset into the line. + defp wf_mask_quoted(text), do: wf_mask_quoted(text, nil, "") + + defp wf_mask_quoted(<<>>, _q, out), do: out + + defp wf_mask_quoted(<>, nil, out) + when out == "" or binary_part(out, byte_size(out) - 1, 1) in [" ", "\t"], + do: out + + defp wf_mask_quoted(<>, nil, out) when c in [?", ?'], + do: wf_mask_quoted(rest, c, out <> <>) + + defp wf_mask_quoted(<>, nil, out), do: wf_mask_quoted(rest, nil, out <> <>) + + defp wf_mask_quoted(<>, ?", out), + do: wf_mask_quoted(rest, ?", out <> "xx") + + defp wf_mask_quoted(<>, q, out), do: wf_mask_quoted(rest, nil, out <> <>) + defp wf_mask_quoted(<<_, rest::binary>>, q, out), do: wf_mask_quoted(rest, q, out <> "x") + defp wf_unquote(text) do text |> String.trim() @@ -2094,7 +2116,7 @@ defmodule Hypatia.Rules.WorkflowAudit do # not a call, so the fleet is never sent to rewrite script data. # The 40 hex characters end the ref (`@-patched` is a tag), in # either case: git resolves the commit ID case-insensitively. - reusable = Regex.escape(rev.reusable) + reusable = wf027_reusable_re(rev.reusable) ref_re = ~r/^\s*(?:-\s+)?["']?uses["']?\s*:\s*["']?#{reusable}@([0-9a-fA-F]{40})(?=["'\s#,}\]]|$)/ @@ -2133,6 +2155,18 @@ defmodule Hypatia.Rules.WorkflowAudit do end) end + # The reusable's identity as a regex source. GitHub resolves the owner and + # the repository name in either case (`Hyperpolymath/Standards` is + # `hyperpolymath/standards`); the workflow path inside the repository is a + # file path and is matched as written. The fleet fixer reads the pin + # case-insensitively and rewrites it as the caller spells it. + defp wf027_reusable_re(reusable) do + case String.split(reusable, "/", parts: 3) do + [owner, repo, path] -> "(?i:#{Regex.escape(owner <> "/" <> repo)})/#{Regex.escape(path)}" + _ -> Regex.escape(reusable) + end + end + # Is the `uses` at `uses_at` the entry of a flow mapping a job key opens? # The job's own line under `jobs:` (`rust: { uses: ... }`), the whole # `jobs:` mapping written on one line (`jobs: { rust: { uses: ... } }`), or diff --git a/scripts/sweeps/estate-pin-integrity.sh b/scripts/sweeps/estate-pin-integrity.sh index 357715f7..5fa1f44c 100755 --- a/scripts/sweeps/estate-pin-integrity.sh +++ b/scripts/sweeps/estate-pin-integrity.sh @@ -431,6 +431,7 @@ repairable=0 flagged=0 confirmed=0 unread=0 +: > "$OUT_DIR/.unread-repos" while IFS=$'\t' read -r org repo path; do [ -n "${path:-}" ] || continue @@ -439,6 +440,7 @@ while IFS=$'\t' read -r org repo path; do if [ -z "$content" ]; then log " ! cannot read ${org}/${repo}/${path} (binary, missing, or no permission)" unread=$((unread + 1)) + printf '%s\n' "${org}/${repo}" >> "$OUT_DIR/.unread-repos" continue fi @@ -555,12 +557,34 @@ if [ "$confirmed" -eq 0 ]; then exit 0 fi -# One repair PR per repository. +# One repair PR per repository, and only for a repository the sweep has +# proved whole: every confirmed file repairable and every candidate file +# read. Repairing one workflow while a flagged lock entry or fixture in the +# same repository keeps the denylisted pin (or an unread file may) is a +# partial repair, and arming it would merge a repository that is still +# poisoned. Such a repository is held for owner review with its findings. +held=0 if [ "$repairable" -gt 0 ]; then - jq -src ' - map(select(.status == "repairable")) - | group_by(.repo) + unread_repos=$(jq -Rsc 'split("\n") | map(select(. != "")) | unique' "$OUT_DIR/.unread-repos") + jq -sr --argjson unread "$unread_repos" ' + group_by(.repo)[] + | select(any(.status == "repairable")) + | .[0].repo as $r + | if any($unread[]; . == $r) then "\($r)\tunread candidate file" + elif all(.status == "repairable") | not then + "\($r)\tflagged sibling file(s): \([.[] | select(.status != "repairable") | .path] | join(", "))" + else empty end + ' "$FINDINGS" > "$OUT_DIR/.held" + held=$(grep -c . "$OUT_DIR/.held" || true) + while IFS=$'\t' read -r r why; do + log " held ${r}: repairable file(s) not planned — ${why}" + done < "$OUT_DIR/.held" + + jq -src --argjson unread "$unread_repos" ' + group_by(.repo) | .[] + | select(all(.status == "repairable")) + | select(.[0].repo as $r | any($unread[]; . == $r) | not) | {repo: .[0].repo, files: length, paths: [.[].path], @@ -579,7 +603,9 @@ log " candidate files : ${total_hits}" log " confirmed denylisted : ${confirmed}" log " mechanically repairable : ${repairable}" log " needs owner review : ${flagged}" -log " repos with a repair ready : $(grep -c . "$PLAN" 2>/dev/null || echo 0)" +planned=$(grep -c . "$PLAN" 2>/dev/null || true) +log " repos with a repair ready : ${planned:-0}" +log " repos held (partial repair): ${held}" log " findings : ${FINDINGS}" log " repair plan : ${PLAN}" [ "$REWRITE" -eq 1 ] && log " repaired trees : ${OUT_DIR}/patched" diff --git a/test/rules/pin_integrity_test.exs b/test/rules/pin_integrity_test.exs index b3aa5a81..1aff8f05 100644 --- a/test/rules/pin_integrity_test.exs +++ b/test/rules/pin_integrity_test.exs @@ -295,6 +295,52 @@ defmodule Hypatia.Rules.PinIntegrityTest do assert PI.pi001_denylisted_pin(new_content, @policy) == [] end + test "a compact flow step is a site: caught by PI001 and spliced at its own offset" do + # `- { name: ..., uses: ... }` carries the dash, the brace and a sibling + # key; the old whole-line pattern allowed only one of the first two and + # nothing after the ref, so this pin was invisible to the file scan. + content = """ + jobs: + a: + steps: + - { name: "scan #{@poison}", uses: github/codeql-action/init@#{@poison} } # v4.38.1 + - uses: actions/checkout@v7.0.1 + """ + + assert [%{line: 4, detail: %{denylist_id: "PIN-001"}}] = + PI.pi001_denylisted_pin(content, @policy) + + assert {:ok, new_content, [excision]} = PI.substitute_denylisted_pins(content, @policy) + assert excision.from_ref == @poison + + # Only the site moves: the same 40 hex characters inside `name:` are + # text and stay as written, and the one-site line's label is relabelled. + assert new_content =~ + ~s(- { name: "scan #{@poison}", uses: github/codeql-action/init@#{@good} } # v4.38.0) + + assert PI.pi001_denylisted_pin(new_content, @policy) == [] + end + + test "a multi-site flow line is repaired site by site and its shared label is kept" do + content = + " steps: [ { uses: github/codeql-action/init@#{@poison} }, " <> + "{ uses: actions/checkout@v7.0.1 } ] # v7.0.1\n" + + assert [_] = PI.pi001_denylisted_pin(content, @policy) + assert {:ok, new_content, [_]} = PI.substitute_denylisted_pins(content, @policy) + assert new_content =~ "init@#{@good} }, { uses: actions/checkout@v7.0.1 } ] # v7.0.1" + end + + test "a mixed-case action spelling meets its lower-case denylist entry" do + # GitHub resolves `GitHub/CodeQL-Action` to `github/codeql-action`. + content = " - uses: GitHub/CodeQL-Action/init@#{@poison} # v4.38.1\n" + + assert [%{rule_id: "PI001"}] = PI.pi001_denylisted_pin(content, @policy) + + assert {:ok, new_content, [_]} = PI.substitute_denylisted_pins(content, @policy) + assert new_content == " - uses: GitHub/CodeQL-Action/init@#{@good} # v4.38.0\n" + end + test "a file with no denylisted site is returned untouched" do clean = "- uses: actions/checkout@v7.0.1\n" diff --git a/test/rules/pr_automerge_test.exs b/test/rules/pr_automerge_test.exs index 9cff8c34..8e934bf3 100644 --- a/test/rules/pr_automerge_test.exs +++ b/test/rules/pr_automerge_test.exs @@ -482,6 +482,66 @@ defmodule Hypatia.Rules.PrAutomergeTest do assert [%{action: "actions/checkout", ref: "v4.1.7"}] = PA.parse_pins(line) end + + test "repeated sites of one action pair one-to-one, so the second site's major bump counts" do + # The first checkout stays at v4.1.7 and the second moves to v7.0.1. + # Pairing both removals with the first addition (the unchanged one) + # hid the major bump and armed an excision instead of a close. + patch = """ + @@ -1,1 +1,1 @@ + - steps: [ { uses: actions/checkout@v4.1.7 }, { uses: actions/checkout@v4.1.7 }, { uses: github/codeql-action/init@#{@good} } ] + + steps: [ { uses: actions/checkout@v4.1.7 }, { uses: actions/checkout@v7.0.1 }, { uses: github/codeql-action/init@#{@poison} } ] + """ + + resolution = %{ + {"actions/checkout", "v4.1.7"} => "4.1.7", + {"actions/checkout", "v7.0.1"} => "7.0.1" + } + + f = file(".github/workflows/ci.yml", patch) + + assert [%{action: "actions/checkout", from: "4.1.7", to: "7.0.1", major?: true}] = + f + |> List.wrap() + |> PA.pin_deltas(resolution, []) + |> Enum.filter(&(&1.action == "actions/checkout")) + + decision = PA.classify(pr([f], resolution: resolution), @policy) + assert decision.disposition == :close_poison_and_majors + end + + test "a hunk that opens inside a script, header out of view, flags a poison pin rather than closing" do + # More than three lines below `run: |`, the hunk shows no header: the + # pin may be fixture text. Its neighbours at its own indentation are + # shell, so the context is unknown and the PR goes to an owner. + patch = """ + @@ -40,3 +40,3 @@ + cat > fixture.yml <<'EOF' + - - uses: github/codeql-action/init@#{@good} + + - uses: github/codeql-action/init@#{@poison} + EOF + """ + + f = file(".github/workflows/ci.yml", patch) + assert [%{certain: false}] = PA.poison_sites([f], @policy) + refute PA.pin_lines_only?(f) + + decision = PA.classify(pr([f]), @policy) + assert decision.disposition == :flag + assert decision.blocked_by == "denylisted_pin_context_unknown" + + # Control: the same change among YAML siblings is a live pin. + live = """ + @@ -12,3 +12,3 @@ + - name: Init CodeQL + - uses: github/codeql-action/init@#{@good} + + uses: github/codeql-action/init@#{@poison} + with: + """ + + live_decision = PA.classify(pr([file(".github/workflows/ci.yml", live)]), @policy) + assert live_decision.disposition == :close_poison_only + end end describe "body_claims/1" do diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs index 44886644..46524541 100644 --- a/test/rules/workflow_audit_pins_test.exs +++ b/test/rules/workflow_audit_pins_test.exs @@ -476,6 +476,25 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do assert c.auto_fixable end + test "anchor text inside a quoted scalar or a comment is not the anchor" do + # The quoted `&rust-inputs` comes first; reading children from its line + # found none and reported the aliased step as missing `toolchain`. + wf = """ + env: + NOTE: "see &rust-inputs below" # and &rust-inputs here + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: &rust-inputs + toolchain: stable + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: *rust-inputs + """ + + assert [] == WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + test "an alias with no anchor cannot be read: reported, never passed silently" do wf = """ jobs: @@ -729,6 +748,25 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => wf}) end + test "the owner and repository match in either case; the workflow path as written" do + # GitHub resolves `Hyperpolymath/Standards` to the same repository. + mixed = "Hyperpolymath/Standards/.github/workflows/rust-ci-reusable.yml" + wf = "jobs:\n rust-ci:\n uses: #{mixed}@#{@broken_standards}\n" + + assert [%{rule: "WF027", line: 3, replacement_sha: @good_standards, auto_fixable: true}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => wf}) + + inline = "jobs: { rust: { uses: #{mixed}@#{@broken_standards} } }\n" + + assert [%{rule: "WF027"}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => inline}) + + # A file path inside the repository is case-sensitive: another file. + other = "hyperpolymath/standards/.github/workflows/Rust-CI-Reusable.yml" + wf2 = "jobs:\n rust-ci:\n uses: #{other}@#{@broken_standards}\n" + assert [] == WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => wf2}) + end + test "a job written as a flow mapping is a call: on its own line or inside jobs: { }" do wf = """ jobs: From 7aa7d274ca452ccc0c59c5d7004f525598ce7706 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 03:03:20 +0000 Subject: [PATCH 30/47] fix(pins): excise only a mergeable remainder, pin-only needs the same actions, plans held on partial enumeration Codex round on 20c8657. - PrAutomerge: finding a poisoned pin no longer upgrades a PR that would be flagged without it. What remains after the excision is classified on its own: a file whose every added line carries only denylisted sites drops out, and so does a delta onto a denylisted ref. Only a remainder that arms by itself gives excise_poison_then_merge; otherwise the PR is flagged (introduces_denylisted_pin_alongside_unmergeable_changes, with remainder_blocked_by naming why). A poisoned workflow pin beside a lockfile bump still arms; beside a permissions: edit, a source file or an unresolved bump it does not. - PrAutomerge.pin_lines_only?/1 (MGX-001): the removed and added sites must name the same actions, as many times each, so a diff that adds a flow step or swaps one action for another is not pin-only. - estate-pin-integrity.sh: a ref is case-folded only when it is a 40-hex commit ID, in the denylist check and when the site is located for the rewrite; `@V4.38.1` is not the blocked tag `v4.38.1`. Repositories with a candidate left out by --sample are held like unreadable ones, and a search that failed or came back partial holds every repository: no armed plan rests on a repository the sweep has not proved whole. - docs: the new blocked_by value and the held-plan rule. Tests: two existing expectations encoded the upgrade and now expect the flag; 4 new cases fail on 20c8657, plus a control that excision still arms beside a lockfile bump. Fake-gh sweep: 4 new modes (partialbad, samplesib, tagcase, tagupper) differ from 20c8657 as intended; the 18 earlier modes are byte-identical, under gawk and mawk. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- docs/operations/estate-automerge.adoc | 11 ++- lib/rules/pr_automerge.ex | 97 ++++++++++++++++++++------ scripts/sweeps/estate-pin-integrity.sh | 49 +++++++++---- test/rules/pr_automerge_test.exs | 82 ++++++++++++++++++++-- 4 files changed, 197 insertions(+), 42 deletions(-) diff --git a/docs/operations/estate-automerge.adoc b/docs/operations/estate-automerge.adoc index f9f3d2e3..ee731f27 100644 --- a/docs/operations/estate-automerge.adoc +++ b/docs/operations/estate-automerge.adoc @@ -89,7 +89,10 @@ Every examined PR gets one line in `pr-decisions.jsonl`: | `blocked_by` | `awaiting_required_checks`, `introduces_denylisted_pin`, `introduces_denylisted_pin_and_major_bumps`, - `introduces_denylisted_pin_alongside_wanted_updates`, + `introduces_denylisted_pin_alongside_wanted_updates` (armed only when what + remains after the excision would arm on its own), + `introduces_denylisted_pin_alongside_unmergeable_changes` (it would not; + `remainder_blocked_by` names why), `denylisted_pin_context_unknown` (the hunk shows the pin among script text; brain only — the shell actuator does not yet read hunk context), `major_version_delta`, @@ -136,8 +139,10 @@ substituted file trees ready for a branch and a PR. (`hypatia/pin-integrity-known-good`), and only for a repository whose every confirmed file is repairable and whose every candidate file was read. A repository with a flagged sibling (an `actions.lock` entry, a fixture) or an - unread candidate is held for owner review: repairing the workflow alone - would arm a merge on a repository that still carries the pin. + unread candidate (unreadable, or left out by `--sample`) is held for owner + review: repairing the workflow alone would arm a merge on a repository that + still carries the pin. A search that failed or came back partial may hide a + sibling anywhere, so then every repository is held and no plan is armed. * Exit: `0` clean, `1` error, `2` findings or an `INCOMPLETE` run (failed or partial searches, listings or clones, unread candidates, or the limits `--max-repos` and `--sample` impose), `3` nothing to do. diff --git a/lib/rules/pr_automerge.ex b/lib/rules/pr_automerge.ex index d9860183..c00b8e66 100644 --- a/lib/rules/pr_automerge.ex +++ b/lib/rules/pr_automerge.ex @@ -86,28 +86,59 @@ defmodule Hypatia.Rules.PrAutomerge do resolution = field(pr, :version_resolution, %{}) claims = body_claims(body) - deltas = pin_deltas(files, resolution, claims) - poison = poison_sites(files, policy) - lock_only = files != [] and Enum.all?(files, &lockfile?(filename_of(&1), policy)) - meta_only = files != [] and Enum.all?(files, &metadata_path?(filename_of(&1))) - pin_only = files != [] and Enum.all?(files, &pin_lines_only?/1) - manifest_file? = Enum.any?(files, &dependency_manifest?(filename_of(&1))) - licence_touch? = Enum.any?(files, &licence_path?(filename_of(&1))) + files + |> scan_of(pin_deltas(files, resolution, claims), poison_sites(files, policy), policy) + |> verdict(pr, policy) + end + defp scan_of(files, deltas, poison, policy) do %{ deltas: deltas, poison_sites: poison, major_delta: Enum.any?(deltas, &major_delta?/1), unresolved: Enum.count(deltas, &(&1.status == :unresolved)), conflicts: Enum.count(deltas, &(&1.status == :conflict)), - lock_only: lock_only, - meta_only: meta_only, - pin_only: pin_only, - manifest_file: manifest_file?, - licence_touch: licence_touch? + lock_only: files != [] and Enum.all?(files, &lockfile?(filename_of(&1), policy)), + meta_only: files != [] and Enum.all?(files, &metadata_path?(filename_of(&1))), + pin_only: files != [] and Enum.all?(files, &pin_lines_only?/1), + manifest_file: Enum.any?(files, &dependency_manifest?(filename_of(&1))), + licence_touch: Enum.any?(files, &licence_path?(filename_of(&1))) } - |> verdict(pr, policy) + end + + # The PR as it would stand once its poisoned pins are excised: a file whose + # every added line carries only denylisted sites drops out, and so does a + # delta onto a denylisted ref. Everything else is still there to classify. + defp remainder_scan(scan, files, policy) do + kept = Enum.reject(files, &poison_only_file?(&1, policy)) + + deltas = + Enum.reject( + scan.deltas, + &denylisted_pin?(%{base: action_base(&1.action), ref: &1.to_ref}, policy) + ) + + scan_of(kept, deltas, [], policy) + end + + defp poison_only_file?(file, policy) do + added = added_lines(file) + + pin_lines_only?(file) and added != [] and + Enum.all?(added, fn line -> + pins = parse_pins(line) + pins != [] and Enum.all?(pins, &denylisted_pin?(&1, policy)) + end) + end + + defp denylisted_pin?(%{base: base, ref: ref}, policy) do + policy + |> Map.get("pin_denylist", []) + |> Enum.any?(fn entry -> + String.downcase(base) == String.downcase(Map.get(entry, "action", "")) and + denylisted_ref?(entry, ref) + end) end # ─── PA001 · denylisted pin ─────────────────────────────────────────── @@ -185,7 +216,9 @@ defmodule Hypatia.Rules.PrAutomerge do This is the proof obligation behind the meta-guard exemption `MGX-001`. A workflow edit that is *not* pin-only — a new step, a changed `permissions:` block, a moved `if:` — fails this, which is what keeps the reflexivity guard - from being talked around by a plausible title. + from being talked around by a plausible title. So does a diff of pin lines + that adds or swaps an action: the removed and added sites must name the + same actions, as many times each. """ @spec pin_lines_only?(map()) :: boolean() def pin_lines_only?(file) do @@ -194,10 +227,18 @@ defmodule Hypatia.Rules.PrAutomerge do # A changed line inside a block scalar (`run: |`) is script text, never a # pin line, so a script edit is not a pin-only change. # A line the hunk shows among script text is not known to be a pin line. + # And every pin the file adds replaces a removed pin of the same action: + # a new step (`+ { uses: a/b@main },`) or a swapped action is a change to + # what the workflow runs, not a re-pin of what it already ran. lines != [] and Enum.all?(lines, fn {line, scalar?, certain?} -> certain? and not scalar? and pin_line?(line) - end) + end) and + action_multiset(removed_lines(file)) == action_multiset(added_lines(file)) + end + + defp action_multiset(lines) do + lines |> Enum.flat_map(&parse_pins/1) |> Enum.map(&String.downcase(&1.action)) |> Enum.sort() end # ─── PA002/PA003/PA004 · version deltas ─────────────────────────────── @@ -280,11 +321,12 @@ defmodule Hypatia.Rules.PrAutomerge do end end - defp delta(old, _new, from, to, status, source) do + defp delta(old, new, from, to, status, source) do %{ action: old.action, from: from, to: to, + to_ref: new.ref, status: status, source: source, file: nil, @@ -368,12 +410,23 @@ defmodule Hypatia.Rules.PrAutomerge do reject(base, :close_poison_only, "introduces_denylisted_pin", "P1") scan.poison_sites != [] -> - accept( - base, - :excise_poison_then_merge, - "introduces_denylisted_pin_alongside_wanted_updates", - "P1" - ) + # Finding the poison must not upgrade a PR that would be flagged + # without it: what remains after the excision is classified on its + # own, and only a remainder that would arm by itself is merged. + rest = verdict(remainder_scan(scan, field(pr, :files, []), policy), pr, policy) + + if rest.safety == "arm_auto" do + accept( + base, + :excise_poison_then_merge, + "introduces_denylisted_pin_alongside_wanted_updates", + "P1" + ) + else + base + |> Map.put(:remainder_blocked_by, rest.blocked_by) + |> reject(:flag, "introduces_denylisted_pin_alongside_unmergeable_changes", "P1") + end scan.major_delta -> reject(base, :flag, "major_version_delta", "P2") diff --git a/scripts/sweeps/estate-pin-integrity.sh b/scripts/sweeps/estate-pin-integrity.sh index 5fa1f44c..65d21447 100755 --- a/scripts/sweeps/estate-pin-integrity.sh +++ b/scripts/sweeps/estate-pin-integrity.sh @@ -124,11 +124,16 @@ lc() { printf '%s' "$1" | tr '[:upper:]' '[:lower:]'; } # in either spelling. The action is compared case-folded too: GitHub resolves # `GitHub/CodeQL-Action` and `github/codeql-action` to one repository, so a # mixed-case caller meets the entry its lower-case spelling would. +# Only a 40-hex commit ID is case-folded: git reads an object ID in either +# case, but a ref NAME is case-sensitive, so `@V4.38.1` is a different tag +# from the blocked `v4.38.1` and may point somewhere else. denylisted_pair() { - jq -e --arg a "$(lc "$1")" --arg r "$(lc "$2")" ' - [.pin_denylist[] | select((.action | ascii_downcase) == $a) - | (.blocked_shas // []), (.blocked_versions // []), ((.blocked_versions // []) | map("v" + .))] - | add // [] | map(ascii_downcase) | index($r) != null' "$POLICY" >/dev/null 2>&1 + jq -e --arg a "$(lc "$1")" --arg r "$2" ' + def norm: if test("^[0-9a-fA-F]{40}$") then ascii_downcase else . end; + ($r | norm) as $r + | [.pin_denylist[] | select((.action | ascii_downcase) == $a) + | (.blocked_shas // []), (.blocked_versions // []), ((.blocked_versions // []) | map("v" + .))] + | add // [] | map(norm) | index($r) != null' "$POLICY" >/dev/null 2>&1 } # The action and ref of one site as `grep -o` printed it: the key and the @@ -203,12 +208,17 @@ replace_site() { # a = ENVIRON["PIN_ACTION"]; r = ENVIRON["PIN_REF"]; good = ENVIRON["PIN_GOOD"] tail = "(^|[-{,[:space:]])[\"" sq "]?uses[\"" sq "]?[[:space:]]*:[[:space:]]*[\"" sq "]?$" ends = "]\"" sq " \t#,}" - lm = tolower(masked); needle = tolower(a "@" r); from = 1 - while ((p = index(substr(lm, from), needle)) > 0) { + # The action is found case-folded; the ref only when it is a 40-hex + # commit ID (a tag name is case-sensitive: `@V4.38.1` is not `@v4.38.1`). + lm = tolower(masked); la = tolower(a "@"); from = 1 + hex = (length(r) == 40 && r ~ /^[0-9a-fA-F]+$/) + while ((p = index(substr(lm, from), la)) > 0) { p += from - 1 - nxt = substr(lm, p + length(needle), 1) - if ((nxt == "" || index(ends, nxt) > 0) && substr(lm, 1, p - 1) ~ tail) { - print substr(line, 1, p + length(a)) good substr(line, p + length(needle)) + got = substr(masked, p + length(la), length(r)) + same = hex ? (tolower(got) == tolower(r)) : (got == r) + nxt = substr(lm, p + length(la) + length(r), 1) + if (same && (nxt == "" || index(ends, nxt) > 0) && substr(lm, 1, p - 1) ~ tail) { + print substr(line, 1, p + length(a)) good substr(line, p + length(la) + length(r)) exit } from = p + 1 @@ -392,10 +402,14 @@ fi total_hits=$(wc -l < "$HITS") log "phase 1: ${total_hits} candidate file(s)" +# Repositories with a candidate file the sweep did not read (left out by +# --sample, or unreadable in phase 2): no repair is planned for them. +: > "$OUT_DIR/.unread-repos" sampled=0 if [ "$SAMPLE" -gt 0 ] && [ "$total_hits" -gt "$SAMPLE" ]; then # A bounded run reads the first N candidates only: INCOMPLETE by construction. sampled=$((total_hits - SAMPLE)) + tail -n +"$((SAMPLE + 1))" "$HITS" | cut -f1,2 | tr '\t' '/' >> "$OUT_DIR/.unread-repos" head -n "$SAMPLE" "$HITS" > "$HITS.sample" && mv "$HITS.sample" "$HITS" log " --sample ${SAMPLE}: ${sampled} candidate file(s) left unread; the sweep cannot report clean" fi @@ -431,7 +445,6 @@ repairable=0 flagged=0 confirmed=0 unread=0 -: > "$OUT_DIR/.unread-repos" while IFS=$'\t' read -r org repo path; do [ -n "${path:-}" ] || continue @@ -563,14 +576,23 @@ fi # same repository keeps the denylisted pin (or an unread file may) is a # partial repair, and arming it would merge a repository that is still # poisoned. Such a repository is held for owner review with its findings. +# A search that failed or came back partial may have hidden any sibling in +# any repository, so then no repository is proved whole and none is armed; +# a clone, a listing or --max-repos that missed a repository missed all of +# it, which leaves the repositories that were cloned whole. held=0 +search_partial=false +if [ "$VIA" = "search" ] && { [ "$incomplete" -gt 0 ] || [ "$failed" -gt 0 ]; }; then + search_partial=true +fi if [ "$repairable" -gt 0 ]; then unread_repos=$(jq -Rsc 'split("\n") | map(select(. != "")) | unique' "$OUT_DIR/.unread-repos") - jq -sr --argjson unread "$unread_repos" ' + jq -sr --argjson unread "$unread_repos" --argjson partial "$search_partial" ' group_by(.repo)[] | select(any(.status == "repairable")) | .[0].repo as $r - | if any($unread[]; . == $r) then "\($r)\tunread candidate file" + | if $partial then "\($r)\tphase 1 search failed or was partial, so a sibling file may be unseen" + elif any($unread[]; . == $r) then "\($r)\tcandidate file not read (unreadable, or left out by --sample)" elif all(.status == "repairable") | not then "\($r)\tflagged sibling file(s): \([.[] | select(.status != "repairable") | .path] | join(", "))" else empty end @@ -580,9 +602,10 @@ if [ "$repairable" -gt 0 ]; then log " held ${r}: repairable file(s) not planned — ${why}" done < "$OUT_DIR/.held" - jq -src --argjson unread "$unread_repos" ' + jq -src --argjson unread "$unread_repos" --argjson partial "$search_partial" ' group_by(.repo) | .[] + | select($partial | not) | select(all(.status == "repairable")) | select(.[0].repo as $r | any($unread[]; . == $r) | not) | {repo: .[0].repo, diff --git a/test/rules/pr_automerge_test.exs b/test/rules/pr_automerge_test.exs index 8e934bf3..14b7c39e 100644 --- a/test/rules/pr_automerge_test.exs +++ b/test/rules/pr_automerge_test.exs @@ -184,9 +184,12 @@ defmodule Hypatia.Rules.PrAutomergeTest do decision = PA.classify(pr([file(".github/workflows/codeql.yml", patch)]), @policy) - assert decision.disposition == :excise_poison_then_merge + assert [_] = decision.poison_sites refute decision.pin_only - assert decision.blocked_by == "introduces_denylisted_pin_alongside_wanted_updates" + # The `with:` beside the pin makes the line more than a pin line, so + # what an excision would leave is not known to be mergeable: flagged. + assert decision.disposition == :flag + assert decision.blocked_by == "introduces_denylisted_pin_alongside_unmergeable_changes" end test "a quoted uses: key and a quoted value are still a pin PA001 sees" do @@ -202,7 +205,10 @@ defmodule Hypatia.Rules.PrAutomergeTest do assert decision.blocked_by == "introduces_denylisted_pin" end - test "poison alongside a wanted update is excised, then merged" do + test "poison alongside a change that would be flagged on its own is flagged, not armed" do + # A `permissions:` edit is no pin bump (MGX-001): without the poison this + # PR is not_unambiguously_classifiable, and finding the poison must not + # upgrade it to an armed excise-then-merge. patch = """ @@ -1,4 +1,4 @@ - - uses: github/codeql-action/init@#{@good} # v4.38.0 @@ -214,9 +220,51 @@ defmodule Hypatia.Rules.PrAutomergeTest do decision = PA.classify(pr([file(".github/workflows/codeql.yml", patch)]), @policy) + assert decision.disposition == :flag + assert decision.safety == "flag" + assert decision.blocked_by == "introduces_denylisted_pin_alongside_unmergeable_changes" + assert decision.remainder_blocked_by == "not_unambiguously_classifiable" + refute decision.pin_only + end + + test "poison alongside a wanted lockfile update is excised, then merged" do + # The workflow's only change is the poisoned pin: excised, it drops out, + # and what remains is a lockfile-only bump that arms on its own. + decision = + PA.classify( + pr([ + file(".github/workflows/codeql.yml", codeql_patch(@good, @poison)), + file("mix.lock", "@@ -1,1 +1,1 @@\n- a 1.0.0\n+ a 1.0.1\n") + ]), + @policy + ) + assert decision.disposition == :excise_poison_then_merge + assert decision.safety == "arm_auto" assert decision.blocked_by == "introduces_denylisted_pin_alongside_wanted_updates" - refute decision.pin_only + end + + test "an unresolved bump sharing a line with a blocked pin, beside a source edit, is flagged" do + # Codex's shape: excision leaves the unresolved ordinary bump and the + # source file, neither of which arms on its own. + patch = """ + @@ -1,1 +1,1 @@ + - steps: [ { uses: actions/cache@v4.0.0 }, { uses: github/codeql-action/init@#{@good} } ] + + steps: [ { uses: actions/cache@v4.1.0 }, { uses: github/codeql-action/init@#{@poison} } ] + """ + + decision = + PA.classify( + pr([ + file(".github/workflows/ci.yml", patch), + file("lib/runtime.ex", "@@ -1,1 +1,1 @@\n-x = 1\n+x = 2\n") + ]), + @policy + ) + + assert decision.safety == "flag" + assert decision.blocked_by == "introduces_denylisted_pin_alongside_unmergeable_changes" + assert decision.remainder_blocked_by == "pin_delta_unresolvable" end test "poison plus a major bump closes both reasons" do @@ -471,6 +519,32 @@ defmodule Hypatia.Rules.PrAutomergeTest do assert PA.poison_sites([file(".github/workflows/ci.yml", free_text)], @policy) == [] end + test "a diff of pin lines that adds or swaps an action is not pin-only, and is not armed" do + added = """ + @@ -3,2 +3,3 @@ + steps: [ + + { uses: owner/action@main }, + { uses: actions/checkout@v7.0.1 }, + """ + + swapped = """ + @@ -1,1 +1,1 @@ + - - uses: actions/checkout@v7.0.1 + + - uses: evil/checkout@v7.0.1 + """ + + for patch <- [added, swapped] do + f = file(".github/workflows/ci.yml", patch) + refute PA.pin_lines_only?(f) + assert PA.classify(pr([f]), @policy).safety == "flag" + end + + # Control: the same action moved to another ref is a re-pin. + assert PA.pin_lines_only?( + file(".github/workflows/ci.yml", checkout_patch("v7.0.0", "v7.0.1")) + ) + end + test "a pin inside an input's mapping is data, not a site" do line = "+ with: { uses: github/codeql-action/init@#{@poison} }" assert PA.parse_pins(line) == [] From 82b6ec5a2afae24c3f29f2389f52b2c3c9d738cc Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 03:17:35 +0000 Subject: [PATCH 31/47] fix(rules): WF026/WF027 advisory severity; a uses: inside a multi-line quoted scalar is text Codex round on 7aa7d27. - WF026 and WF027 emit `severity: :low`, not `:medium`. `:medium` is the CLI's default failure threshold, so every `hypatia scan` of an affected repository exited 1, against AGENTIC `must-not-promote-to-blocking` (the rules report and route; a gate that blocks on them is an owner decision). `:low` is the gate-safe level `flawed_regex` already uses; fleet routing reads fix_script/auto_fixable, and severity only feeds a priority score. - A `uses:` line that sits inside a quoted scalar opened earlier in the same step (WF026) or job (WF027) and not yet closed (`- run: "echo start` ... `echo end"`) is string text: no finding, so no dispatch that would rewrite the string. Only that step's or job's lines are scanned, quotes open only where a scalar can start, and block-scalar content is skipped, so a quote misread elsewhere cannot hide a real pin. Tests: 4 assertions fail on 7aa7d27 (two severities, the WF026 and WF027 quoted cases); controls show a closed quote and a quote in `run: |` still leave the bare step reported. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- lib/rules/workflow_audit.ex | 94 +++++++++++++++++++++++-- test/rules/workflow_audit_pins_test.exs | 57 ++++++++++++++- 2 files changed, 144 insertions(+), 7 deletions(-) diff --git a/lib/rules/workflow_audit.ex b/lib/rules/workflow_audit.ex index d7d700b2..a03ab133 100644 --- a/lib/rules/workflow_audit.ex +++ b/lib/rules/workflow_audit.ex @@ -1232,6 +1232,11 @@ defmodule Hypatia.Rules.WorkflowAudit do wf026_key_indent(line) != key_indent -> [] + # A quoted scalar the step opened above (`- run: "echo start`) and has + # not closed by this line makes the `uses:` line part of the string. + wf_in_open_quote?(lines, opener, number) -> + [] + # A list item is a step only under `steps:`: one inside a block # scalar (a fixture kept in `env:`, a `run: |` script) is text. not wf026_under_steps?(lines, opener) -> @@ -1376,7 +1381,11 @@ defmodule Hypatia.Rules.WorkflowAudit do file: filename, line: number, current_sha: sha, - severity: :medium, + # Report and route, never block (AGENTIC `must-not-promote-to-blocking`): + # `:low` (rank 4) sits below the CLI's default `--severity medium` and + # the reusable's high/critical threshold, as `flawed_regex` does. The + # fleet routing reads `fix_script`/`auto_fixable`, not the severity. + severity: :low, auto_fixable: repairable?, fix_script: if(repairable?, do: "fix-rust-toolchain-input.sh", else: nil), reason: @@ -1935,6 +1944,79 @@ defmodule Hypatia.Rules.WorkflowAudit do defp wf_strip_comment([q | rest], q, acc), do: wf_strip_comment(rest, nil, [q | acc]) defp wf_strip_comment([c | rest], q, acc), do: wf_strip_comment(rest, q, [c | acc]) + # Is line `number` (1-based) inside a quoted scalar that a line from `from` + # up to it opened and did not close? A double- or single-quoted value may + # run over several physical lines (`- run: "echo start` … `echo end"`), and + # a `uses:` written on one of them is string text, not a key. The content + # of a block scalar (`run: |`) is skipped: a quote there is script text and + # opens nothing. Only the lines of the one step or job are read, so a quote + # the scan misreads elsewhere in the file cannot hide a real pin. + defp wf_in_open_quote?(lines, from, number) when from < number do + lines + |> Enum.slice((from - 1)..(number - 2)//1) + |> Enum.reduce({nil, nil}, &wf_quote_line/2) + |> elem(0) + |> Kernel.!=(nil) + end + + defp wf_in_open_quote?(_lines, _from, _number), do: false + + # One line of that scan: `{open_quote | nil, block_scalar_key_indent | nil}`. + defp wf_quote_line(line, {nil, block}) when is_integer(block) do + if String.trim(line) == "" or wf026_indent(line) > block, + do: {nil, block}, + else: wf_quote_line(line, {nil, nil}) + end + + defp wf_quote_line(line, {q, _}) do + case wf_quote_scan(line, q, "") do + {nil, outside} -> + if Regex.match?(~r/:\s*[|>][-+0-9]*\s*$/, outside), + do: {nil, wf026_key_indent(line)}, + else: {nil, nil} + + {open, _} -> + {open, nil} + end + end + + # Walk one line from quote state `q` (nil outside a quoted scalar), keeping + # the text outside quotes. A quote opens a scalar only where a scalar can + # start (after `: `, a list dash, `[`, `{` or `,`, or at the line's start); + # `echo it's` is a plain scalar. A comment ends the line. + defp wf_quote_scan(<<>>, q, out), do: {q, out} + defp wf_quote_scan(<>, ?", out), do: wf_quote_scan(rest, ?", out) + defp wf_quote_scan(<>, ?', out), do: wf_quote_scan(rest, ?', out) + + defp wf_quote_scan(<>, q, out) when c == q, + do: wf_quote_scan(rest, nil, out <> "x") + + defp wf_quote_scan(<<_, rest::binary>>, q, out) when q != nil, do: wf_quote_scan(rest, q, out) + + defp wf_quote_scan(<>, nil, out) + when out == "" or binary_part(out, byte_size(out) - 1, 1) in [" ", "\t"], + do: {nil, out} + + defp wf_quote_scan(<>, nil, out) when c in [?", ?'] do + if Regex.match?(~r/(?:^\s*(?:-\s+)*|:\s+|[\[{,]\s*)$/, out), + do: wf_quote_scan(rest, c, out <> "x"), + else: wf_quote_scan(rest, nil, out <> <>) + end + + defp wf_quote_scan(<>, nil, out), do: wf_quote_scan(rest, nil, out <> <>) + + # The nearest line above `number` that is shallower than it: the key whose + # value holds it (a job's key line for a job-level `uses:`). + defp wf_parent_line(lines, number) do + indent = wf026_indent(Enum.at(lines, number - 1)) + + (number - 1)..1//-1 + |> Enum.find(number, fn n -> + l = Enum.at(lines, n - 1) + wf026_plain?(l) and wf026_indent(l) < indent + end) + end + # The line with its comment dropped and every quoted scalar's body blanked # to `x` byte for byte, so `&name` or `{` written inside a string is not # structure, and an offset into the result is an offset into the line. @@ -2130,9 +2212,10 @@ defmodule Hypatia.Rules.WorkflowAudit do shas = case Regex.run(ref_re, line) do [_full, sha] -> - if match?([_job, "jobs" | _], wf_ancestor_keys(lines, number)), - do: [sha], - else: [] + if match?([_job, "jobs" | _], wf_ancestor_keys(lines, number)) and + not wf_in_open_quote?(lines, wf_parent_line(lines, number), number), + do: [sha], + else: [] nil -> inline_re @@ -2211,7 +2294,8 @@ defmodule Hypatia.Rules.WorkflowAudit do reusable: rev.reusable, current_sha: sha, replacement_sha: rev.replacement, - severity: :medium, + # Advisory, as WF026: reported and routed, never a blocking finding. + severity: :low, auto_fixable: true, fix_script: "fix-stale-reusable-pin.sh", reason: diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs index 46524541..a8eb1ec5 100644 --- a/test/rules/workflow_audit_pins_test.exs +++ b/test/rules/workflow_audit_pins_test.exs @@ -25,7 +25,9 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do [f] = WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) assert f.rule == "WF026" - assert f.severity == :medium + # Advisory: below the CLI's default `--severity medium`, so a scan + # reports and routes it without failing (AGENTIC must-not-promote-to-blocking). + assert f.severity == :low assert f.file == "rust.yml" assert f.line == 5 assert f.current_sha == @toolchain_sha @@ -476,6 +478,45 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do assert c.auto_fixable end + test "a uses: line inside a quoted scalar spread over lines is string text" do + # The `run:` value opens a double-quoted scalar that runs to `echo end"`; + # the `uses:` line between is part of the string, not the step's key. + quoted = """ + jobs: + check: + steps: + - run: "echo start + uses: dtolnay/rust-toolchain@#{@toolchain_sha} + echo end" + """ + + assert [] == WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"q.yml" => quoted}) + + # Controls: a quote closed on its own line, and a quote inside a + # `run: |` script, open nothing — the bare step still fires. + closed = """ + jobs: + check: + steps: + - name: "it's the toolchain" + uses: dtolnay/rust-toolchain@#{@toolchain_sha} + """ + + script = """ + jobs: + check: + steps: + - run: | + echo "unterminated + uses: dtolnay/rust-toolchain@#{@toolchain_sha} + """ + + for wf <- [closed, script] do + assert [%{rule: "WF026"}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"c.yml" => wf}) + end + end + test "anchor text inside a quoted scalar or a comment is not the anchor" do # The quoted `&rust-inputs` comes first; reading children from its line # found none and reported the aliased step as missing `toolchain`. @@ -646,7 +687,7 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do wf = "jobs:\n rust-ci:\n uses: #{@rust_ci}@#{@broken_standards}\n" [f] = WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => wf}) assert f.rule == "WF027" - assert f.severity == :medium + assert f.severity == :low assert f.reusable == @rust_ci assert f.current_sha == @broken_standards assert f.replacement_sha == @good_standards @@ -748,6 +789,18 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => wf}) end + test "a job-level uses: inside a quoted scalar spread over lines is string text" do + wf = """ + jobs: + rust-ci: + name: "first line + uses: #{@rust_ci}@#{@broken_standards} + last line" + """ + + assert [] == WorkflowAudit.check_known_broken_reusable_revision(%{"x.yml" => wf}) + end + test "the owner and repository match in either case; the workflow path as written" do # GitHub resolves `Hyperpolymath/Standards` to the same repository. mixed = "Hyperpolymath/Standards/.github/workflows/rust-ci-reusable.yml" From ec7f2c0a54f2d50f9af2e694018495097c3f601f Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 03:50:55 +0000 Subject: [PATCH 32/47] fix(pins): advisory CLI channel for WF026/WF027; scalar state carried across lines in PI001 and the sweep Codex round on 82b6ec5. - CLI: a finding may carry `advisory: true` (WF026 and WF027 now do; the workflow_audit normalizer keeps the key). `Hypatia.CLI.report_sets/2` splits a scan into what is shown and what decides exit 1: an advisory finding never blocks at any threshold, and `--advisory` / `HYPATIA_ADVISORY` reports it below the threshold. Before this the only way to see a `:low` WF026/WF027 was to lower `--severity`, which also made it, and every other low finding, fail the scan. Opt-in rather than always on: standards' governance baseline gate rejects every unbaselined finding it is shown (threshold `info` over a default scan), so default output is unchanged. A stderr line counts advisory findings reported. - PinSites.mask_lines/1: a file read line by line with the bytes of a scalar opened on an earlier line blanked to `x` (block-scalar content; a multi-line quoted scalar's continuation through its closing quote). A quote opens one only where YAML lets a scalar start; a quote still open at end of file was misread and hides nothing. PinIntegrity.pin_sites reads through it, so PI001-PI005 and substitute_denylisted_pins/3 no longer report or rewrite `uses:` text inside `run: |` or a continued `run: "...`. - estate-pin-integrity.sh: the mask carries quoted-scalar state from record to record with the same opener rules, so a continuation line is no site and its denylisted SHA is flagged (denylisted_token_outside_uses_key), never planned arm_auto. Tests: 9 CLI/WF assertions and 6 PinIntegrity cases fail on 82b6ec5; controls (apostrophe in a plain scalar, quote inside run: |, quote open at EOF, KYAML flow step) pass on both. Sweep: new fake-gh mode `multiquote` plans arm_auto on 82b6ec5 and flags now; control `aposctl` arms on both; the 22 existing modes are byte-identical to 82b6ec5 under gawk and mawk. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- README.adoc | 2 +- docs/wiki-pages/Operations.md | 3 +- lib/hypatia/cli.ex | 101 +++++++++++----- lib/rules/pin_integrity.ex | 7 +- lib/rules/pin_sites.ex | 149 ++++++++++++++++++++++++ lib/rules/workflow_audit.ex | 16 ++- scripts/sweeps/estate-pin-integrity.sh | 80 +++++++++++-- test/cli_test.exs | 74 ++++++++++++ test/rules/pin_integrity_test.exs | 109 +++++++++++++++++ test/rules/workflow_audit_pins_test.exs | 2 + 10 files changed, 499 insertions(+), 44 deletions(-) diff --git a/README.adoc b/README.adoc index 4e3d76d4..0feac51a 100644 --- a/README.adoc +++ b/README.adoc @@ -104,7 +104,7 @@ The escript exposes five commands: [source,bash] ---- -hypatia scan [--format json|text|github|sarif] [--exit-zero] +hypatia scan [--format json|text|github|sarif] [--exit-zero] [--advisory] hypatia report hypatia pr-eligibility --owner --repo --pr hypatia version diff --git a/docs/wiki-pages/Operations.md b/docs/wiki-pages/Operations.md index 8de53245..a0af81c6 100644 --- a/docs/wiki-pages/Operations.md +++ b/docs/wiki-pages/Operations.md @@ -62,6 +62,7 @@ The supervision tree starts Bandit on port 9090 (9099 under `MIX_ENV=test`): | `HYPATIA_ALERT_WEBHOOK_URL` | Webhook for watcher alerts | | `HYPATIA_ALERT_LOG_FILE` | File sink for watcher alerts | | `HYPATIA_EXIT_ZERO` | Make the CLI exit 0 even when findings are present | +| `HYPATIA_ADVISORY` | Also report advisory findings (WF026, WF027) below the severity threshold; they never set the exit code | ## CLI @@ -69,7 +70,7 @@ The escript supports exactly five commands — `scan`, `report`, `pr-eligibility `help`. Anything else exits with "unknown command". ```bash -hypatia scan [--format json|text|github|sarif] [--exit-zero] +hypatia scan [--format json|text|github|sarif] [--exit-zero] [--advisory] hypatia report hypatia pr-eligibility --owner --repo --pr ``` diff --git a/lib/hypatia/cli.ex b/lib/hypatia/cli.ex index cd937d31..64bceb15 100644 --- a/lib/hypatia/cli.ex +++ b/lib/hypatia/cli.ex @@ -33,11 +33,14 @@ defmodule Hypatia.CLI do --format Output format: json (default), text, github, sarif --severity Minimum severity to report: critical, high, medium (default), low, info --path Path to scan (alternative to positional argument) + --advisory Also report advisory findings below the severity threshold + (they never count toward the exit code) ## Environment Variables HYPATIA_FORMAT Override --format HYPATIA_SEVERITY Override --severity + HYPATIA_ADVISORY Truthy: behaves as --advisory """ @version "2.0.0" @@ -90,6 +93,8 @@ defmodule Hypatia.CLI do * `0` -- success (no findings at/above the severity threshold), or `--exit-zero` / `HYPATIA_EXIT_ZERO` was set and the scan ran cleanly. * `1` -- findings exist at/above the threshold (default behaviour). + An advisory finding (`advisory: true`) never counts: see + `report_sets/2`. * `2` -- error (bad arguments, scan failed, etc.). Findings are always written to stdout in the requested format. A @@ -107,6 +112,7 @@ defmodule Hypatia.CLI do help: :boolean, version: :boolean, exit_zero: :boolean, + advisory: :boolean, # pr-eligibility flags owner: :string, repo: :string, @@ -126,13 +132,15 @@ defmodule Hypatia.CLI do format = opts[:format] || System.get_env("HYPATIA_FORMAT") || "json" severity = opts[:severity] || System.get_env("HYPATIA_SEVERITY") || "medium" exit_zero = opts[:exit_zero] || env_flag?("HYPATIA_EXIT_ZERO") + advisory = opts[:advisory] || env_flag?("HYPATIA_ADVISORY") config = %{ format: format, severity: severity, rules: parse_rules(opts[:rules]), path: opts[:path], - exit_zero: exit_zero + exit_zero: exit_zero, + advisory: advisory } case args do @@ -191,26 +199,52 @@ defmodule Hypatia.CLI do severity_floor: config.severity ) - # Filter by severity threshold - filtered = - findings - |> Enum.filter(fn f -> - sev = Map.get(f, :severity, "medium") |> to_string() - rank = Map.get(@severity_order, sev, 5) - threshold = Map.get(@severity_order, config.severity, 3) - rank <= threshold - end) + {shown, blocking} = report_sets(findings, config) - output(filtered, config.format) - emit_finding_summary(filtered, config) + output(shown, config.format) + emit_finding_summary(blocking, config, length(shown) - length(blocking)) cond do - length(filtered) == 0 -> :ok + blocking == [] -> :ok config.exit_zero -> :ok true -> System.halt(1) end end + @doc """ + Splits a scan's findings into `{shown, blocking}`. + + `shown` is what is written to stdout: every finding at or above the + `config.severity` threshold, plus, when `config.advisory` is set + (`--advisory` / `HYPATIA_ADVISORY`), every advisory finding below it. + `blocking` decides exit 1: the shown findings at or above the threshold + that are not advisory. + + An advisory finding carries `advisory: true`. It names a repair a fleet + bot can route (WF026, WF027), and its rule must never fail a gate + (AGENTIC `must-not-promote-to-blocking`). Before this split the only way + to see one was to lower `--severity` to its level, which also made it, and + everything else at that level, fail the scan. Advisory output is opt-in + rather than always on because a gate that rejects every unbaselined + finding it is shown (standards' governance baseline gate runs at threshold + `info` over a default scan) would otherwise start failing on them. + """ + @spec report_sets([map()], map()) :: {[map()], [map()]} + def report_sets(findings, config) do + threshold = Map.get(@severity_order, config.severity, 3) + advisory? = Map.get(config, :advisory, false) + + within? = fn f -> + sev = Map.get(f, :severity, "medium") |> to_string() + Map.get(@severity_order, sev, 5) <= threshold + end + + shown = Enum.filter(findings, &(within?.(&1) or (advisory? and advisory_finding?(&1)))) + {shown, Enum.filter(shown, &(within?.(&1) and not advisory_finding?(&1)))} + end + + defp advisory_finding?(f), do: Map.get(f, :advisory) == true + # ─── Report command ─────────────────────────────────────────────────── defp run_report(path, config) do @@ -222,22 +256,14 @@ defmodule Hypatia.CLI do end findings = collect_findings(abs_path, config.rules) - - filtered = - findings - |> Enum.filter(fn f -> - sev = Map.get(f, :severity, "medium") |> to_string() - rank = Map.get(@severity_order, sev, 5) - threshold = Map.get(@severity_order, config.severity, 3) - rank <= threshold - end) + {shown, blocking} = report_sets(findings, config) # Report always uses text format with extra detail - output_report(filtered, abs_path) - emit_finding_summary(filtered, config) + output_report(shown, abs_path) + emit_finding_summary(blocking, config, length(shown) - length(blocking)) cond do - length(filtered) == 0 -> :ok + blocking == [] -> :ok config.exit_zero -> :ok true -> System.halt(1) end @@ -266,7 +292,7 @@ defmodule Hypatia.CLI do # signal that includes severity counts and the exit code about to be # returned. See hypatia#213. - defp emit_finding_summary(filtered, config) do + defp emit_finding_summary(filtered, config, advisory_count) do counts = Enum.reduce(filtered, %{}, fn f, acc -> sev = Map.get(f, :severity, "medium") |> to_string() @@ -294,6 +320,17 @@ defmodule Hypatia.CLI do if(config.exit_zero and total > 0, do: " (--exit-zero suppressed exit 1)", else: "") ) + # Advisory findings are written to stdout but counted neither above nor + # toward the exit code (report_sets/2); their own line keeps that line's + # shape. + if advisory_count > 0 do + IO.puts( + :stderr, + "[hypatia] advisory: #{advisory_count} finding(s) reported, " <> + "not counted toward the exit code" + ) + end + # A separate line, so the summary above keeps its shape for any caller # that pattern-matches it. Levels are derived (Hypatia.Verdict), never # stored by a rule. @@ -499,7 +536,8 @@ defmodule Hypatia.CLI do :fix_script, :current_sha, :replacement_sha, - :reusable + :reusable, + :advisory ]) ) end) @@ -1559,11 +1597,16 @@ defmodule Hypatia.CLI do --exit-zero Always exit 0 after a successful scan, even when findings exist. Use in CI when a downstream step gates on severity counts. (See HYPATIA_EXIT_ZERO.) + --advisory Also report advisory findings (advisory: true, + e.g. WF026/WF027) below the --severity threshold. + They never count toward exit 1. + (See HYPATIA_ADVISORY.) EXIT CODES: 0 No findings at or above the configured severity threshold, OR --exit-zero / HYPATIA_EXIT_ZERO was set and the scan ran cleanly. - 1 Findings exist at/above the threshold (default behaviour). + 1 Non-advisory findings exist at/above the threshold (default + behaviour). Advisory findings never cause exit 1. 2 Error (bad arguments, scan failed, etc.). ENVIRONMENT: @@ -1571,12 +1614,14 @@ defmodule Hypatia.CLI do HYPATIA_SEVERITY Override --severity HYPATIA_EXIT_ZERO If set to a truthy value (1, true, anything non-empty/non-zero), behaves as --exit-zero. + HYPATIA_ADVISORY If set to a truthy value, behaves as --advisory. EXAMPLES: hypatia scan . hypatia scan ~/repos/my-project --format text hypatia scan . --rules root_hygiene,code_safety --severity high hypatia scan . --exit-zero # CI: emit findings, never fail step + hypatia scan . --advisory # also report routable advisory findings hypatia report . > report.txt HYPATIA_FORMAT=json hypatia scan . """) diff --git a/lib/rules/pin_integrity.ex b/lib/rules/pin_integrity.ex index 24827be4..ac1490cf 100644 --- a/lib/rules/pin_integrity.ex +++ b/lib/rules/pin_integrity.ex @@ -63,7 +63,10 @@ defmodule Hypatia.Rules.PinIntegrity do step of a compact flow sequence or mapping (`steps: [{ uses: a/b@x }]`, `- { name: x, uses: a/b@x }`) — so the file scan sees the pins the diff rules (PA001-PA004) see. A `uses:` inside a quoted `run:` string, a - comment or script text is not one. + comment or script text is not one, nor is one on a line of a `run: |` + block or of a quoted scalar continued from an earlier line: the file is + read through `PinSites.mask_lines/1`, which carries that state from line + to line. Returns a list of maps with `:line`, `:action` (the bare action path as written, e.g. `github/codeql-action/init`), `:action_base` (the first @@ -75,7 +78,7 @@ defmodule Hypatia.Rules.PinIntegrity do @spec pin_sites(String.t()) :: [map()] def pin_sites(content) when is_binary(content) do content - |> String.split("\n") + |> PinSites.mask_lines() |> Enum.with_index(1) |> Enum.flat_map(fn {line, number} -> line_sites(line, number) end) end diff --git a/lib/rules/pin_sites.ex b/lib/rules/pin_sites.ex index 2044c171..9043e2f1 100644 --- a/lib/rules/pin_sites.ex +++ b/lib/rules/pin_sites.ex @@ -120,6 +120,155 @@ defmodule Hypatia.Rules.PinSites do @spec head(String.t()) :: String.t() def head(line) when is_binary(line), do: binary_part(line, 0, byte_size(mask(line))) + @doc """ + The lines of a whole file, each with the bytes that belong to a scalar + opened on an earlier line blanked to `x`, byte for byte: every content + line of a block scalar (`run: |`), and a multi-line quoted scalar's + continuation up to and including its closing quote. A line read through + `sites/1` one at a time cannot see either, so a `uses:` written in a + `run: |` script reads as a key; read from these lines it does not. + + A quote opens a scalar only where YAML lets one start: after `key: `, a + list dash, or, inside a flow collection, `[`, `{` or `,`; at the start of + a line only when the line above left a value pending (`key:`). `echo + it's` stays a plain scalar. A quoted scalar still open at the end of the + file was misread, and the lines after its opening quote are returned as + written, each judged on its own as before: a misread quote never hides a + pin. + """ + @spec mask_lines(String.t()) :: [String.t()] + def mask_lines(content) when is_binary(content) do + lines = String.split(content, "\n") + + {masked, st} = + lines + |> Enum.with_index() + |> Enum.map_reduce(%{q: nil, opened: nil, block: nil, depth: 0, pending: false}, fn + {line, n}, st -> mask_file_line(line, n, st) + end) + + if st.q, + do: Enum.take(masked, st.opened + 1) ++ Enum.drop(lines, st.opened + 1), + else: masked + end + + # Inside a block scalar: a blank line or one deeper than the key is its + # content; anything else ends it. + defp mask_file_line(line, n, %{block: col} = st) when is_integer(col) do + if String.trim(line) == "" or indent_of(line) > col, + do: {String.duplicate("x", byte_size(line)), st}, + else: mask_file_line(line, n, %{st | block: nil}) + end + + defp mask_file_line(line, n, st) do + {cont, st} = + case st.q && close_at(line, st.q, 0) do + nil -> {0, st} + :open -> {byte_size(line), st} + at -> {at + 1, %{st | q: nil, opened: nil}} + end + + rest = binary_part(line, cont, byte_size(line) - cont) + view = String.duplicate("x", cont) <> rest + st = if st.q, do: st, else: walk(rest, String.duplicate("x", cont), n, st) + {view, st} + end + + # The byte offset of the quote that closes a `q` scalar in `line` from + # `at`, or :open. `\` escapes inside double quotes; `''` is a literal `'`. + defp close_at(line, _q, at) when at >= byte_size(line), do: :open + + defp close_at(line, ?", at) do + case :binary.at(line, at) do + ?\\ -> close_at(line, ?", at + 2) + ?" -> at + _ -> close_at(line, ?", at + 1) + end + end + + defp close_at(line, ?', at) do + cond do + :binary.at(line, at) != ?' -> close_at(line, ?', at + 1) + at + 1 < byte_size(line) and :binary.at(line, at + 1) == ?' -> close_at(line, ?', at + 2) + true -> at + end + end + + # One line outside any scalar, `seen` being what precedes `rest` on it + # (quote bodies as `x`). Tracks flow depth, opens a quote that does not + # close on the line, and at its end a block scalar or a pending value. + defp walk(<<>>, seen, _n, st) do + trimmed = String.trim_trailing(seen) + + cond do + String.trim(seen) == "" -> + st + + st.depth == 0 and + Regex.match?(~r/(?:^\s*(?:-\s+)*|:\s+)(?:[&!]\S*\s+)*[|>][-+0-9]*$/, trimmed) -> + %{st | block: block_parent(seen), pending: false} + + true -> + pending = + Regex.match?(~r/(?::|^\s*-)$/, trimmed) or + (st.depth > 0 and Regex.match?(~r/[\[{,]$/, trimmed)) + + %{st | pending: pending} + end + end + + defp walk(<>, seen, n, st) + when seen == "" or binary_part(seen, byte_size(seen) - 1, 1) in [" ", "\t"], + do: walk(<<>>, seen, n, st) + + defp walk(<>, seen, n, st) when q in [?", ?'] do + if scalar_start?(seen, st) do + case close_at(tail, q, 0) do + :open -> + %{st | q: q, opened: n} + + at -> + rest = binary_part(tail, at + 1, byte_size(tail) - at - 1) + walk(rest, seen <> String.duplicate("x", at + 2), n, st) + end + else + walk(tail, seen <> <>, n, st) + end + end + + defp walk(<>, seen, n, st) when c in [?[, ?{] do + st = if st.depth > 0 or value_start?(seen), do: %{st | depth: st.depth + 1}, else: st + walk(tail, seen <> <>, n, st) + end + + defp walk(<>, seen, n, %{depth: d} = st) when c in [?], ?}] and d > 0, + do: walk(tail, seen <> <>, n, %{st | depth: d - 1}) + + defp walk(<>, seen, n, st), do: walk(tail, seen <> <>, n, st) + + # Where a value starts: after `key: ` or a list dash, past any anchor or + # tag (`key: &a [`, `- !!str "`). + defp value_start?(seen), do: Regex.match?(~r/(?:^\s*(?:-\s+)*|:\s+)(?:[&!]\S*\s+)*$/, seen) + + defp scalar_start?(seen, st) do + cond do + Regex.match?(~r/^\s*$/, seen) -> st.pending or st.depth > 0 + Regex.match?(~r/(?:^\s*(?:-\s+)+|:\s+)(?:[&!]\S*\s+)*$/, seen) -> true + st.depth > 0 -> Regex.match?(~r/[\[{,]\s*$/, seen) + true -> false + end + end + + # A block scalar's content is deeper than its key (`run: |`, `- run: |`) + # or, for a bare sequence entry (`- |`), deeper than the dash. + defp block_parent(seen) do + if Regex.match?(~r/^\s*-\s+(?:[&!]\S*\s+)*[|>]/, seen), + do: indent_of(seen), + else: byte_size(hd(Regex.run(~r/^\s*(?:-\s+)*/, seen))) + end + + defp indent_of(line), do: byte_size(line) - byte_size(String.trim_leading(line, " ")) + # `prefix` is the masked line up to the site's action. defp structural?(prefix) do cond do diff --git a/lib/rules/workflow_audit.ex b/lib/rules/workflow_audit.ex index a03ab133..8b84545a 100644 --- a/lib/rules/workflow_audit.ex +++ b/lib/rules/workflow_audit.ex @@ -1155,8 +1155,11 @@ defmodule Hypatia.Rules.WorkflowAudit do not an action input and does not rescue the step. * The step is bounded by indentation, so a later step's `toolchain:` cannot rescue an earlier step. - * Severity is medium: the rule reports and routes to the fleet; it is - never a blocking gate (AGENTIC `must-not-promote-to-blocking`). + * Severity is low and the finding is `advisory: true`: the rule reports + and routes to the fleet; it is never a blocking gate (AGENTIC + `must-not-promote-to-blocking`). The CLI never counts it toward exit 1 + and reports it below the threshold under `--advisory` + (`Hypatia.CLI.report_sets/2`). * A KYAML workflow (standards YAML-POLICY Y-3, the output of `yq -o kyaml`) is judged as its block equivalent, line for line (`ResearchExtensions.flow_to_block/1`), so a quoted, braced step is @@ -1385,7 +1388,10 @@ defmodule Hypatia.Rules.WorkflowAudit do # `:low` (rank 4) sits below the CLI's default `--severity medium` and # the reusable's high/critical threshold, as `flawed_regex` does. The # fleet routing reads `fix_script`/`auto_fixable`, not the severity. + # `advisory: true` keeps it out of the CLI's exit code at any + # threshold and lets `--advisory` report it below one. severity: :low, + advisory: true, auto_fixable: repairable?, fix_script: if(repairable?, do: "fix-rust-toolchain-input.sh", else: nil), reason: @@ -2182,8 +2188,9 @@ defmodule Hypatia.Rules.WorkflowAudit do Auto-fixable: gitbot-fleet `scripts/fix-stale-reusable-pin.sh` rewrites the pin to the replacement the table names, never to "newest". - Severity is medium: like WF026 the rule reports and routes to the fleet; - it is never a blocking gate (AGENTIC `must-not-promote-to-blocking`). + Severity is low and the finding is `advisory: true`: like WF026 the rule + reports and routes to the fleet; it is never a blocking gate (AGENTIC + `must-not-promote-to-blocking`). """ def check_known_broken_reusable_revision(workflow_contents) do Enum.flat_map(workflow_contents, fn {filename, content} -> @@ -2296,6 +2303,7 @@ defmodule Hypatia.Rules.WorkflowAudit do replacement_sha: rev.replacement, # Advisory, as WF026: reported and routed, never a blocking finding. severity: :low, + advisory: true, auto_fixable: true, fix_script: "fix-stale-reusable-pin.sh", reason: diff --git a/scripts/sweeps/estate-pin-integrity.sh b/scripts/sweeps/estate-pin-integrity.sh index 65d21447..600008d2 100755 --- a/scripts/sweeps/estate-pin-integrity.sh +++ b/scripts/sweeps/estate-pin-integrity.sh @@ -148,11 +148,19 @@ site_parts() { # per input line: a quoted scalar that is neither a key nor the value of a # `uses` key has its text replaced by `x` (its quotes kept), a comment by # spaces, and the content lines of a block scalar (`run: |`, `script: >-`) -# entirely by `x` after their indentation. Lengths are kept, so a site found -# on a masked line sits at the same column of the real one. A `uses:` written -# inside `- run: "echo uses: a/b@"`, a `run: |` script or a comment is -# therefore no site this sweep rewrites; a denylisted SHA there is still -# seen by line_has_denied_sha on the raw line, and flagged for the owner. +# entirely by `x` after their indentation. A quoted scalar that runs over +# several lines (`- run: "echo start` ... `echo end"`) is carried from record +# to record like a block scalar: each continuation is `x` up to and including +# its closing quote. A quote opens such a scalar only where YAML lets one +# start (as Hypatia.Rules.PinSites.mask_lines/1 reads it), so `echo it's` +# opens nothing. Lengths are kept, so a site found on a masked line sits at +# the same column of the real one. A `uses:` written inside +# `- run: "echo uses: a/b@"`, on a continuation line of a quoted scalar, +# in a `run: |` script or in a comment is therefore no site this sweep +# rewrites; a denylisted SHA there is still seen by line_has_denied_sha on +# the raw line, and flagged for the owner. A quote never closed before the +# end of the file keeps its lines masked: whatever they hold is flagged, +# never repaired. # shellcheck disable=SC2016 # an awk program, not a shell expansion MASK_AWK=' function key_col(s, t) { t = s; sub(/[^[:space:]-].*$/, "", t); return length(t) } @@ -176,14 +184,70 @@ function mask(s, out, n, i, c, j, body) { } return out } -BEGIN { USES_TAIL = "(^|[-{,[:space:]])[\"" sq "]?uses[\"" sq "]?[[:space:]]*:[[:space:]]*$"; in_scalar = 0 } +# The quote closing a q scalar in s from position j, or 0: `\` escapes inside +# double quotes, and `'"''"'` is a literal quote inside single ones. +function close_pos(s, j, q, n, c) { + n = length(s) + while (j <= n) { + c = substr(s, j, 1) + if (q == "\"" && c == "\\") { j += 2; continue } + if (c == q) { if (q == sq && substr(s, j + 1, 1) == sq) { j += 2; continue }; return j } + j++ + } + return 0 +} +# Can a scalar start after seen? After `key: ` or a list dash (past any +# anchor or tag), after `[`, `{` or `,` inside a flow collection, and at the +# start of a line when the line above left a value pending (`key:`). In a +# plain scalar (`echo it'"'"'s`) a quote opens nothing. +function scalar_start(seen) { + if (seen ~ /^[[:space:]]*$/) return PENDING || DEPTH > 0 + if (seen ~ /(^[[:space:]]*(-[[:space:]]+)+|:[[:space:]]+)([&!][^[:space:]]*[[:space:]]+)*$/) return 1 + if (DEPTH > 0) return seen ~ /[{,[][[:space:]]*$/ + return 0 +} +# Walk s from position i outside any quote (seen is what precedes it, quote +# bodies as x): track the flow depth and whether the line leaves a value +# pending, and return the quote a scalar opened on this line and left open, +# or "". +function walk(s, i, seen, n, c, j, t) { + n = length(s) + while (i <= n) { + c = substr(s, i, 1) + if (c == "#" && (seen == "" || seen ~ /[[:space:]]$/)) break + if ((c == "\"" || c == sq) && scalar_start(seen)) { + j = close_pos(s, i + 1, c) + if (j == 0) return c + seen = seen xs(j - i + 1); i = j + 1; continue + } + if (c == "[" || c == "{") { + if (DEPTH > 0 || seen ~ /(^[[:space:]]*(-[[:space:]]+)*|:[[:space:]]+)([&!][^[:space:]]*[[:space:]]+)*$/) DEPTH++ + } else if ((c == "]" || c == "}") && DEPTH > 0) DEPTH-- + seen = seen c; i++ + } + t = seen; sub(/[[:space:]]+$/, "", t) + if (t !~ /^[[:space:]]*$/) PENDING = (t ~ /:$/ || t ~ /^[[:space:]]*-$/ || (DEPTH > 0 && t ~ /[{,[]$/)) + return "" +} +BEGIN { USES_TAIL = "(^|[-{,[:space:]])[\"" sq "]?uses[\"" sq "]?[[:space:]]*:[[:space:]]*$"; in_scalar = 0; inq = ""; DEPTH = 0; PENDING = 0 } { if (in_scalar) { if ($0 ~ /^[[:space:]]*$/ || key_col($0) > scalar_col) { t = $0; sub(/[^[:space:]].*$/, "", t); print t xs(length($0) - length(t)); next } in_scalar = 0 } - m = mask($0); print m - if (m ~ /:[[:space:]]*[|>][-+0-9]*[[:space:]]*$/) { in_scalar = 1; scalar_col = key_col($0) } + # The continuation of a quoted scalar an earlier line opened is text up to + # and including its closing quote; the rest of the line is masked as a line. + if (inq != "") { + j = close_pos($0, 1, inq) + if (j == 0) { print xs(length($0)); next } + m = xs(j) mask(substr($0, j + 1)) + inq = walk($0, j + 1, xs(j)) + } else { + m = mask($0) + inq = walk($0, 1, "") + } + print m + if (inq == "" && m ~ /:[[:space:]]*[|>][-+0-9]*[[:space:]]*$/) { in_scalar = 1; scalar_col = key_col($0) } }' # The masked form of one file, line for line (block scalars tracked). diff --git a/test/cli_test.exs b/test/cli_test.exs index ef1fad84..c95fca3d 100644 --- a/test/cli_test.exs +++ b/test/cli_test.exs @@ -40,6 +40,80 @@ defmodule Hypatia.CLITest do assert help =~ "HYPATIA_EXIT_ZERO" assert help =~ "EXIT CODES" end + + test "documents --advisory and HYPATIA_ADVISORY" do + help = capture_io(fn -> Hypatia.CLI.main(["help"]) end) + + assert help =~ "--advisory" + assert help =~ "HYPATIA_ADVISORY" + end + end + + # WF026/WF027 name a repair the fleet can route and must never fail a gate + # (AGENTIC must-not-promote-to-blocking). At `:low` the default threshold + # hid them from every consumer; lowering `--severity` to see them made them, + # and every other low finding, fail the scan. `report_sets/2` separates + # what is shown from what decides exit 1. + describe "report_sets/2 -- advisory findings are shown on request, never blocking" do + @advisory %{rule: "WF027", severity: "low", advisory: true, fix_script: "x.sh"} + @plain_low %{rule: "flawed_regex", severity: "low"} + @medium %{rule: "unpinned_action", severity: "medium"} + @base %{severity: "medium", exit_zero: false, advisory: false} + + test "default threshold: an advisory low finding is neither shown nor blocking" do + assert {[@medium], [@medium]} = + Hypatia.CLI.report_sets([@advisory, @plain_low, @medium], @base) + end + + test "--advisory shows it below the threshold, in order, without blocking" do + config = %{@base | advisory: true} + + assert {[@advisory, @medium], [@medium]} = + Hypatia.CLI.report_sets([@advisory, @plain_low, @medium], config) + end + + test "an advisory finding alone never decides exit 1" do + assert {[@advisory], []} = + Hypatia.CLI.report_sets([@advisory], %{@base | advisory: true}) + end + + test "at --severity low it is shown, a plain low finding blocks, it does not" do + config = %{@base | severity: "low"} + + assert {[@advisory, @plain_low, @medium], [@plain_low, @medium]} = + Hypatia.CLI.report_sets([@advisory, @plain_low, @medium], config) + end + + test "a config without the :advisory key behaves as before" do + assert {[@medium], [@medium]} = + Hypatia.CLI.report_sets([@advisory, @medium], %{severity: "medium"}) + end + + test "advisory survives the workflow_audit normalizer" do + dir = Path.join(System.tmp_dir!(), "hyp-advisory-#{:erlang.unique_integer([:positive])}") + File.mkdir_p!(Path.join(dir, ".github/workflows")) + on_exit(fn -> File.rm_rf!(dir) end) + + File.write!(Path.join(dir, ".github/workflows/rust-ci.yml"), """ + name: rust-ci + on: push + permissions: {} + jobs: + rust-ci: + uses: hyperpolymath/standards/.github/workflows/rust-ci-reusable.yml@d135b05bfc647d0c0fbfedc7e80f37ea50f49236 + """) + + wf027 = + dir + |> Hypatia.CLI.collect_findings([:workflow_audit]) + |> Enum.filter(&(&1.type == "known_broken_reusable_revision")) + + assert [%{advisory: true, severity: "low", fix_script: "fix-stale-reusable-pin.sh"}] = + wf027 + + assert {[], []} = Hypatia.CLI.report_sets(wf027, %{@base | advisory: false}) + assert {^wf027, []} = Hypatia.CLI.report_sets(wf027, %{@base | advisory: true}) + end end # Regression coverage for the placeholder-message defect. diff --git a/test/rules/pin_integrity_test.exs b/test/rules/pin_integrity_test.exs index 1aff8f05..667ff3ff 100644 --- a/test/rules/pin_integrity_test.exs +++ b/test/rules/pin_integrity_test.exs @@ -80,6 +80,115 @@ defmodule Hypatia.Rules.PinIntegrityTest do end end + # Read one line at a time, a `uses:` written inside a `run: |` script or a + # quoted scalar continued from an earlier line looked like a key: PI001 + # reported script text and substitute_denylisted_pins/3 rewrote it as an + # automatic repair. The file is now read through PinSites.mask_lines/1. + describe "pin_sites/1 across lines — block and quoted scalars are text" do + @real_step " - uses: github/codeql-action/init@#{@poison} # v4.38.0\n" + + test "a uses: line inside a run: | script is no site, the step after it is" do + content = + "jobs:\n b:\n steps:\n - run: |\n" <> + " cat > fixture.yml <<'EOF'\n" <> + " uses: github/codeql-action/init@#{@poison}\n" <> + " - uses: github/codeql-action/analyze@#{@poison} # v4.38.1\n" <> + " EOF\n\n" <> + " shell: bash\n" <> @real_step + + assert [%{line: 11}] = PI.pi001_denylisted_pin(content, @policy) + + assert {:ok, new_content, [_]} = PI.substitute_denylisted_pins(content, @policy) + assert new_content =~ " uses: github/codeql-action/init@#{@poison}\n" + assert new_content =~ "analyze@#{@poison} # v4.38.1\n" + assert new_content =~ " - uses: github/codeql-action/init@#{@good} # v4.38.0\n" + end + + test "a script that is all text leaves the file untouched" do + content = + "steps:\n - name: write\n run: >-\n uses: github/codeql-action/init@#{@poison}\n" + + assert PI.pi001_denylisted_pin(content, @policy) == [] + assert {:ok, ^content, []} = PI.substitute_denylisted_pins(content, @policy) + end + + test "a bare block-scalar sequence entry is text until the dash's level" do + content = + " - args:\n - |\n uses: github/codeql-action/init@#{@poison}\n" <> + @real_step + + assert [%{line: 4}] = PI.pi001_denylisted_pin(content, @policy) + end + + test "the continuation of a multi-line double-quoted run: is text" do + content = + " - run: \"echo start\n" <> + " uses: github/codeql-action/init@#{@poison}\n" <> + " echo \\\"end\\\"\"\n" <> @real_step + + assert [%{line: 4}] = PI.pi001_denylisted_pin(content, @policy) + assert {:ok, new_content, [_]} = PI.substitute_denylisted_pins(content, @policy) + assert new_content =~ " uses: github/codeql-action/init@#{@poison}\n" + end + + test "a single-quoted one, with an escaped quote inside, is text too" do + content = + " - run: 'echo it''s\n" <> + " uses: github/codeql-action/init@#{@poison}\n" <> + " done'\n" <> @real_step + + assert [%{line: 4}] = PI.pi001_denylisted_pin(content, @policy) + end + + test "a value pending from the line above may open the quote" do + content = + " - run:\n \"echo start\n" <> + " uses: github/codeql-action/init@#{@poison}\"\n" <> @real_step + + assert [%{line: 4}] = PI.pi001_denylisted_pin(content, @policy) + end + + test "an apostrophe in a plain scalar opens nothing" do + content = " - run: echo it's here\n" <> @real_step <> " - run: echo don't\n" + + assert [%{line: 2}] = PI.pi001_denylisted_pin(content, @policy) + end + + test "a quote inside a run: | script opens nothing after the script" do + content = " - run: |\n echo \"unterminated\n" <> @real_step + + assert [%{line: 3}] = PI.pi001_denylisted_pin(content, @policy) + end + + test "a quote never closed by the end of the file hides nothing" do + content = " - run: \"echo start\n" <> @real_step + + assert [%{line: 2}] = PI.pi001_denylisted_pin(content, @policy) + end + + test "the file mask keeps every line's bytes where they were" do + content = + " - run: |\n uses: a/b@#{@poison}\n - run: \"x\n y\" # c\n" <> + @real_step + + masked = Hypatia.Rules.PinSites.mask_lines(content) + lines = String.split(content, "\n") + + assert Enum.map(masked, &byte_size/1) == Enum.map(lines, &byte_size/1) + assert Enum.at(masked, 1) == String.duplicate("x", byte_size(Enum.at(lines, 1))) + assert Enum.at(masked, 3) == "xxxx # c" + assert Enum.at(masked, 4) == String.trim_trailing(@real_step, "\n") + end + + test "a KYAML flow step on its own line is still a site" do + content = + "steps: [\n { name: \"x\", run: \"echo 'a\" },\n" <> + " { uses: \"github/codeql-action/init@#{@poison}\" },\n]\n" + + assert [%{line: 3}] = PI.pi001_denylisted_pin(content, @policy) + end + end + describe "action_base/1" do test "keys on the release unit, not the sub-action" do assert PI.action_base("github/codeql-action/init") == "github/codeql-action" diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs index a8eb1ec5..69d9efeb 100644 --- a/test/rules/workflow_audit_pins_test.exs +++ b/test/rules/workflow_audit_pins_test.exs @@ -28,6 +28,7 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do # Advisory: below the CLI's default `--severity medium`, so a scan # reports and routes it without failing (AGENTIC must-not-promote-to-blocking). assert f.severity == :low + assert f.advisory == true assert f.file == "rust.yml" assert f.line == 5 assert f.current_sha == @toolchain_sha @@ -688,6 +689,7 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do [f] = WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => wf}) assert f.rule == "WF027" assert f.severity == :low + assert f.advisory == true assert f.reusable == @rust_ci assert f.current_sha == @broken_standards assert f.replacement_sha == @good_standards From 22cdff05df768095f686f25f6f190a2f1b0ad631 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 04:10:22 +0000 Subject: [PATCH 33/47] fix(pins): diff hunks read through the file's scalar reader; JSON-style flow colons; escaped quotes in the sweep's comment column Codex round on ec7f2c0. - PinSites.line_states/1: mask_lines/1 with each line's standing, :yaml, :text (block content or a quoted scalar's continuation) or :open (after a quote the text never closes). PrAutomerge now reads each side of a hunk through it instead of its own block-scalar-only tracker: a changed line inside a multi-line quoted `run:` is script text (no PA001 site, never pin-only), and a line after an opening quote the hunk never shows closed is not certain, so a fixture-shaped hunk cut off before its closing quote flags denylisted_pin_context_unknown instead of closing as close_poison_only. - Inside a flow collection a scalar also starts directly after `:` (`{"run":"echo ...`), in PinSites and in the sweep's MASK_AWK, so that continuation is text to PI001, substitute_denylisted_pins/3 and the sweep. - estate-pin-integrity.sh comment_column skips `\` escapes inside double quotes, as MASK_AWK does: `name: "C\" # v4.38.1 build"` holds no comment and is no longer relabelled. Tests: three new cases fail on ec7f2c0 (the cut-off and closed quoted hunks, the JSON-style colon, line_states/1). Sweep: new fake-gh modes `flowcolon` (armed on ec7f2c0, flagged now) and `escquote` (name text relabelled on ec7f2c0, kept now); the other 24 modes give identical findings, plans, exit codes and patched trees under gawk and mawk. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- lib/rules/pin_sites.ex | 33 +++++++++--- lib/rules/pr_automerge.ex | 72 +++++++++++--------------- scripts/sweeps/estate-pin-integrity.sh | 12 +++-- test/rules/pin_integrity_test.exs | 21 ++++++++ test/rules/pr_automerge_test.exs | 40 ++++++++++++++ 5 files changed, 124 insertions(+), 54 deletions(-) diff --git a/lib/rules/pin_sites.ex b/lib/rules/pin_sites.ex index 9043e2f1..035031b6 100644 --- a/lib/rules/pin_sites.ex +++ b/lib/rules/pin_sites.ex @@ -137,7 +137,21 @@ defmodule Hypatia.Rules.PinSites do pin. """ @spec mask_lines(String.t()) :: [String.t()] - def mask_lines(content) when is_binary(content) do + def mask_lines(content) when is_binary(content), + do: content |> line_states() |> Enum.map(&elem(&1, 0)) + + @doc """ + The lines of `mask_lines/1`, each with what the reader made of it: + `:yaml` (structure, as written), `:text` (inside a scalar an earlier line + opened: a block scalar's content, or a quoted scalar's continuation — at + least up to its closing quote) or `:open` (after the opening quote of a + quoted scalar the text never closes, returned as written: what it is + cannot be said from these lines alone). A diff hunk, which shows a few + lines of a file, is read this way too: a hunk that cuts a quoted scalar + off leaves its lines `:open`. + """ + @spec line_states(String.t()) :: [{String.t(), :yaml | :text | :open}] + def line_states(content) when is_binary(content) do lines = String.split(content, "\n") {masked, st} = @@ -147,16 +161,19 @@ defmodule Hypatia.Rules.PinSites do {line, n}, st -> mask_file_line(line, n, st) end) - if st.q, - do: Enum.take(masked, st.opened + 1) ++ Enum.drop(lines, st.opened + 1), - else: masked + if st.q do + Enum.take(masked, st.opened + 1) ++ + Enum.map(Enum.drop(lines, st.opened + 1), &{&1, :open}) + else + masked + end end # Inside a block scalar: a blank line or one deeper than the key is its # content; anything else ends it. defp mask_file_line(line, n, %{block: col} = st) when is_integer(col) do if String.trim(line) == "" or indent_of(line) > col, - do: {String.duplicate("x", byte_size(line)), st}, + do: {{String.duplicate("x", byte_size(line)), :text}, st}, else: mask_file_line(line, n, %{st | block: nil}) end @@ -171,7 +188,7 @@ defmodule Hypatia.Rules.PinSites do rest = binary_part(line, cont, byte_size(line) - cont) view = String.duplicate("x", cont) <> rest st = if st.q, do: st, else: walk(rest, String.duplicate("x", cont), n, st) - {view, st} + {{view, if(cont > 0, do: :text, else: :yaml)}, st} end # The byte offset of the quote that closes a `q` scalar in `line` from @@ -250,11 +267,13 @@ defmodule Hypatia.Rules.PinSites do # tag (`key: &a [`, `- !!str "`). defp value_start?(seen), do: Regex.match?(~r/(?:^\s*(?:-\s+)*|:\s+)(?:[&!]\S*\s+)*$/, seen) + # Inside a flow collection a scalar also starts after `[`, `{` or `,`, and + # directly after a `:`, JSON-style (`{"run":"echo ...`). defp scalar_start?(seen, st) do cond do Regex.match?(~r/^\s*$/, seen) -> st.pending or st.depth > 0 Regex.match?(~r/(?:^\s*(?:-\s+)+|:\s+)(?:[&!]\S*\s+)*$/, seen) -> true - st.depth > 0 -> Regex.match?(~r/[\[{,]\s*$/, seen) + st.depth > 0 -> Regex.match?(~r/[\[{,:]\s*$/, seen) true -> false end end diff --git a/lib/rules/pr_automerge.ex b/lib/rules/pr_automerge.ex index c00b8e66..bfe74d8f 100644 --- a/lib/rules/pr_automerge.ex +++ b/lib/rules/pr_automerge.ex @@ -587,13 +587,16 @@ defmodule Hypatia.Rules.PrAutomerge do # Every changed line of a patch as `{line, scalar?, certain?}`. # - # `scalar?`: the line sits inside a block scalar (`run: |`, `script: >-`) - # on its own side of the diff, as far as the hunk shows — the context and - # changed lines of each side are read in order, and a `key: |` line opens - # a scalar that runs while lines are indented deeper than its key. A pin + # `scalar?`: the line sits inside a scalar an earlier line of its own side + # opened, as far as the hunk shows — a block scalar (`run: |`, + # `script: >-`) or a quoted one continued over lines (`- run: "echo` ... + # `done"`). Each side's context and changed lines are read in order by + # `PinSites.line_states/1`, the reader PI001 reads a file with. A pin # written inside a script is text. # - # `certain?`: the hunk does not show the line among script text. A hunk + # `certain?`: the hunk does not show the line among script text. A line + # after an opening quote the hunk never shows closed is not certain: the + # scalar may run on past the hunk. A hunk # that starts inside a long `run: |` has no header to see, so the line's # YAML ancestry is unknown; the context around it says which. Every other # line of its side at its indentation or shallower — its siblings and @@ -634,32 +637,34 @@ defmodule Hypatia.Rules.PrAutomerge do end # One side of a hunk (context lines plus the changed lines of `mark`), in - # order, as `{hunk_index, text, scalar?}`. + # order, as `{hunk_index, text, state}` — `state` as + # `PinSites.line_states/1` reads the side: `:yaml`, `:text` or `:open`. defp side_view(hunk, mark) do - hunk - |> Enum.with_index() - |> Enum.filter(fn {line, _} -> - String.starts_with?(line, mark) or String.starts_with?(line, " ") or line == "" - end) - |> Enum.map_reduce(nil, fn {line, i}, state -> - text = String.slice(line, 1..-1//1) - {inside?, state} = scalar_step(state, text) - {{i, text, inside?}, state} - end) - |> elem(0) + entries = + hunk + |> Enum.with_index() + |> Enum.filter(fn {line, _} -> + String.starts_with?(line, mark) or String.starts_with?(line, " ") or line == "" + end) + |> Enum.map(fn {line, i} -> {i, String.slice(line, 1..-1//1)} end) + + states = entries |> Enum.map_join("\n", &elem(&1, 1)) |> PinSites.line_states() + + Enum.zip_with(entries, states, fn {i, text}, {_, state} -> {i, text, state} end) end defp line_status(line, i, view) do - {before, [{_, text, scalar?} | rest]} = Enum.split_while(view, fn {j, _, _} -> j != i end) + {before, [{_, text, state} | rest]} = Enum.split_while(view, fn {j, _, _} -> j != i end) indent = indent_of(text) certain? = - (before ++ rest) - |> Enum.reject(fn {_, t, inside?} -> inside? or String.trim(t) == "" end) - |> Enum.filter(fn {_, t, _} -> indent_of(t) <= indent end) - |> Enum.all?(fn {_, t, _} -> yaml_like?(t) end) + state != :open and + (before ++ rest) + |> Enum.reject(fn {_, t, s} -> s == :text or String.trim(t) == "" end) + |> Enum.filter(fn {_, t, _} -> indent_of(t) <= indent end) + |> Enum.all?(fn {_, t, _} -> yaml_like?(t) end) - {line, scalar?, certain?} + {line, state == :text, certain?} end defp indent_of(text), do: String.length(text) - String.length(String.trim_leading(text, " ")) @@ -673,27 +678,10 @@ defmodule Hypatia.Rules.PrAutomerge do Regex.match?(~r/^-(?:\s|$)/, t) or Regex.match?(~r/^[{}\[\],]/, t) end - # One line of one side: is it inside the open block scalar (`state` is the - # column of the key that opened it, or nil), and what is open after it? - defp scalar_step(state, text) do - indent = indent_of(text) - inside? = state != nil and (String.trim(text) == "" or indent > state) - - cond do - inside? -> - {true, state} - - Regex.match?(~r/:\s*[|>][-+0-9]*\s*$/, PinSites.mask(text)) -> - {false, text |> String.replace(~r/[^\s-].*$/, "") |> String.length()} - - true -> - {false, nil} - end - end - defp added_lines(file), do: for({line, _} <- added_entries(file), do: line) - # Added lines outside block scalars, each with its certainty. + # Added lines outside block and continued quoted scalars, each with its + # certainty. defp added_entries(file), do: for( diff --git a/scripts/sweeps/estate-pin-integrity.sh b/scripts/sweeps/estate-pin-integrity.sh index 600008d2..0bd298ff 100755 --- a/scripts/sweeps/estate-pin-integrity.sh +++ b/scripts/sweeps/estate-pin-integrity.sh @@ -197,13 +197,14 @@ function close_pos(s, j, q, n, c) { return 0 } # Can a scalar start after seen? After `key: ` or a list dash (past any -# anchor or tag), after `[`, `{` or `,` inside a flow collection, and at the -# start of a line when the line above left a value pending (`key:`). In a -# plain scalar (`echo it'"'"'s`) a quote opens nothing. +# anchor or tag); inside a flow collection after `[`, `{` or `,`, and +# directly after a `:` (JSON-style `{"run":"echo ...`); and at the start of +# a line when the line above left a value pending (`key:`). In a plain +# scalar (`echo it'"'"'s`) a quote opens nothing. function scalar_start(seen) { if (seen ~ /^[[:space:]]*$/) return PENDING || DEPTH > 0 if (seen ~ /(^[[:space:]]*(-[[:space:]]+)+|:[[:space:]]+)([&!][^[:space:]]*[[:space:]]+)*$/) return 1 - if (DEPTH > 0) return seen ~ /[{,[][[:space:]]*$/ + if (DEPTH > 0) return seen ~ /[:{,[][[:space:]]*$/ return 0 } # Walk s from position i outside any quote (seen is what precedes it, quote @@ -327,7 +328,8 @@ comment_column() { if (q == "") { if (c == "\"" || c == sq) q = c else if (c == "#" && (i == 1 || substr($0, i - 1, 1) ~ /[[:space:]]/)) { print i; exit } - } else if (c == q) q = "" + } else if (q == "\"" && c == "\\") i++ + else if (c == q) q = "" } print 0 }' diff --git a/test/rules/pin_integrity_test.exs b/test/rules/pin_integrity_test.exs index 667ff3ff..af7947da 100644 --- a/test/rules/pin_integrity_test.exs +++ b/test/rules/pin_integrity_test.exs @@ -180,6 +180,27 @@ defmodule Hypatia.Rules.PinIntegrityTest do assert Enum.at(masked, 4) == String.trim_trailing(@real_step, "\n") end + test "a JSON-style flow value after a bare colon opens its quote too" do + content = + " - {\"run\":\"echo start\n" <> + " uses: github/codeql-action/init@#{@poison}\n" <> + " echo end\"}\n" <> @real_step + + assert [%{line: 4}] = PI.pi001_denylisted_pin(content, @policy) + assert {:ok, new_content, [_]} = PI.substitute_denylisted_pins(content, @policy) + assert new_content =~ " uses: github/codeql-action/init@#{@poison}\n" + end + + test "line_states/1 says which lines are text and which a cut-off quote leaves open" do + states = + Hypatia.Rules.PinSites.line_states( + "- run: |\n uses: a/b@x\n- uses: c/d@y\n- run: \"open\n uses: e/f@z" + ) + + assert Enum.map(states, &elem(&1, 1)) == [:yaml, :text, :yaml, :yaml, :open] + assert List.last(states) == {" uses: e/f@z", :open} + end + test "a KYAML flow step on its own line is still a site" do content = "steps: [\n { name: \"x\", run: \"echo 'a\" },\n" <> diff --git a/test/rules/pr_automerge_test.exs b/test/rules/pr_automerge_test.exs index 14b7c39e..3f9630ae 100644 --- a/test/rules/pr_automerge_test.exs +++ b/test/rules/pr_automerge_test.exs @@ -616,6 +616,46 @@ defmodule Hypatia.Rules.PrAutomergeTest do live_decision = PA.classify(pr([file(".github/workflows/ci.yml", live)]), @policy) assert live_decision.disposition == :close_poison_only end + + test "a pin inside a multi-line quoted run: is text; one the hunk never closes is uncertain" do + # The quote opens on a context line and closes inside the hunk: every + # line between is string text, never a site. + closed = """ + @@ -1,7 +1,7 @@ + - run: "cat > fixture.yml < fixture.yml < Date: Tue, 20 Oct 2026 04:22:53 +0000 Subject: [PATCH 34/47] fix(pins): a quote after an anchor or tag, or after a JSON-style colon, opens a scalar in WF026 and in flow values Codex round on 22cdff0. - PinSites.scalar_start?/2 and the sweep's MASK_AWK scalar_start accept an anchor or tag between a flow delimiter (or a line start with a value pending) and the quote: `{"run":&cmd "echo start` ... `echo end"}` is a string, so its `uses:` continuation is neither reported by PI001 nor rewritten by substitute_denylisted_pins/3 or the sweep. - WorkflowAudit's quote scanner (WF026/WF027) shares those openers through wf_scalar_start?/1: past an anchor or tag (`- run: &cmd "echo`), and directly after `:` inside a flow collection the line opens (`steps: [{"run":"echo`). WF026 starts the scan at the step's opener or the line's YAML parent, whichever is earlier, so a string opened on the `steps: [` line itself is seen. Tests: the WF026 quoted-scalar test gains three shapes (JSON-style colon, anchored flow value, anchored block value) and a PinIntegrity case covers three anchored/tagged openers; both fail on 22cdff0. Sweep: new fake-gh mode `flowanchor` armed on 22cdff0, flagged now; the other 26 modes are byte-identical under gawk and mawk. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- lib/rules/pin_sites.ex | 7 ++++--- lib/rules/workflow_audit.ex | 24 ++++++++++++++++++++---- scripts/sweeps/estate-pin-integrity.sh | 9 +++++---- test/rules/pin_integrity_test.exs | 16 ++++++++++++++++ test/rules/workflow_audit_pins_test.exs | 13 +++++++++++++ 5 files changed, 58 insertions(+), 11 deletions(-) diff --git a/lib/rules/pin_sites.ex b/lib/rules/pin_sites.ex index 035031b6..9be744ed 100644 --- a/lib/rules/pin_sites.ex +++ b/lib/rules/pin_sites.ex @@ -268,12 +268,13 @@ defmodule Hypatia.Rules.PinSites do defp value_start?(seen), do: Regex.match?(~r/(?:^\s*(?:-\s+)*|:\s+)(?:[&!]\S*\s+)*$/, seen) # Inside a flow collection a scalar also starts after `[`, `{` or `,`, and - # directly after a `:`, JSON-style (`{"run":"echo ...`). + # directly after a `:`, JSON-style (`{"run":"echo ...`). Every start may + # carry an anchor or tag before the quote (`"run": &cmd "echo`). defp scalar_start?(seen, st) do cond do - Regex.match?(~r/^\s*$/, seen) -> st.pending or st.depth > 0 + Regex.match?(~r/^\s*(?:[&!]\S*\s+)*$/, seen) -> st.pending or st.depth > 0 Regex.match?(~r/(?:^\s*(?:-\s+)+|:\s+)(?:[&!]\S*\s+)*$/, seen) -> true - st.depth > 0 -> Regex.match?(~r/[\[{,:]\s*$/, seen) + st.depth > 0 -> Regex.match?(~r/[\[{,:]\s*(?:[&!]\S*\s+)*$/, seen) true -> false end end diff --git a/lib/rules/workflow_audit.ex b/lib/rules/workflow_audit.ex index 8b84545a..02033bea 100644 --- a/lib/rules/workflow_audit.ex +++ b/lib/rules/workflow_audit.ex @@ -1237,7 +1237,10 @@ defmodule Hypatia.Rules.WorkflowAudit do # A quoted scalar the step opened above (`- run: "echo start`) and has # not closed by this line makes the `uses:` line part of the string. - wf_in_open_quote?(lines, opener, number) -> + # The scan starts at the step's opener or, where the string opens on + # the line that holds the step (`steps: [{"run":"echo start`, no list + # item of its own), at the line's YAML parent. + wf_in_open_quote?(lines, min(opener, wf_parent_line(lines, number)), number) -> [] # A list item is a step only under `steps:`: one inside a block @@ -1988,8 +1991,8 @@ defmodule Hypatia.Rules.WorkflowAudit do # Walk one line from quote state `q` (nil outside a quoted scalar), keeping # the text outside quotes. A quote opens a scalar only where a scalar can - # start (after `: `, a list dash, `[`, `{` or `,`, or at the line's start); - # `echo it's` is a plain scalar. A comment ends the line. + # start (wf_scalar_start?/1); `echo it's` is a plain scalar. A comment + # ends the line. defp wf_quote_scan(<<>>, q, out), do: {q, out} defp wf_quote_scan(<>, ?", out), do: wf_quote_scan(rest, ?", out) defp wf_quote_scan(<>, ?', out), do: wf_quote_scan(rest, ?', out) @@ -2004,13 +2007,26 @@ defmodule Hypatia.Rules.WorkflowAudit do do: {nil, out} defp wf_quote_scan(<>, nil, out) when c in [?", ?'] do - if Regex.match?(~r/(?:^\s*(?:-\s+)*|:\s+|[\[{,]\s*)$/, out), + if wf_scalar_start?(out), do: wf_quote_scan(rest, c, out <> "x"), else: wf_quote_scan(rest, nil, out <> <>) end defp wf_quote_scan(<>, nil, out), do: wf_quote_scan(rest, nil, out <> <>) + # Can a scalar start after `out` (the line so far, quote bodies dropped)? + # After `: `, a list dash, `[`, `{` or `,`, or at the line's start; inside + # a flow collection this line opens, also directly after `:` (JSON-style + # `{"run":"echo`); in each case past any anchor or tag (`run: &cmd "echo`), + # as Hypatia.Rules.PinSites reads a file. + defp wf_scalar_start?(out) do + Regex.match?(~r/(?:^\s*(?:-\s+)*|:\s+|[\[{,]\s*)(?:[&!]\S*\s+)*$/, out) or + (wf_flow_open?(out) and Regex.match?(~r/:\s*(?:[&!]\S*\s+)*$/, out)) + end + + defp wf_flow_open?(out), + do: length(Regex.scan(~r/[\[{]/, out)) > length(Regex.scan(~r/[\]}]/, out)) + # The nearest line above `number` that is shallower than it: the key whose # value holds it (a job's key line for a job-level `uses:`). defp wf_parent_line(lines, number) do diff --git a/scripts/sweeps/estate-pin-integrity.sh b/scripts/sweeps/estate-pin-integrity.sh index 0bd298ff..ef9674bc 100755 --- a/scripts/sweeps/estate-pin-integrity.sh +++ b/scripts/sweeps/estate-pin-integrity.sh @@ -199,12 +199,13 @@ function close_pos(s, j, q, n, c) { # Can a scalar start after seen? After `key: ` or a list dash (past any # anchor or tag); inside a flow collection after `[`, `{` or `,`, and # directly after a `:` (JSON-style `{"run":"echo ...`); and at the start of -# a line when the line above left a value pending (`key:`). In a plain -# scalar (`echo it'"'"'s`) a quote opens nothing. +# a line when the line above left a value pending (`key:`); each past any +# anchor or tag (`"run": &cmd "echo`). In a plain scalar (`echo it'"'"'s`) a +# quote opens nothing. function scalar_start(seen) { - if (seen ~ /^[[:space:]]*$/) return PENDING || DEPTH > 0 + if (seen ~ /^[[:space:]]*([&!][^[:space:]]*[[:space:]]+)*$/) return PENDING || DEPTH > 0 if (seen ~ /(^[[:space:]]*(-[[:space:]]+)+|:[[:space:]]+)([&!][^[:space:]]*[[:space:]]+)*$/) return 1 - if (DEPTH > 0) return seen ~ /[:{,[][[:space:]]*$/ + if (DEPTH > 0) return seen ~ /[:{,[][[:space:]]*([&!][^[:space:]]*[[:space:]]+)*$/ return 0 } # Walk s from position i outside any quote (seen is what precedes it, quote diff --git a/test/rules/pin_integrity_test.exs b/test/rules/pin_integrity_test.exs index af7947da..892f0362 100644 --- a/test/rules/pin_integrity_test.exs +++ b/test/rules/pin_integrity_test.exs @@ -191,6 +191,22 @@ defmodule Hypatia.Rules.PinIntegrityTest do assert new_content =~ " uses: github/codeql-action/init@#{@poison}\n" end + test "an anchor or tag before the quote does not hide where the scalar starts" do + for opener <- [ + " - {\"run\": &cmd \"echo start\n", + " - {\"run\":&cmd \"echo start\n", + " - run: !!str \"echo start\n" + ] do + content = + opener <> + " uses: github/codeql-action/init@#{@poison}\n" <> + " echo end\"#{if String.contains?(opener, "{"), do: "}", else: ""}\n" <> + @real_step + + assert [%{line: 4}] = PI.pi001_denylisted_pin(content, @policy) + end + end + test "line_states/1 says which lines are text and which a cut-off quote leaves open" do states = Hypatia.Rules.PinSites.line_states( diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs index 69d9efeb..6060a58f 100644 --- a/test/rules/workflow_audit_pins_test.exs +++ b/test/rules/workflow_audit_pins_test.exs @@ -493,6 +493,19 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do assert [] == WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"q.yml" => quoted}) + # The same string opened JSON-style in a flow step (`{"run":"echo`), or + # past an anchor (`run: &cmd "echo`), is a string all the same. + for text <- [ + "jobs:\n check:\n steps: [{\"run\":\"echo start\n" <> + " uses: dtolnay/rust-toolchain@#{@toolchain_sha}\n echo end\"}]\n", + "jobs:\n check:\n steps: [{\"run\": &cmd \"echo start\n" <> + " uses: dtolnay/rust-toolchain@#{@toolchain_sha}\n echo end\"}]\n", + "jobs:\n check:\n steps:\n - run: &cmd \"echo start\n" <> + " uses: dtolnay/rust-toolchain@#{@toolchain_sha}\n echo end\"\n" + ] do + assert [] == WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"q.yml" => text}) + end + # Controls: a quote closed on its own line, and a quote inside a # `run: |` script, open nothing — the bare step still fires. closed = """ From a023459786d42b19990a2916fbe394cf41c30654 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 04:34:50 +0000 Subject: [PATCH 35/47] fix(rules): WF026/WF027 carry flow depth across lines; merge sequences and tagged anchors in WF026 with: Codex round on a239f53. - The step-local quote scan (WF026 and WF027) carries the depth of open flow collections from line to line, as the shared pin reader does, so a JSON-style `"run":"echo` on a line inside a `{` or `steps: [` opened above is a string, and its `uses:`-looking continuation is not a step. - WF026 resolves a merge sequence `<<: [*base, *rust]` alias by alias, like a single `<<: *x`; a merge value it does not resolve (`<<: { ... }`, a block sequence) leaves the inputs unknown and routes to review (`:merge`), never to the fixer, which would override a merged channel. - A `with:` carrying an anchor after a tag (`!!map &inputs`) or before one is an anchored mapping, routed to review as the fixer refuses those. Tests: three WF026 cases fail on a239f53 (the flow-depth string, the merge sequence and inline merge, the tagged anchor). All workflow_audit, pin, automerge, KYAML and CLI suites pass. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- lib/rules/workflow_audit.ex | 138 ++++++++++++++++-------- test/rules/workflow_audit_pins_test.exs | 54 +++++++++- 2 files changed, 145 insertions(+), 47 deletions(-) diff --git a/lib/rules/workflow_audit.ex b/lib/rules/workflow_audit.ex index 02033bea..c9ecba6e 100644 --- a/lib/rules/workflow_audit.ex +++ b/lib/rules/workflow_audit.ex @@ -1582,7 +1582,7 @@ defmodule Hypatia.Rules.WorkflowAudit do # `with: {` / `with: &inputs {` — a flow mapping, read to its brace. String.contains?(value, "{") -> - form = if String.starts_with?(value, "&"), do: :anchor, else: :flow + form = if wf026_anchored?(value), do: :anchor, else: :flow [{at, _}] = Regex.run(~r/\{/, line, return: :index) case wf_flow_interior([line | rest], 1, at) do @@ -1594,11 +1594,15 @@ defmodule Hypatia.Rules.WorkflowAudit do # an input; the contents of another input's block scalar # (`components: |` carrying `toolchain: stable`) sit deeper and are text. true -> - form = if String.starts_with?(value, "&"), do: :anchor, else: :block + form = if wf026_anchored?(value), do: :anchor, else: :block wf026_entries_verdict(wf_block_children(rest, key_indent), lines, form) end end + # Does a `with:` value carry an anchor? Node properties come in either + # order: `&inputs`, `&inputs !!map`, `!!map &inputs`. + defp wf026_anchored?(value), do: Regex.match?(~r/^(?:!\S*\s+)*&/, value) + # The `with` entry of a flow-style step's own entries. defp wf026_with_entry(entries, lines) do case List.keyfind(entries, "with", 0) do @@ -1614,7 +1618,7 @@ defmodule Hypatia.Rules.WorkflowAudit do wf026_alias_verdict(name, lines, :alias) String.contains?(value, "{") -> - form = if String.starts_with?(value, "&"), do: :anchor, else: :flow + form = if wf026_anchored?(value), do: :anchor, else: :flow [{at, _}] = Regex.run(~r/\{/, value, return: :index) case wf_flow_interior([value], 1, at) do @@ -1629,9 +1633,12 @@ defmodule Hypatia.Rules.WorkflowAudit do end # Does a list of `{key, value}` entries name a toolchain? A `<<: *x` merge - # key brings the anchored mapping's entries in, but an explicit key wins - # over a merged one (YAML merge semantics): `toolchain:` written empty - # beside `<<: *inputs` is the empty value, whatever the anchor names. + # key brings the anchored mapping's entries in, as does each alias of a + # `<<: [*base, *rust]` sequence, but an explicit key wins over a merged one + # (YAML merge semantics): `toolchain:` written empty beside `<<: *inputs` + # is the empty value, whatever the anchor names. A merge this reader does + # not resolve (`<<: { ... }`, a block sequence) leaves the inputs unknown: + # the finding routes to review (`:merge`), never to the fixer. # The key present with no value (`toolchain:`, `null`, `~`) is not an input # the action can read; the fleet fixer fills such a key in place. defp wf026_entries_verdict(entries, lines, form, seen \\ []) do @@ -1646,20 +1653,47 @@ defmodule Hypatia.Rules.WorkflowAudit do true -> entries - |> Enum.filter(fn {k, v} -> - k == "<<" and Regex.match?(~r/^\*[\w.-]+$/, String.trim(v)) - end) + |> Enum.filter(fn {k, _} -> k == "<<" end) |> Enum.find_value({:missing, form}, fn {_, v} -> - name = v |> String.trim() |> String.trim_leading("*") + case wf026_merge_aliases(v) do + {:ok, names} -> + Enum.find_value(names, fn name -> + case wf026_alias_verdict(name, lines, :merge, seen) do + {:missing, _} -> nil + verdict -> verdict + end + end) - case wf026_alias_verdict(name, lines, :merge, seen) do - {:missing, _} -> nil - verdict -> verdict + :error -> + {:missing, :merge} end end) end end + # The aliases a merge key's value names: `*x`, or a flow sequence of + # aliases `[*a, *b]`, in order; :error for anything else. + defp wf026_merge_aliases(value) do + v = String.trim(value) + + cond do + m = Regex.run(~r/^\*([\w.-]+)$/, v) -> + {:ok, [Enum.at(m, 1)]} + + m = Regex.run(~r/^\[\s*(\*[\w.-]+(?:\s*,\s*\*[\w.-]+)*)\s*,?\s*\]$/, v) -> + names = + m + |> Enum.at(1) + |> String.split(",") + |> Enum.map(&(&1 |> String.trim() |> String.trim_leading("*"))) + + {:ok, names} + + true -> + :error + end + end + # YAML's null: nothing, `~`, or `null` in the three spellings the core # schema admits, unquoted. `"null"` is the string null, an input the # action receives; `""` and `''` are the empty string, which it does not. @@ -1963,69 +1997,81 @@ defmodule Hypatia.Rules.WorkflowAudit do defp wf_in_open_quote?(lines, from, number) when from < number do lines |> Enum.slice((from - 1)..(number - 2)//1) - |> Enum.reduce({nil, nil}, &wf_quote_line/2) + |> Enum.reduce({nil, nil, 0}, &wf_quote_line/2) |> elem(0) |> Kernel.!=(nil) end defp wf_in_open_quote?(_lines, _from, _number), do: false - # One line of that scan: `{open_quote | nil, block_scalar_key_indent | nil}`. - defp wf_quote_line(line, {nil, block}) when is_integer(block) do + # One line of that scan: `{open_quote | nil, block_scalar_key_indent | nil, + # flow_depth}`. The depth of the flow collections still open is carried from + # line to line, as the shared pin reader carries it: a `{"run":"echo` whose + # `{` (or `steps: [`) opened on an earlier line is still inside one. + defp wf_quote_line(line, {nil, block, depth}) when is_integer(block) do if String.trim(line) == "" or wf026_indent(line) > block, - do: {nil, block}, - else: wf_quote_line(line, {nil, nil}) + do: {nil, block, depth}, + else: wf_quote_line(line, {nil, nil, depth}) end - defp wf_quote_line(line, {q, _}) do - case wf_quote_scan(line, q, "") do + defp wf_quote_line(line, {q, _, depth}) do + case wf_quote_scan(line, q, "", depth) do {nil, outside} -> + depth = max(depth + wf_bracket_delta(outside), 0) + if Regex.match?(~r/:\s*[|>][-+0-9]*\s*$/, outside), - do: {nil, wf026_key_indent(line)}, - else: {nil, nil} + do: {nil, wf026_key_indent(line), depth}, + else: {nil, nil, depth} - {open, _} -> - {open, nil} + {open, outside} -> + {open, nil, max(depth + wf_bracket_delta(outside), 0)} end end # Walk one line from quote state `q` (nil outside a quoted scalar), keeping - # the text outside quotes. A quote opens a scalar only where a scalar can - # start (wf_scalar_start?/1); `echo it's` is a plain scalar. A comment - # ends the line. - defp wf_quote_scan(<<>>, q, out), do: {q, out} - defp wf_quote_scan(<>, ?", out), do: wf_quote_scan(rest, ?", out) - defp wf_quote_scan(<>, ?', out), do: wf_quote_scan(rest, ?', out) + # the text outside quotes; `depth` is the flow depth the line starts at. A + # quote opens a scalar only where a scalar can start (wf_scalar_start?/2); + # `echo it's` is a plain scalar. A comment ends the line. + defp wf_quote_scan(<<>>, q, out, _depth), do: {q, out} + + defp wf_quote_scan(<>, ?", out, depth), + do: wf_quote_scan(rest, ?", out, depth) + + defp wf_quote_scan(<>, ?', out, depth), + do: wf_quote_scan(rest, ?', out, depth) - defp wf_quote_scan(<>, q, out) when c == q, - do: wf_quote_scan(rest, nil, out <> "x") + defp wf_quote_scan(<>, q, out, depth) when c == q, + do: wf_quote_scan(rest, nil, out <> "x", depth) - defp wf_quote_scan(<<_, rest::binary>>, q, out) when q != nil, do: wf_quote_scan(rest, q, out) + defp wf_quote_scan(<<_, rest::binary>>, q, out, depth) when q != nil, + do: wf_quote_scan(rest, q, out, depth) - defp wf_quote_scan(<>, nil, out) + defp wf_quote_scan(<>, nil, out, _depth) when out == "" or binary_part(out, byte_size(out) - 1, 1) in [" ", "\t"], do: {nil, out} - defp wf_quote_scan(<>, nil, out) when c in [?", ?'] do - if wf_scalar_start?(out), - do: wf_quote_scan(rest, c, out <> "x"), - else: wf_quote_scan(rest, nil, out <> <>) + defp wf_quote_scan(<>, nil, out, depth) when c in [?", ?'] do + if wf_scalar_start?(out, depth), + do: wf_quote_scan(rest, c, out <> "x", depth), + else: wf_quote_scan(rest, nil, out <> <>, depth) end - defp wf_quote_scan(<>, nil, out), do: wf_quote_scan(rest, nil, out <> <>) + defp wf_quote_scan(<>, nil, out, depth), + do: wf_quote_scan(rest, nil, out <> <>, depth) # Can a scalar start after `out` (the line so far, quote bodies dropped)? # After `: `, a list dash, `[`, `{` or `,`, or at the line's start; inside - # a flow collection this line opens, also directly after `:` (JSON-style - # `{"run":"echo`); in each case past any anchor or tag (`run: &cmd "echo`), - # as Hypatia.Rules.PinSites reads a file. - defp wf_scalar_start?(out) do + # a flow collection, opened on this line or one before it (`depth`), also + # directly after `:` (JSON-style `{"run":"echo`); in each case past any + # anchor or tag (`run: &cmd "echo`), as Hypatia.Rules.PinSites reads a file. + defp wf_scalar_start?(out, depth) do Regex.match?(~r/(?:^\s*(?:-\s+)*|:\s+|[\[{,]\s*)(?:[&!]\S*\s+)*$/, out) or - (wf_flow_open?(out) and Regex.match?(~r/:\s*(?:[&!]\S*\s+)*$/, out)) + (depth + wf_bracket_delta(out) > 0 and Regex.match?(~r/:\s*(?:[&!]\S*\s+)*$/, out)) end - defp wf_flow_open?(out), - do: length(Regex.scan(~r/[\[{]/, out)) > length(Regex.scan(~r/[\]}]/, out)) + # Flow brackets opened minus closed in text with its quote bodies dropped. + defp wf_bracket_delta(out), + do: length(Regex.scan(~r/[\[{]/, out)) - length(Regex.scan(~r/[\]}]/, out)) # The nearest line above `number` that is shallower than it: the key whose # value holds it (a job's key line for a job-level `uses:`). diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs index 6060a58f..8abb0205 100644 --- a/test/rules/workflow_audit_pins_test.exs +++ b/test/rules/workflow_audit_pins_test.exs @@ -479,6 +479,54 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do assert c.auto_fixable end + test "a merge sequence is resolved alias by alias; a merge it cannot read routes to review" do + wf = """ + x-base: &base { components: clippy } + x-rust: &rust { toolchain: nightly } + x-wasm: &wasm { targets: wasm32-unknown-unknown } + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + <<: [*base, *rust] + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + <<: [*base, *wasm] + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + <<: { toolchain: nightly } + """ + + # The first step's channel comes from `*rust`: no finding, and nothing + # for the fixer to override with `stable`. + assert [%{line: 10} = none, %{line: 13} = inline] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + + assert none.auto_fixable + refute inline.auto_fixable + end + + test "an anchor written after a tag (`with: !!map &inputs`) is still an anchored mapping" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: !!map &inputs + components: clippy + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: &inputs2 !!map + components: clippy + """ + + assert [%{line: 4} = a, %{line: 7} = b] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + + refute a.auto_fixable + refute b.auto_fixable + end + test "a uses: line inside a quoted scalar spread over lines is string text" do # The `run:` value opens a double-quoted scalar that runs to `echo end"`; # the `uses:` line between is part of the string, not the step's key. @@ -501,7 +549,11 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do "jobs:\n check:\n steps: [{\"run\": &cmd \"echo start\n" <> " uses: dtolnay/rust-toolchain@#{@toolchain_sha}\n echo end\"}]\n", "jobs:\n check:\n steps:\n - run: &cmd \"echo start\n" <> - " uses: dtolnay/rust-toolchain@#{@toolchain_sha}\n echo end\"\n" + " uses: dtolnay/rust-toolchain@#{@toolchain_sha}\n echo end\"\n", + # The flow collections open on earlier lines (`steps: [`, `{`). + "jobs:\n check:\n steps: [\n {\n \"run\":\"echo start\n" <> + " uses: dtolnay/rust-toolchain@#{@toolchain_sha}\n echo end\"\n" <> + " }\n ]\n" ] do assert [] == WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"q.yml" => text}) end From 6e2e1fa143b1bd4f77f729022d3fc47a0fff4fb0 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 04:49:36 +0000 Subject: [PATCH 36/47] fix(pins): continued uses: and toolchain values, merge precedence, anchors outside block scalars; owner-aware search, refreshed clones Codex round on a023459. - PinSites.continued_site/2: a `uses:` key with nothing after it holds the reference YAML continues on the next, deeper line (`- uses:` then ` github/codeql-action/init@`); PinIntegrity.pin_sites/1 adds those sites, so PI001 reports such a denylisted pin and substitute_denylisted_pins/3 repairs it in place. - WF026: a `with:` child with no value on its line takes the lines below it as its value (`toolchain:` then `nightly` is set, not null); in a merge sequence the first mapping that defines `toolchain` decides, an explicit null included (`<<: [*empty, *rust]` is still missing); and an alias resolves to an anchor outside block-scalar content (PinSites.line_states/1), never to `&inputs` written in a `run: |` fixture. - estate-pin-integrity.sh: code search qualifies each owner by its kind, `user:` or `org:` (asked of GitHub; unknown is a failed search), since `org:` on the default personal account matched nothing and read as a clean estate. `--via clone` brings a clone cached by an earlier run to the default branch's current head before reading it; a refresh that fails is an incomplete run. Tests: four Elixir cases fail on a023459 (continued uses:, continued toolchain, null merge precedence, anchor in a run: | fixture). Sweep: new fake-gh mode `ownerkind` found nothing and exited 0 on a023459 and finds the pin now; a fake-git second `--via clone` run over a changed tree was clean on a023459 and finds the pin now; the other 29 modes are byte-identical under gawk and mawk. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- lib/rules/pin_integrity.ex | 45 +++++++++---- lib/rules/pin_sites.ex | 29 +++++++++ lib/rules/workflow_audit.ex | 87 +++++++++++++++++++------ scripts/sweeps/estate-pin-integrity.sh | 36 +++++++++- test/rules/pin_integrity_test.exs | 21 ++++++ test/rules/workflow_audit_pins_test.exs | 48 ++++++++++++++ 6 files changed, 228 insertions(+), 38 deletions(-) diff --git a/lib/rules/pin_integrity.ex b/lib/rules/pin_integrity.ex index ac1490cf..682cb584 100644 --- a/lib/rules/pin_integrity.ex +++ b/lib/rules/pin_integrity.ex @@ -66,7 +66,9 @@ defmodule Hypatia.Rules.PinIntegrity do comment or script text is not one, nor is one on a line of a `run: |` block or of a quoted scalar continued from an earlier line: the file is read through `PinSites.mask_lines/1`, which carries that state from line - to line. + to line. A `uses:` whose reference YAML continues on the next line + (`- uses:` then ` a/b@x`) is a site on that line + (`PinSites.continued_site/2`). Returns a list of maps with `:line`, `:action` (the bare action path as written, e.g. `github/codeql-action/init`), `:action_base` (the first @@ -77,10 +79,23 @@ defmodule Hypatia.Rules.PinIntegrity do """ @spec pin_sites(String.t()) :: [map()] def pin_sites(content) when is_binary(content) do - content - |> PinSites.mask_lines() - |> Enum.with_index(1) + numbered = content |> PinSites.mask_lines() |> Enum.with_index(1) + + # A `uses:` whose reference continues on the next line is a site there. + continued = + numbered + |> Enum.zip(Enum.drop(numbered, 1)) + |> Enum.flat_map(fn {{key, _}, {next, number}} -> + case PinSites.continued_site(key, next) do + nil -> [] + site -> [site_entry(site, number, next |> PinSites.comment() |> String.trim())] + end + end) + + numbered |> Enum.flat_map(fn {line, number} -> line_sites(line, number) end) + |> Kernel.++(continued) + |> Enum.sort_by(&{&1.line, &1.ref_at}) end def pin_sites(_), do: [] @@ -89,16 +104,18 @@ defmodule Hypatia.Rules.PinIntegrity do sites = PinSites.sites(line) comment = if length(sites) == 1, do: line |> PinSites.comment() |> String.trim(), else: "" - for site <- sites do - %{ - line: number, - action: site.action, - action_base: action_base(site.action), - ref: site.ref, - ref_at: site.ref_at, - comment: comment - } - end + for site <- sites, do: site_entry(site, number, comment) + end + + defp site_entry(site, number, comment) do + %{ + line: number, + action: site.action, + action_base: action_base(site.action), + ref: site.ref, + ref_at: site.ref_at, + comment: comment + } end # GitHub resolves `GitHub/CodeQL-Action` and `github/codeql-action` to one diff --git a/lib/rules/pin_sites.ex b/lib/rules/pin_sites.ex index 9be744ed..0a6659b0 100644 --- a/lib/rules/pin_sites.ex +++ b/lib/rules/pin_sites.ex @@ -120,6 +120,35 @@ defmodule Hypatia.Rules.PinSites do @spec head(String.t()) :: String.t() def head(line) when is_binary(line), do: binary_part(line, 0, byte_size(mask(line))) + @doc """ + The site a `uses:` key with nothing after it on its line holds on the next + line, where YAML continues its scalar (`- uses:` then + ` github/codeql-action/init@`), as `sites/1` returns one, its + offsets into `next_line`; nil unless `next_line` sits deeper than the key + and holds nothing but the reference, quoted or plain, and a comment. Both + lines are as `mask_lines/1` returns them. + """ + @spec continued_site(String.t(), String.t()) :: map() | nil + def continued_site(key_line, next_line) when is_binary(key_line) and is_binary(next_line) do + with [_, col] <- Regex.run(~r/^(\s*(?:-\s+)?)["']?uses["']?\s*:\s*$/, head(key_line)), + true <- indent_of(next_line) > byte_size(col), + [_, {a_at, a_len}, {r_at, r_len}] <- + Regex.run( + ~r/^\s*["']?([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)["']?\s*(?:#.*)?$/, + next_line, + return: :index + ) do + %{ + action: binary_part(next_line, a_at, a_len), + ref: binary_part(next_line, r_at, r_len), + action_at: a_at, + ref_at: r_at + } + else + _ -> nil + end + end + @doc """ The lines of a whole file, each with the bytes that belong to a scalar opened on an earlier line blanked to `x`, byte for byte: every content diff --git a/lib/rules/workflow_audit.ex b/lib/rules/workflow_audit.ex index c9ecba6e..71adcec4 100644 --- a/lib/rules/workflow_audit.ex +++ b/lib/rules/workflow_audit.ex @@ -1642,35 +1642,59 @@ defmodule Hypatia.Rules.WorkflowAudit do # The key present with no value (`toolchain:`, `null`, `~`) is not an input # the action can read; the fleet fixer fills such a key in place. defp wf026_entries_verdict(entries, lines, form, seen \\ []) do - explicit = Enum.find(entries, fn {k, _} -> k == "toolchain" end) - - cond do - match?({_, v} when is_binary(v), explicit) and not wf026_null_value?(elem(explicit, 1)) -> - {:toolchain, form} + case wf026_toolchain_in(entries, lines, seen) do + {:set, :own} -> {:toolchain, form} + {:set, :merge} -> {:toolchain, :merge} + :null -> {:missing, form} + :absent -> {:missing, form} + :unknown -> {:missing, :merge} + {:unreadable, _, _} = unreadable -> unreadable + end + end - explicit != nil -> - {:missing, form} + # Where a mapping's `toolchain` comes from, under YAML merge semantics: its + # own key first; else each merged mapping in order, the first that defines + # the key deciding — an explicit `toolchain: null` in `*empty` of + # `<<: [*empty, *rust]` is the input, `*rust`'s value is not. Returns + # {:set, :own | :merge}, :null, :absent, :unknown (a merge this reader + # does not resolve) or {:unreadable, :alias, name}. + defp wf026_toolchain_in(entries, lines, seen) do + case Enum.find(entries, fn {k, _} -> k == "toolchain" end) do + {_, v} when is_binary(v) -> + if wf026_null_value?(v), do: :null, else: {:set, :own} - true -> + _ -> entries |> Enum.filter(fn {k, _} -> k == "<<" end) - |> Enum.find_value({:missing, form}, fn {_, v} -> + |> Enum.find_value(:absent, fn {_, v} -> case wf026_merge_aliases(v) do {:ok, names} -> Enum.find_value(names, fn name -> - case wf026_alias_verdict(name, lines, :merge, seen) do - {:missing, _} -> nil - verdict -> verdict + case wf026_alias_lookup(name, lines, seen) do + :absent -> nil + {:set, _} -> {:set, :merge} + other -> other end end) :error -> - {:missing, :merge} + :unknown end end) end end + defp wf026_alias_lookup(name, lines, seen) do + if name in seen do + {:unreadable, :alias, name} + else + case wf_anchor_entries(lines, name) do + {:ok, entries} -> wf026_toolchain_in(entries, lines, [name | seen]) + :error -> {:unreadable, :alias, name} + end + end + end + # The aliases a merge key's value names: `*x`, or a flow sequence of # aliases `[*a, *b]`, in order; :error for anything else. defp wf026_merge_aliases(value) do @@ -1721,20 +1745,26 @@ defmodule Hypatia.Rules.WorkflowAudit do end # The entries of the mapping anchored `&name`: the first `&name` in the - # document outside comments and quoted scalars, read as a flow mapping + # document outside comments, quoted scalars and block-scalar content (a + # `run: |` fixture writing `fake: &inputs` is text), read as a flow mapping # when a `{` follows it, else as the block mapping under its line. A # `note: "see &name"` before the real anchor is text. defp wf_anchor_entries(lines, name) do re = ~r/&#{Regex.escape(name)}(?=[\s,{}\[\]]|$)/ + states = lines |> Enum.join("\n") |> Hypatia.Rules.PinSites.line_states() + lines + |> Enum.zip(states) |> Enum.with_index(1) - |> Enum.find(fn {l, _} -> wf026_plain?(l) and Regex.match?(re, wf_mask_quoted(l)) end) + |> Enum.find(fn {{l, {_, state}}, _} -> + state != :text and wf026_plain?(l) and Regex.match?(re, wf_mask_quoted(l)) + end) |> case do nil -> :error - {line, number} -> + {{line, _}, number} -> masked = wf_mask_quoted(line) [{at, len}] = Regex.run(re, masked, return: :index) after_anchor = binary_part(masked, at + len, byte_size(masked) - at - len) @@ -1755,7 +1785,8 @@ defmodule Hypatia.Rules.WorkflowAudit do # The direct children of a block mapping whose key sits at `key_indent`, # as `{key, value}` entries: the plain lines deeper than the key, at the # first child's indentation. Deeper lines are nested mappings or scalar - # text, not entries. + # text, not entries; under a child with no value on its own line they are + # that value (`toolchain:` then `nightly` indented is `toolchain: nightly`). defp wf_block_children(rest, key_indent) do children = rest @@ -1773,16 +1804,30 @@ defmodule Hypatia.Rules.WorkflowAudit do child_indent = wf026_indent(first) children - |> Enum.filter(&(wf026_indent(&1) == child_indent)) - |> Enum.flat_map(fn l -> + |> Enum.with_index() + |> Enum.filter(fn {l, _} -> wf026_indent(l) == child_indent end) + |> Enum.flat_map(fn {l, i} -> case Regex.run(~r/^\s*(?:-\s+)?["']?([^"':\s]+)["']?\s*:(.*)$/, l) do - [_, k, v] -> [{k, v |> wf_strip_comment() |> String.trim()}] - nil -> [] + [_, k, v] -> + v = v |> wf_strip_comment() |> String.trim() + [{k, if(v == "", do: wf_continued_value(children, i, child_indent), else: v)}] + + nil -> + [] end end) end end + # The value of the child at `i` continued on the lines below it, deeper + # than its key and up to the next child; "" when there are none. + defp wf_continued_value(children, i, child_indent) do + children + |> Enum.drop(i + 1) + |> Enum.take_while(&(wf026_indent(&1) > child_indent)) + |> Enum.map_join(" ", &(&1 |> wf_strip_comment() |> String.trim())) + end + # ─── flow-mapping reading (quote- and depth-aware) ────────────────── # Byte column of the innermost `{` still open at byte `at` of `line`, read diff --git a/scripts/sweeps/estate-pin-integrity.sh b/scripts/sweeps/estate-pin-integrity.sh index ef9674bc..863f3a96 100755 --- a/scripts/sweeps/estate-pin-integrity.sh +++ b/scripts/sweeps/estate-pin-integrity.sh @@ -87,6 +87,21 @@ list_repos() { --jq '.[] | select(.archived == false and .fork == false and .disabled == false) | .name' 2>/dev/null } +# Code search names an owner by its kind: `org:` for an organisation and +# `user:` for a personal account (the default `hyperpolymath` is one); `org:` +# on a personal account matches nothing at all, which read as a clean org. +# The kind is asked of GitHub; an owner whose kind cannot be read is a failed +# search, never an empty one. +owner_qualifier() { + local kind + kind=$(gh api "users/$1" --jq '.type' 2>/dev/null) || return 1 + case "$kind" in + Organization) printf 'org' ;; + User) printf 'user' ;; + *) return 1 ;; + esac +} + # ─── Policy extraction ─────────────────────────────────────────────────── mapfile -t DENY_SHAS < <(jq -r '.pin_denylist[] | .blocked_shas[]?' "$POLICY") mapfile -t DENY_VERSIONS < <(jq -r '.pin_denylist[] | .blocked_versions[]?' "$POLICY") @@ -402,10 +417,14 @@ failed=0 skipped_repos=0 if [ "$VIA" = "search" ]; then for org in "${ORGS[@]}"; do + if ! qual=$(owner_qualifier "$org"); then + log " ! ${org}: could not tell whether it is a user or an organisation, so it cannot be searched — phase 1 is INCOMPLETE for this org, use --via clone" + failed=$((failed + 1)); continue + fi for token in "${SEARCH_TOKENS[@]}"; do page=1 while :; do - if ! gh api "search/code?q=${token}+org:${org}&per_page=100&page=${page}" \ + if ! gh api "search/code?q=${token}+${qual}:${org}&per_page=100&page=${page}" \ > "$OUT_DIR/.search.json" 2> "$OUT_DIR/.search.err"; then log " ! ${org}: search for '${token}' (page ${page}) failed: $(head -c 200 "$OUT_DIR/.search.err" | tr '\n' ' ') — phase 1 is INCOMPLETE for this org, use --via clone" failed=$((failed + 1)) @@ -457,8 +476,19 @@ else mkdir -p "$WORK/$org" for name in "${reponames[@]}"; do d="$WORK/$org/$name" - [ -d "$d/.git" ] || git clone --depth 1 --quiet "https://github.com/${org}/${name}.git" "$d" 2>/dev/null || { - log " ! clone failed: ${org}/${name} — phase 1 is INCOMPLETE for this org"; failed=$((failed + 1)); continue; } + # A clone kept from an earlier run with the same --out is brought to the + # default branch's current head before it is read: an old checkout + # misses a pin added since, and a candidate never listed is never read. + if [ -d "$d/.git" ]; then + if ! { git -C "$d" fetch --depth 1 --quiet origin HEAD 2>/dev/null \ + && git -C "$d" reset --hard --quiet FETCH_HEAD 2>/dev/null \ + && git -C "$d" clean -fdxq 2>/dev/null; }; then + log " ! refreshing the cached clone of ${org}/${name} failed — phase 1 is INCOMPLETE for this org" + failed=$((failed + 1)); continue + fi + elif ! git clone --depth 1 --quiet "https://github.com/${org}/${name}.git" "$d" 2>/dev/null; then + log " ! clone failed: ${org}/${name} — phase 1 is INCOMPLETE for this org"; failed=$((failed + 1)); continue + fi while IFS= read -r f; do printf '%s\t%s\t%s\n' "$org" "$name" "${f#"$d"/}" >> "$HITS" done < <(grep -rliE "$REF_ALTERNATION" "$d" --include='*.yml' --include='*.yaml' --include='*.lock' --include='*.toml' 2>/dev/null || true) diff --git a/test/rules/pin_integrity_test.exs b/test/rules/pin_integrity_test.exs index 892f0362..d9d33c5a 100644 --- a/test/rules/pin_integrity_test.exs +++ b/test/rules/pin_integrity_test.exs @@ -207,6 +207,27 @@ defmodule Hypatia.Rules.PinIntegrityTest do end end + test "a uses: whose reference continues on the next line is a site there" do + content = + " - uses:\n github/codeql-action/init@#{@poison} # v4.38.0\n" <> + " - uses:\n \"github/codeql-action/analyze@#{@poison}\"\n" <> @real_step + + assert [%{line: 2}, %{line: 4}, %{line: 5}] = PI.pi001_denylisted_pin(content, @policy) + + assert {:ok, new_content, [_, _, _]} = PI.substitute_denylisted_pins(content, @policy) + refute new_content =~ @poison + assert new_content =~ " github/codeql-action/init@#{@good} # v4.38.0\n" + assert new_content =~ " \"github/codeql-action/analyze@#{@good}\"\n" + + # Controls: another key's continued value, and a `uses:` whose next + # line is no deeper, hold no site. + assert PI.pi001_denylisted_pin( + " - run:\n github/codeql-action/init@#{@poison}\n" <> + " - uses:\n - name: github/codeql-action/init@#{@poison}\n", + @policy + ) == [] + end + test "line_states/1 says which lines are text and which a cut-off quote leaves open" do states = Hypatia.Rules.PinSites.line_states( diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs index 8abb0205..d70f57fc 100644 --- a/test/rules/workflow_audit_pins_test.exs +++ b/test/rules/workflow_audit_pins_test.exs @@ -507,6 +507,54 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do refute inline.auto_fixable end + test "in a merge sequence the first mapping that defines toolchain decides, null included" do + wf = """ + x-empty: &empty { toolchain: null } + x-rust: &rust { toolchain: nightly } + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + <<: [*empty, *rust] + """ + + assert [%{line: 6}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "a toolchain value continued on the next line is the value, not null" do + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + toolchain: + nightly + """ + + assert [] == WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "an anchor written inside a run: | script is text, not the anchor an alias names" do + wf = """ + jobs: + check: + steps: + - run: | + fake: &inputs + toolchain: nightly + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: &inputs {} + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: *inputs + """ + + assert [%{line: 7}, %{line: 9}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + test "an anchor written after a tag (`with: !!map &inputs`) is still an anchored mapping" do wf = """ jobs: From 7bce409b09004eda8857b571ece86820f06553d0 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 05:12:39 +0000 Subject: [PATCH 37/47] fix(pins): lock entries per action and ref, hex-case agreement, continued uses: past comments, aliases to the latest anchor - PinIntegrity.locked_refs/1 keyed entries by ref alone, so `github/codeql-action@v4` replaced `actions/checkout@v4`: PI004 lost the checkout entry and PI003 reported one action per moving ref. It now keeps one entry per action and ref, in file order. - PI004 compared refs exactly: an upper-case commit ID against the same commit locked in lower case read as divergence. Commit IDs now compare case-folded; symbolic refs (`V4` vs `v4`) still compare exactly. - PinSites.continuations/1: a `uses:` whose reference YAML continues below it, past blank and comment-only lines. PI001 uses it, and so do WF026 and WF027, which until now read only a reference on the key's own line. A continued value is one the fleet fixers refuse, so those WF026/WF027 findings route to review (auto_fixable: false). - WF026 resolves an alias to the latest `&name` before it, as YAML does, not the first in the file; an anchor defined only after the alias defines nothing for it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- lib/rules/pin_integrity.ex | 54 ++++--- lib/rules/pin_sites.ex | 30 +++- lib/rules/workflow_audit.ex | 199 +++++++++++++++++------- test/rules/pin_integrity_test.exs | 56 +++++++ test/rules/workflow_audit_pins_test.exs | 98 ++++++++++++ 5 files changed, 359 insertions(+), 78 deletions(-) diff --git a/lib/rules/pin_integrity.ex b/lib/rules/pin_integrity.ex index 682cb584..a394178e 100644 --- a/lib/rules/pin_integrity.ex +++ b/lib/rules/pin_integrity.ex @@ -66,9 +66,9 @@ defmodule Hypatia.Rules.PinIntegrity do comment or script text is not one, nor is one on a line of a `run: |` block or of a quoted scalar continued from an earlier line: the file is read through `PinSites.mask_lines/1`, which carries that state from line - to line. A `uses:` whose reference YAML continues on the next line - (`- uses:` then ` a/b@x`) is a site on that line - (`PinSites.continued_site/2`). + to line. A `uses:` whose reference YAML continues below it (`- uses:` + then ` a/b@x`, blank and comment-only lines allowed between) is a site + on the value's line (`PinSites.continuations/1`). Returns a list of maps with `:line`, `:action` (the bare action path as written, e.g. `github/codeql-action/init`), `:action_base` (the first @@ -79,18 +79,16 @@ defmodule Hypatia.Rules.PinIntegrity do """ @spec pin_sites(String.t()) :: [map()] def pin_sites(content) when is_binary(content) do - numbered = content |> PinSites.mask_lines() |> Enum.with_index(1) + masked = PinSites.mask_lines(content) + numbered = Enum.with_index(masked, 1) - # A `uses:` whose reference continues on the next line is a site there. + # A `uses:` whose reference continues below it — past any blank or + # comment-only lines — is a site on the value's line. continued = - numbered - |> Enum.zip(Enum.drop(numbered, 1)) - |> Enum.flat_map(fn {{key, _}, {next, number}} -> - case PinSites.continued_site(key, next) do - nil -> [] - site -> [site_entry(site, number, next |> PinSites.comment() |> String.trim())] - end - end) + for {_key, number, site} <- PinSites.continuations(masked) do + next = Enum.at(masked, number - 1) + site_entry(site, number, next |> PinSites.comment() |> String.trim()) + end numbered |> Enum.flat_map(fn {line, number} -> line_sites(line, number) end) @@ -397,15 +395,18 @@ defmodule Hypatia.Rules.PinIntegrity do end @doc """ - Extract `action@sha` pairs from a `gh actions-lock` file (TOML-ish), - keyed by the resolved SHA or ref. Used by PI003 and PI004. + Extract `action@sha` pairs from a `gh actions-lock` file (TOML-ish). + Used by PI003 and PI004. Reads the first single-quoted `action@ref` on each line and returns - `%{ref => {action_base, line_number}}`, with one-based line numbers and - action paths reduced to their first two segments. Later entries replace - earlier ones with the same ref; no matches produce an empty map. + `[{ref, {action_base, line_number}}]` in file order, with one-based line + numbers and action paths reduced to their first two segments. One entry + per action and ref: two actions locked at the same ref + (`actions/checkout@v4`, `github/codeql-action@v4`) are two entries, and + the same pair met again (a workflow's edge and its dependency record) + keeps the line of its last mention. No matches produce `[]`. """ - @spec locked_refs(String.t()) :: %{String.t() => {String.t(), integer()}} + @spec locked_refs(String.t()) :: [{String.t(), {String.t(), integer()}}] def locked_refs(content) do content |> String.split("\n") @@ -421,7 +422,16 @@ defmodule Hypatia.Rules.PinIntegrity do [{ref, {action_base(action), number}}] end end) - |> Map.new() + |> Enum.reverse() + |> Enum.uniq_by(fn {ref, {action, _}} -> {String.downcase(action), ref_identity(ref)} end) + |> Enum.reverse() + end + + # What a ref names, for comparing two spellings of it: a commit ID is one + # object in either case (`1C5B…` is `1c5b…`); a symbolic ref (`main`, + # `v4`) is a name and compares exactly — `v4` and `4` are two refs. + defp ref_identity(ref) do + if Regex.match?(~r/^[0-9a-fA-F]{40}$/, ref), do: String.downcase(ref), else: ref end # ─── PI004 · lock/spec divergence ───────────────────────────────────── @@ -449,12 +459,14 @@ defmodule Hypatia.Rules.PinIntegrity do locked |> Enum.filter(fn {_ref, {action, _line}} -> same_action?(action, site.action_base) end) |> Enum.map(fn {ref, _} -> ref end) + |> Enum.uniq_by(&ref_identity/1) cond do locked_refs_for_action == [] -> [] - site.ref in locked_refs_for_action -> + # The same commit in another case is agreement, not divergence. + ref_identity(site.ref) in Enum.map(locked_refs_for_action, &ref_identity/1) -> [] true -> diff --git a/lib/rules/pin_sites.ex b/lib/rules/pin_sites.ex index 0a6659b0..68e725f2 100644 --- a/lib/rules/pin_sites.ex +++ b/lib/rules/pin_sites.ex @@ -122,8 +122,9 @@ defmodule Hypatia.Rules.PinSites do @doc """ The site a `uses:` key with nothing after it on its line holds on the next - line, where YAML continues its scalar (`- uses:` then - ` github/codeql-action/init@`), as `sites/1` returns one, its + line with content, where YAML continues its scalar (`- uses:` then + ` github/codeql-action/init@`; `continuations/1` steps over the + blank and comment-only lines between), as `sites/1` returns one, its offsets into `next_line`; nil unless `next_line` sits deeper than the key and holds nothing but the reference, quoted or plain, and a comment. Both lines are as `mask_lines/1` returns them. @@ -149,6 +150,31 @@ defmodule Hypatia.Rules.PinSites do end end + @doc """ + Every `uses:` key in `lines` (as `mask_lines/1` returns them) whose + reference YAML continues below it, as `{key_line, value_line, site}`: + one-based line numbers and the `continued_site/2` site on the value line. + The value is the next line that holds anything but a comment — YAML lets + blank and comment-only lines stand between a key and its scalar + (`- uses:`, ` # rationale`, ` github/codeql-action/init@`). + """ + @spec continuations([String.t()]) :: [{pos_integer(), pos_integer(), map()}] + def continuations(lines) when is_list(lines) do + content = + lines + |> Enum.with_index(1) + |> Enum.reject(fn {line, _} -> String.trim(head(line)) == "" end) + + content + |> Enum.zip(Enum.drop(content, 1)) + |> Enum.flat_map(fn {{key, k}, {next, v}} -> + case continued_site(key, next) do + nil -> [] + site -> [{k, v, site}] + end + end) + end + @doc """ The lines of a whole file, each with the bytes that belong to a scalar opened on an earlier line blanked to `x`, byte for byte: every content diff --git a/lib/rules/workflow_audit.ex b/lib/rules/workflow_audit.ex index 71adcec4..98f36e9a 100644 --- a/lib/rules/workflow_audit.ex +++ b/lib/rules/workflow_audit.ex @@ -1174,17 +1174,22 @@ defmodule Hypatia.Rules.WorkflowAudit do in a job in `jobs`. * `with:` may be an anchor (`with: &inputs`, block or flow), an alias (`with: *inputs`) or carry a merge key (`<<: *inputs`): the anchored - mapping is resolved from the document and its direct entries decide. - An alias with no anchor in the file cannot be read; the step is + mapping is resolved from the document — as YAML resolves an alias, to + the latest `&inputs` before it — and its direct entries decide. An + alias with no anchor before it in the file cannot be read; the step is reported so, not passed silently. + * A `uses:` whose value YAML continues below its key (`- uses:`, then — + past any blank or comment-only lines — the reference on a deeper line) + is the step's key all the same. Auto-fixable: gitbot-fleet `scripts/fix-rust-toolchain-input.sh` inserts `toolchain: stable` (the channel the estate's reusables pass) into a block-style step whose `with:` is absent, a block mapping or a flow mapping. The fixer refuses every other shape rather than guess (exit 2, - file untouched), so a finding on a KYAML document, a flow-style step, or - a `with:` that goes through an anchor or alias carries - `auto_fixable: false` and routes to review instead of to the fixer. + file untouched), so a finding on a KYAML document, a flow-style step, a + `uses:` value continued below its key, or a `with:` that goes through an + anchor or alias carries `auto_fixable: false` and routes to review instead + of to the fixer. """ def check_sha_pinned_toolchain_without_input(workflow_contents) do Enum.flat_map(workflow_contents, fn {filename, content} -> @@ -1192,17 +1197,27 @@ defmodule Hypatia.Rules.WorkflowAudit do normalised = Hypatia.Rules.ResearchExtensions.flow_to_block(content) kyaml? = normalised != content lines = String.split(normalised, "\n") + continued = wf_continued_uses(lines) lines |> Enum.with_index(1) |> Enum.flat_map(fn {line, number} -> + # `- uses:` with the reference below it is read as the one-line key. + {line, continued?} = + case Map.fetch(continued, number) do + {:ok, joined} -> {joined, true} + :error -> {line, false} + end + case Regex.run(@toolchain_sha_pin, line) do [_full, sha] -> - # The fleet fixer matches the identity in lower case only; any - # other spelling is reported for review rather than handed to it. + # The fleet fixer matches the identity in lower case only, and a + # value on its key's line only; any other spelling or layout is + # reported for review rather than handed to it. review = cond do kyaml? -> :kyaml + continued? -> :continued not String.contains?(line, "dtolnay/rust-toolchain@") -> :spelling true -> nil end @@ -1249,7 +1264,7 @@ defmodule Hypatia.Rules.WorkflowAudit do [] true -> - case wf026_with(step, key_indent, lines) do + case wf026_with(step, key_indent, lines, opener) do {:toolchain, _form} -> [] @@ -1267,7 +1282,7 @@ defmodule Hypatia.Rules.WorkflowAudit do with {:ok, open_col} <- wf_flow_opener_before(line, uses_at), true <- wf026_inline_step_at?(lines, line, number, open_col), {:ok, interior, _} <- wf_flow_interior(lines, number, open_col) do - case wf026_with_entry(wf_flow_entries(interior), lines) do + case wf026_with_entry(wf_flow_entries(interior), lines, {number, :eol}) do {:toolchain, _} -> [] verdict -> [wf026_finding(filename, number, sha, verdict, :flow_step, review)] end @@ -1357,8 +1372,8 @@ defmodule Hypatia.Rules.WorkflowAudit do shape = case {verdict, form} do {{:unreadable, :alias, name}, _} -> - " Its `with:` is the alias `*#{name}` and no `&#{name}` anchor is defined in " <> - "the file, so the inputs cannot be read; the alias must be resolved by hand." + " Its `with:` is the alias `*#{name}` and no `&#{name}` anchor is defined before " <> + "it in the file, so the inputs cannot be read; the alias must be resolved by hand." {_, :flow_step} -> " The step is written in flow style; the repair script rewrites block-style " <> @@ -1372,6 +1387,10 @@ defmodule Hypatia.Rules.WorkflowAudit do " The workflow is KYAML; the repair script rewrites block-style steps only, " <> "so this one routes to review." + _ when review == :continued -> + " The `uses:` value is continued below its key; the repair script reads a " <> + "reference on the key's own line only, so this one routes to review." + _ when review == :spelling -> " The action is not spelled `dtolnay/rust-toolchain`; GitHub resolves the name in " <> "either case, but the repair script matches it as written in lower case, so this " <> @@ -1556,7 +1575,9 @@ defmodule Hypatia.Rules.WorkflowAudit do # :alias (`with: *x`) or :merge (a `<<: *x` child). Only a `with:` at the # step's key column is the step's own; `toolchain:` under `env:` or any # other key is not an action input. - defp wf026_with(step, key_indent, lines) do + # `step` starts at line `opener`, so its line i is line `opener + i`: where + # an alias in `with:` stands, which decides the anchor it names. + defp wf026_with(step, key_indent, lines, opener) do step |> Enum.with_index() |> Enum.find_value({:missing, :none}, fn {line, i} -> @@ -1564,7 +1585,7 @@ defmodule Hypatia.Rules.WorkflowAudit do case Regex.run(~r/^\s*(?:-\s+)?["']?with["']?\s*:(.*)$/, line) do [_, value] -> value = value |> wf_strip_comment() |> String.trim() - wf026_with_value(value, line, Enum.drop(step, i + 1), key_indent, lines) + wf026_with_value(value, line, Enum.drop(step, i + 1), key_indent, lines, opener + i) nil -> nil @@ -1573,12 +1594,13 @@ defmodule Hypatia.Rules.WorkflowAudit do end) end - defp wf026_with_value(value, line, rest, key_indent, lines) do + defp wf026_with_value(value, line, rest, key_indent, lines, number) do cond do - # `with: *inputs` — the anchored mapping, wherever it is defined. + # `with: *inputs` — the mapping the latest `&inputs` before it anchors. match?([_ | _], Regex.run(~r/^\*([\w.-]+)$/, value)) -> [_, name] = Regex.run(~r/^\*([\w.-]+)$/, value) - wf026_alias_verdict(name, lines, :alias) + [{col, _}] = Regex.run(~r/\*/, line, return: :index) + wf026_alias_verdict(name, lines, :alias, {number, col}) # `with: {` / `with: &inputs {` — a flow mapping, read to its brace. String.contains?(value, "{") -> @@ -1586,8 +1608,11 @@ defmodule Hypatia.Rules.WorkflowAudit do [{at, _}] = Regex.run(~r/\{/, line, return: :index) case wf_flow_interior([line | rest], 1, at) do - {:ok, interior, _} -> wf026_entries_verdict(wf_flow_entries(interior), lines, form) - :error -> {:missing, form} + {:ok, interior, _} -> + wf026_entries_verdict(wf_flow_entries(interior), lines, form, {number, :eol}) + + :error -> + {:missing, form} end # `with:` / `with: &inputs` — a block mapping: only a DIRECT child is @@ -1595,7 +1620,7 @@ defmodule Hypatia.Rules.WorkflowAudit do # (`components: |` carrying `toolchain: stable`) sit deeper and are text. true -> form = if wf026_anchored?(value), do: :anchor, else: :block - wf026_entries_verdict(wf_block_children(rest, key_indent), lines, form) + wf026_entries_verdict(wf_block_children(rest, key_indent), lines, form, {number, :eol}) end end @@ -1603,8 +1628,9 @@ defmodule Hypatia.Rules.WorkflowAudit do # order: `&inputs`, `&inputs !!map`, `!!map &inputs`. defp wf026_anchored?(value), do: Regex.match?(~r/^(?:!\S*\s+)*&/, value) - # The `with` entry of a flow-style step's own entries. - defp wf026_with_entry(entries, lines) do + # The `with` entry of a flow-style step's own entries; `pos` is the step's + # line, where an alias among them stands. + defp wf026_with_entry(entries, lines, pos) do case List.keyfind(entries, "with", 0) do nil -> {:missing, :none} @@ -1615,15 +1641,18 @@ defmodule Hypatia.Rules.WorkflowAudit do cond do match?([_ | _], Regex.run(~r/^\*([\w.-]+)$/, value)) -> [_, name] = Regex.run(~r/^\*([\w.-]+)$/, value) - wf026_alias_verdict(name, lines, :alias) + wf026_alias_verdict(name, lines, :alias, pos) String.contains?(value, "{") -> form = if wf026_anchored?(value), do: :anchor, else: :flow [{at, _}] = Regex.run(~r/\{/, value, return: :index) case wf_flow_interior([value], 1, at) do - {:ok, interior, _} -> wf026_entries_verdict(wf_flow_entries(interior), lines, form) - :error -> {:missing, form} + {:ok, interior, _} -> + wf026_entries_verdict(wf_flow_entries(interior), lines, form, pos) + + :error -> + {:missing, form} end true -> @@ -1640,9 +1669,11 @@ defmodule Hypatia.Rules.WorkflowAudit do # not resolve (`<<: { ... }`, a block sequence) leaves the inputs unknown: # the finding routes to review (`:merge`), never to the fixer. # The key present with no value (`toolchain:`, `null`, `~`) is not an input - # the action can read; the fleet fixer fills such a key in place. - defp wf026_entries_verdict(entries, lines, form, seen \\ []) do - case wf026_toolchain_in(entries, lines, seen) do + # the action can read; the fleet fixer fills such a key in place. `pos` + # (`{line, column}`, column `:eol` for the whole line) is where the + # mapping stands: an alias in it names the latest anchor before it. + defp wf026_entries_verdict(entries, lines, form, pos, seen \\ []) do + case wf026_toolchain_in(entries, lines, seen, pos) do {:set, :own} -> {:toolchain, form} {:set, :merge} -> {:toolchain, :merge} :null -> {:missing, form} @@ -1658,7 +1689,7 @@ defmodule Hypatia.Rules.WorkflowAudit do # `<<: [*empty, *rust]` is the input, `*rust`'s value is not. Returns # {:set, :own | :merge}, :null, :absent, :unknown (a merge this reader # does not resolve) or {:unreadable, :alias, name}. - defp wf026_toolchain_in(entries, lines, seen) do + defp wf026_toolchain_in(entries, lines, seen, pos) do case Enum.find(entries, fn {k, _} -> k == "toolchain" end) do {_, v} when is_binary(v) -> if wf026_null_value?(v), do: :null, else: {:set, :own} @@ -1670,7 +1701,7 @@ defmodule Hypatia.Rules.WorkflowAudit do case wf026_merge_aliases(v) do {:ok, names} -> Enum.find_value(names, fn name -> - case wf026_alias_lookup(name, lines, seen) do + case wf026_alias_lookup(name, lines, seen, pos) do :absent -> nil {:set, _} -> {:set, :merge} other -> other @@ -1684,12 +1715,12 @@ defmodule Hypatia.Rules.WorkflowAudit do end end - defp wf026_alias_lookup(name, lines, seen) do + defp wf026_alias_lookup(name, lines, seen, pos) do if name in seen do {:unreadable, :alias, name} else - case wf_anchor_entries(lines, name) do - {:ok, entries} -> wf026_toolchain_in(entries, lines, [name | seen]) + case wf_anchor_entries(lines, name, pos) do + {:ok, entries, def_pos} -> wf026_toolchain_in(entries, lines, [name | seen], def_pos) :error -> {:unreadable, :alias, name} end end @@ -1733,23 +1764,32 @@ defmodule Hypatia.Rules.WorkflowAudit do # An alias that reaches itself (`&x` whose mapping merges `*x`) is as # unreadable as one with no anchor. - defp wf026_alias_verdict(name, lines, form, seen \\ []) do + defp wf026_alias_verdict(name, lines, form, pos, seen \\ []) do if name in seen do {:unreadable, :alias, name} else - case wf_anchor_entries(lines, name) do - {:ok, entries} -> wf026_entries_verdict(entries, lines, form, [name | seen]) - :error -> {:unreadable, :alias, name} + case wf_anchor_entries(lines, name, pos) do + {:ok, entries, def_pos} -> + wf026_entries_verdict(entries, lines, form, def_pos, [name | seen]) + + :error -> + {:unreadable, :alias, name} end end end - # The entries of the mapping anchored `&name`: the first `&name` in the - # document outside comments, quoted scalars and block-scalar content (a - # `run: |` fixture writing `fake: &inputs` is text), read as a flow mapping - # when a `{` follows it, else as the block mapping under its line. A - # `note: "see &name"` before the real anchor is text. - defp wf_anchor_entries(lines, name) do + # The entries of the mapping anchored `&name` that an alias at `pos` + # (`{line, column}`, column `:eol` for anywhere on the line) names. YAML + # resolves an alias to the latest `&name` before it, and a document may + # define one again (`defaults: &inputs {...}`, then `other: &inputs {}`): + # the last definition before `pos` decides, and one only after it defines + # nothing for it. Definitions are read outside comments, quoted scalars + # and block-scalar content (a `run: |` fixture writing `fake: &inputs` is + # text; so is a `note: "see &name"`), and the mapping as a flow mapping + # when a `{` follows the anchor, else as the block mapping under its line. + # Returns `{:ok, entries, pos}` with the definition's own position (where + # an alias inside the anchored mapping stands), or :error. + defp wf_anchor_entries(lines, name, {pos_line, pos_col}) do re = ~r/&#{Regex.escape(name)}(?=[\s,{}\[\]]|$)/ states = lines |> Enum.join("\n") |> Hypatia.Rules.PinSites.line_states() @@ -1757,27 +1797,37 @@ defmodule Hypatia.Rules.WorkflowAudit do lines |> Enum.zip(states) |> Enum.with_index(1) - |> Enum.find(fn {{l, {_, state}}, _} -> - state != :text and wf026_plain?(l) and Regex.match?(re, wf_mask_quoted(l)) + |> Enum.take(pos_line) + |> Enum.flat_map(fn {{l, {_, state}}, number} -> + if state != :text and wf026_plain?(l) do + masked = wf_mask_quoted(l) + + re + |> Regex.scan(masked, return: :index) + |> Enum.filter(fn [{at, _}] -> number < pos_line or pos_col == :eol or at < pos_col end) + |> Enum.map(fn [{at, len}] -> {l, masked, number, at, len} end) + else + [] + end end) + |> List.last() |> case do nil -> :error - {{line, _}, number} -> - masked = wf_mask_quoted(line) - [{at, len}] = Regex.run(re, masked, return: :index) + {line, masked, number, at, len} -> after_anchor = binary_part(masked, at + len, byte_size(masked) - at - len) case Regex.run(~r/\{/, after_anchor, return: :index) do [{brace, _}] -> case wf_flow_interior(lines, number, at + len + brace) do - {:ok, interior, _} -> {:ok, wf_flow_entries(interior)} + {:ok, interior, _} -> {:ok, wf_flow_entries(interior), {number, :eol}} :error -> :error end nil -> - {:ok, wf_block_children(Enum.drop(lines, number), wf026_key_indent(line))} + children = wf_block_children(Enum.drop(lines, number), wf026_key_indent(line)) + {:ok, children, {number, :eol}} end end end @@ -2014,6 +2064,24 @@ defmodule Hypatia.Rules.WorkflowAudit do do: wf_flow_split(rest, sep, depth, q, [c | cur], out) # Drop a trailing ` # comment` that sits outside quotes. + # The `uses:` keys whose reference YAML continues below them (`- uses:`, + # then — past any blank or comment-only lines — the reference on a deeper + # line), as `%{key_line => joined}`: the key's line with the reference + # written after it, which the rules read as they read a one-line key. + # Found on the masked lines (`PinSites.mask_lines/1`), so a `uses:` in a + # `run: |` script or a quoted string continues nothing. + defp wf_continued_uses(lines) do + masked = lines |> Enum.join("\n") |> Hypatia.Rules.PinSites.mask_lines() + + for {key, _value, site} <- Hypatia.Rules.PinSites.continuations(masked), into: %{} do + line = Enum.at(lines, key - 1) + + {key, + String.trim_trailing(Hypatia.Rules.PinSites.head(line)) <> + " " <> site.action <> "@" <> site.ref} + end + end + defp wf_strip_comment(text) do text |> :binary.bin_to_list() @@ -2292,8 +2360,13 @@ defmodule Hypatia.Rules.WorkflowAudit do rule. Full 40-hex pins only — that is the only form sha_pinning_required lets a caller write. + A job's `uses:` whose value YAML continues below its key (past any blank + or comment-only lines) is the call all the same. + Auto-fixable: gitbot-fleet `scripts/fix-stale-reusable-pin.sh` rewrites - the pin to the replacement the table names, never to "newest". + the pin to the replacement the table names, never to "newest". It refuses + a value continued below its key (exit 2, file untouched), so that finding + carries `auto_fixable: false` and routes to review. Severity is low and the finding is `advisory: true`: like WF026 the rule reports and routes to the fleet; it is never a blocking gate (AGENTIC @@ -2302,6 +2375,7 @@ defmodule Hypatia.Rules.WorkflowAudit do def check_known_broken_reusable_revision(workflow_contents) do Enum.flat_map(workflow_contents, fn {filename, content} -> lines = String.split(content, "\n") + continued = wf_continued_uses(lines) Enum.flat_map(@broken_reusable_revisions, fn rev -> # Anchored to a `uses:` key at the start of a line (quoted or not) that @@ -2323,6 +2397,13 @@ defmodule Hypatia.Rules.WorkflowAudit do lines |> Enum.with_index(1) |> Enum.flat_map(fn {line, number} -> + # A job's `uses:` with the reference below it is the one-line key. + {line, continued?} = + case Map.fetch(continued, number) do + {:ok, joined} -> {joined, true} + :error -> {line, false} + end + shas = case Regex.run(ref_re, line) do [_full, sha] -> @@ -2342,7 +2423,7 @@ defmodule Hypatia.Rules.WorkflowAudit do Enum.flat_map(shas, fn sha -> if String.downcase(sha) in rev.shas do - [wf027_finding(filename, number, sha, rev)] + [wf027_finding(filename, number, sha, rev, continued?)] else [] end @@ -2399,7 +2480,9 @@ defmodule Hypatia.Rules.WorkflowAudit do end end - defp wf027_finding(filename, number, sha, rev) do + # A value continued below its key is one the fleet fixer refuses (exit 2, + # file untouched): such a finding routes to review. + defp wf027_finding(filename, number, sha, rev, continued?) do %{ rule: "WF027", type: :known_broken_reusable_revision, @@ -2411,13 +2494,19 @@ defmodule Hypatia.Rules.WorkflowAudit do # Advisory, as WF026: reported and routed, never a blocking finding. severity: :low, advisory: true, - auto_fixable: true, - fix_script: "fix-stale-reusable-pin.sh", + auto_fixable: not continued?, + fix_script: if(continued?, do: nil, else: "fix-stale-reusable-pin.sh"), reason: "`#{rev.reusable}` is pinned at `@#{sha}`, inside #{rev.window}: " <> rev.reason <> " Fixed by #{rev.fixed_by}; move the pin to " <> - "`@#{rev.replacement}`.", + "`@#{rev.replacement}`." <> + if(continued?, + do: + " The `uses:` value is continued below its key; the repair script rewrites " <> + "a reference on the key's own line only, so this one routes to review.", + else: "" + ), action: :rewrite_stale_reusable_pin } end diff --git a/test/rules/pin_integrity_test.exs b/test/rules/pin_integrity_test.exs index d9d33c5a..778ad22b 100644 --- a/test/rules/pin_integrity_test.exs +++ b/test/rules/pin_integrity_test.exs @@ -228,6 +228,24 @@ defmodule Hypatia.Rules.PinIntegrityTest do ) == [] end + test "blank and comment-only lines between a uses: key and its reference are skipped" do + # YAML lets them stand between a key and its scalar: the reference is + # still the `uses:` value, so it is still a site — reported and repaired. + content = + " - uses:\n\n # pinned for the 4.38 line\n" <> + " github/codeql-action/init@#{@poison}\n" <> @real_step + + assert [%{line: 4}, %{line: 5}] = PI.pi001_denylisted_pin(content, @policy) + assert {:ok, new_content, [_, _]} = PI.substitute_denylisted_pins(content, @policy) + assert new_content =~ " github/codeql-action/init@#{@good}\n" + + # Control: a comment and then a line no deeper than the key end it. + assert PI.pi001_denylisted_pin( + " - uses:\n # note\n - name: github/codeql-action/init@#{@poison}\n", + @policy + ) == [] + end + test "line_states/1 says which lines are text and which a cut-off quote leaves open" do states = Hypatia.Rules.PinSites.line_states( @@ -596,6 +614,20 @@ defmodule Hypatia.Rules.PinIntegrityTest do assert PI.pi003_locked_moving_ref(lock, @policy) == [] end + + test "two actions locked at one moving ref are two findings, each pair once" do + lock = """ + '.github/workflows/ci.yml': + - 'dtolnay/rust-toolchain@stable' + - 'some/other-action@stable' + dependencies: + 'dtolnay/rust-toolchain@stable': + """ + + assert [a, b] = PI.pi003_locked_moving_ref(lock, @policy) + assert {a.detail.action, a.line} == {"some/other-action", 3} + assert {b.detail.action, b.line} == {"dtolnay/rust-toolchain", 5} + end end describe "pi004_lock_divergence/3" do @@ -628,6 +660,30 @@ defmodule Hypatia.Rules.PinIntegrityTest do assert PI.pi004_lock_divergence(workflow, lock) == [] end + + test "an action sharing its locked ref with another action is still compared" do + # Keyed by ref alone, `github/codeql-action@v4` replaced the checkout + # entry and the divergence below went unreported. + workflow = "- uses: actions/checkout@v3\n" + lock = " - 'actions/checkout@v4'\n - 'github/codeql-action@v4'\n" + + assert [finding] = PI.pi004_lock_divergence(workflow, lock) + assert finding.detail.lock_refs == ["v4"] + end + + test "a commit ID in upper case agrees with the same commit locked in lower case" do + workflow = "- uses: github/codeql-action/init@#{String.upcase(@good)}\n" + lock = " - 'github/codeql-action@#{@good}'\n" + + assert PI.pi004_lock_divergence(workflow, lock) == [] + + # Control: a symbolic ref is a name — `V4` is not the tag `v4`. + assert [_] = + PI.pi004_lock_divergence( + "- uses: actions/checkout@V4\n", + " - 'actions/checkout@v4'\n" + ) + end end describe "pi005_stale_pin/2 — advisory, never an action" do diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs index d70f57fc..1e521658 100644 --- a/test/rules/workflow_audit_pins_test.exs +++ b/test/rules/workflow_audit_pins_test.exs @@ -665,6 +665,87 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do assert f.reason =~ "no `&nowhere` anchor" end + test "an alias names the latest anchor before it, not the first in the file" do + # YAML lets a document define `&inputs` again; `*inputs` is the + # empty mapping here, so the step passes no toolchain. + wf = """ + defaults: &inputs {toolchain: nightly} + other: &inputs {} + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: *inputs + """ + + assert [%{line: 6, auto_fixable: false}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + + # Controls: the other order is configured, and an anchor defined only + # after the alias defines nothing for it. + configured = String.replace(wf, "&inputs {toolchain: nightly}", "&inputs {}", global: false) + + configured = + String.replace(configured, "other: &inputs {}", "other: &inputs {toolchain: nightly}") + + assert [] == + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => configured}) + + later = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: *inputs + defaults: &inputs {toolchain: nightly} + """ + + assert [%{line: 4} = f] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => later}) + + assert f.reason =~ "no `&inputs` anchor is defined before it" + end + + test "a uses: value continued below its key, past a comment, is the step's pin" do + wf = """ + jobs: + check: + steps: + - uses: + # pinned + dtolnay/rust-toolchain@#{@toolchain_sha} + - run: cargo test + """ + + assert [%{line: 4, auto_fixable: false, fix_script: nil} = f] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + + assert f.reason =~ "continued below its key" + + # Controls: the same step with its input is configured, and the + # reference inside a `run: |` script continues nothing. + with_input = + String.replace( + wf, + " - run: cargo test", + " with: { toolchain: stable }\n - run: cargo test" + ) + + assert [] == + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => with_input}) + + script = """ + jobs: + check: + steps: + - run: | + uses: + dtolnay/rust-toolchain@#{@toolchain_sha} + """ + + assert [] == WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => script}) + end + test "a toolchain: key with no value is no input: null and ~ included" do wf = """ jobs: @@ -816,6 +897,23 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do assert WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => wf}) == [] end + test "a job's uses: value continued below its key is the call; routed to review" do + # The fleet fixer refuses a continued value (exit 2), so the finding + # is not handed to it. + wf = + "jobs:\n rust-ci:\n uses:\n\n # dead window\n #{@rust_ci}@#{@broken_standards}\n" + + assert [%{line: 3, auto_fixable: false, fix_script: nil} = f] = + WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => wf}) + + assert f.current_sha == @broken_standards + assert f.reason =~ "continued below its key" + + # Control: under `env:` the same layout is data, not a call. + env = "env:\n CALL:\n #{@rust_ci}@#{@broken_standards}\n" + assert WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => env}) == [] + end + test "silent for another reusable at the same revision: the window is per reusable" do wf = "jobs:\n mirror:\n uses: hyperpolymath/standards/.github/workflows/mirror-reusable.yml@#{@broken_standards}\n" From c88153ae93f8688b36c803b507450efe21b9ebb0 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 05:32:25 +0000 Subject: [PATCH 38/47] fix(pins): uses: values with an anchor or tag are pins; the sweep sees blocked versions outside sites MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - PinSites, WF026 and WF027 required the action reference straight after `uses:` (or its quote), so `uses: &rust dtolnay/rust-toolchain@` and `uses: !!str @` were no site: PI001/PA*, WF026 and WF027 stayed silent. YAML node properties before the value are now skipped, on a key's own line and on a continued value. The fleet fixers refuse such values, so those WF026/WF027 findings route to review; so does a WF027 finding on a flow-style job (`rust: { uses: ... }`), which fix-stale-reusable-pin.sh refuses too. - estate-pin-integrity.sh: phase 1 finds a candidate by any blocked token, but the outside-site check in phase 2 looked for SHAs only, so a blocked version written outside a `uses:` site (a `run: |` script) let the file pass as clean. line_has_denied_token also matches a blocked version written after its own action — `some/action@v4.38.1` is still no finding — and such a file goes to the owner, as one with a stray SHA does. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- lib/rules/pin_sites.ex | 17 +++--- lib/rules/workflow_audit.ex | 71 ++++++++++++++++++------- scripts/sweeps/estate-pin-integrity.sh | 29 +++++++--- test/rules/pin_integrity_test.exs | 24 +++++++++ test/rules/workflow_audit_pins_test.exs | 42 +++++++++++++++ 5 files changed, 149 insertions(+), 34 deletions(-) diff --git a/lib/rules/pin_sites.ex b/lib/rules/pin_sites.ex index 68e725f2..f0a35bde 100644 --- a/lib/rules/pin_sites.ex +++ b/lib/rules/pin_sites.ex @@ -26,10 +26,12 @@ defmodule Hypatia.Rules.PinSites do """ # One `uses:` site anywhere on a line, bounded on the left so `reuses:` is - # not one. Matched against the MASKED line. - @site_re ~r/(?:^|[^A-Za-z0-9_.\/-])["']?uses["']?\s*:\s*["']?(?[A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@(?[^\s#"',}\]]+)/ + # not one. Matched against the MASKED line. The value may carry YAML node + # properties before it (`uses: &rust a/b@x`, `uses: !!str a/b@x`): the + # anchor or tag names the node, the scalar is still the pin. + @site_re ~r/(?:^|[^A-Za-z0-9_.\/-])["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*["']?(?[A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@(?[^\s#"',}\]]+)/ - @uses_tail_re ~r/(?:^|[-{,\s])["']?uses["']?\s*:\s*$/ + @uses_tail_re ~r/(?:^|[-{,\s])["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*$/ @doc """ The structural sites of `line`, in order, as @@ -131,11 +133,12 @@ defmodule Hypatia.Rules.PinSites do """ @spec continued_site(String.t(), String.t()) :: map() | nil def continued_site(key_line, next_line) when is_binary(key_line) and is_binary(next_line) do - with [_, col] <- Regex.run(~r/^(\s*(?:-\s+)?)["']?uses["']?\s*:\s*$/, head(key_line)), + with [_, col] <- + Regex.run(~r/^(\s*(?:-\s+)?)["']?uses["']?\s*:(?:\s+[&!]\S*)*\s*$/, head(key_line)), true <- indent_of(next_line) > byte_size(col), [_, {a_at, a_len}, {r_at, r_len}] <- Regex.run( - ~r/^\s*["']?([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)["']?\s*(?:#.*)?$/, + ~r/^\s*(?:[&!]\S*\s+)*["']?([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)["']?\s*(?:#.*)?$/, next_line, return: :index ) do @@ -347,10 +350,10 @@ defmodule Hypatia.Rules.PinSites do # `prefix` is the masked line up to the site's action. defp structural?(prefix) do cond do - Regex.match?(~r/^\s*(?:-\s+)?(?:\{\s*)?["']?uses["']?\s*:\s*["']?$/, prefix) -> + Regex.match?(~r/^\s*(?:-\s+)?(?:\{\s*)?["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*["']?$/, prefix) -> true - Regex.match?(~r/[{,]\s*["']?uses["']?\s*:\s*["']?$/, prefix) -> + Regex.match?(~r/[{,]\s*["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*["']?$/, prefix) -> prefix |> flow_stack() |> structural_flow_stack?() true -> diff --git a/lib/rules/workflow_audit.ex b/lib/rules/workflow_audit.ex index 98f36e9a..ba8b20de 100644 --- a/lib/rules/workflow_audit.ex +++ b/lib/rules/workflow_audit.ex @@ -1117,11 +1117,11 @@ defmodule Hypatia.Rules.WorkflowAudit do # a commit ID in either case, so upper-case hex is the same pin. # The owner/repository is matched in either case — GitHub resolves # `Dtolnay/Rust-Toolchain` to the same action — and the SHA as written. - @toolchain_sha_pin ~r/^\s*(?:-\s+)?["']?uses["']?\s*:\s*["']?(?i:dtolnay\/rust-toolchain)@([0-9a-fA-F]{40})(?=["'\s#,}\]]|$)/ + @toolchain_sha_pin ~r/^\s*(?:-\s+)?["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*["']?(?i:dtolnay\/rust-toolchain)@([0-9a-fA-F]{40})(?=["'\s#,}\]]|$)/ # The same key written inside a flow mapping on the line that opens it: # `- { uses: dtolnay/rust-toolchain@ }`, `steps: [ { uses: ... } ]`. - @toolchain_sha_pin_inline ~r/[{,]\s*["']?uses["']?\s*:\s*["']?(?i:dtolnay\/rust-toolchain)@([0-9a-fA-F]{40})(?=["'\s#,}\]]|$)/ + @toolchain_sha_pin_inline ~r/[{,]\s*["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*["']?(?i:dtolnay\/rust-toolchain)@([0-9a-fA-F]{40})(?=["'\s#,}\]]|$)/ @doc """ WF026: a SHA-pinned `dtolnay/rust-toolchain` step that passes no @@ -1218,6 +1218,7 @@ defmodule Hypatia.Rules.WorkflowAudit do cond do kyaml? -> :kyaml continued? -> :continued + wf_node_properties?(line) -> :properties not String.contains?(line, "dtolnay/rust-toolchain@") -> :spelling true -> nil end @@ -1387,6 +1388,10 @@ defmodule Hypatia.Rules.WorkflowAudit do " The workflow is KYAML; the repair script rewrites block-style steps only, " <> "so this one routes to review." + _ when review == :properties -> + " The `uses:` value carries a YAML anchor or tag; the repair script reads a " <> + "bare reference only, so this one routes to review." + _ when review == :continued -> " The `uses:` value is continued below its key; the repair script reads a " <> "reference on the key's own line only, so this one routes to review." @@ -2064,6 +2069,11 @@ defmodule Hypatia.Rules.WorkflowAudit do do: wf_flow_split(rest, sep, depth, q, [c | cur], out) # Drop a trailing ` # comment` that sits outside quotes. + # Does the line's own `uses:` value carry YAML node properties — an anchor + # or tag before the reference (`uses: &rust a/b@x`, `uses: !!str a/b@x`)? + defp wf_node_properties?(line), + do: Regex.match?(~r/^\s*(?:-\s+)?["']?uses["']?\s*:\s*[&!]/, line) + # The `uses:` keys whose reference YAML continues below them (`- uses:`, # then — past any blank or comment-only lines — the reference on a deeper # line), as `%{key_line => joined}`: the key's line with the reference @@ -2389,10 +2399,10 @@ defmodule Hypatia.Rules.WorkflowAudit do reusable = wf027_reusable_re(rev.reusable) ref_re = - ~r/^\s*(?:-\s+)?["']?uses["']?\s*:\s*["']?#{reusable}@([0-9a-fA-F]{40})(?=["'\s#,}\]]|$)/ + ~r/^\s*(?:-\s+)?["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*["']?#{reusable}@([0-9a-fA-F]{40})(?=["'\s#,}\]]|$)/ inline_re = - ~r/[{,]\s*["']?uses["']?\s*:\s*["']?#{reusable}@([0-9a-fA-F]{40})(?=["'\s#,}\]]|$)/ + ~r/[{,]\s*["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*["']?#{reusable}@([0-9a-fA-F]{40})(?=["'\s#,}\]]|$)/ lines |> Enum.with_index(1) @@ -2404,12 +2414,21 @@ defmodule Hypatia.Rules.WorkflowAudit do :error -> {line, false} end + # Each call with the layout the fleet fixer would meet: it rewrites + # a bare reference on a block key's own line, nothing else. shas = case Regex.run(ref_re, line) do [_full, sha] -> + review = + cond do + continued? -> :continued + wf_node_properties?(line) -> :properties + true -> nil + end + if match?([_job, "jobs" | _], wf_ancestor_keys(lines, number)) and not wf_in_open_quote?(lines, wf_parent_line(lines, number), number), - do: [sha], + do: [{sha, review}], else: [] nil -> @@ -2418,12 +2437,12 @@ defmodule Hypatia.Rules.WorkflowAudit do |> Enum.filter(fn [{at, _} | _] -> wf027_inline_job?(lines, line, number, at + 1) end) - |> Enum.map(fn [_, {sha_at, 40}] -> binary_part(line, sha_at, 40) end) + |> Enum.map(fn [_, {sha_at, 40}] -> {binary_part(line, sha_at, 40), :flow} end) end - Enum.flat_map(shas, fn sha -> + Enum.flat_map(shas, fn {sha, review} -> if String.downcase(sha) in rev.shas do - [wf027_finding(filename, number, sha, rev, continued?)] + [wf027_finding(filename, number, sha, rev, review)] else [] end @@ -2480,9 +2499,10 @@ defmodule Hypatia.Rules.WorkflowAudit do end end - # A value continued below its key is one the fleet fixer refuses (exit 2, - # file untouched): such a finding routes to review. - defp wf027_finding(filename, number, sha, rev, continued?) do + # The fleet fixer refuses (exit 2, file untouched) a value continued below + # its key, one carrying an anchor or tag, and a flow-style job mapping: a + # finding on any of them (`review`) routes to review. + defp wf027_finding(filename, number, sha, rev, review) do %{ rule: "WF027", type: :known_broken_reusable_revision, @@ -2494,23 +2514,34 @@ defmodule Hypatia.Rules.WorkflowAudit do # Advisory, as WF026: reported and routed, never a blocking finding. severity: :low, advisory: true, - auto_fixable: not continued?, - fix_script: if(continued?, do: nil, else: "fix-stale-reusable-pin.sh"), + auto_fixable: review == nil, + fix_script: if(review == nil, do: "fix-stale-reusable-pin.sh", else: nil), reason: "`#{rev.reusable}` is pinned at `@#{sha}`, inside #{rev.window}: " <> rev.reason <> " Fixed by #{rev.fixed_by}; move the pin to " <> - "`@#{rev.replacement}`." <> - if(continued?, - do: - " The `uses:` value is continued below its key; the repair script rewrites " <> - "a reference on the key's own line only, so this one routes to review.", - else: "" - ), + "`@#{rev.replacement}`." <> wf027_review_note(review), action: :rewrite_stale_reusable_pin } end + defp wf027_review_note(nil), do: "" + + defp wf027_review_note(:continued), + do: + " The `uses:` value is continued below its key; the repair script rewrites " <> + "a reference on the key's own line only, so this one routes to review." + + defp wf027_review_note(:properties), + do: + " The `uses:` value carries a YAML anchor or tag; the repair script rewrites " <> + "a bare reference only, so this one routes to review." + + defp wf027_review_note(:flow), + do: + " The job is written as a flow mapping; the repair script rewrites block-style " <> + "jobs only, so this one routes to review." + # ─── WF017: Secret-consuming action without secret-presence gate ────── # Actions whose primary input is a secret and that fail outright when diff --git a/scripts/sweeps/estate-pin-integrity.sh b/scripts/sweeps/estate-pin-integrity.sh index 863f3a96..23c84ae3 100755 --- a/scripts/sweeps/estate-pin-integrity.sh +++ b/scripts/sweeps/estate-pin-integrity.sh @@ -172,7 +172,7 @@ site_parts() { # the same column of the real one. A `uses:` written inside # `- run: "echo uses: a/b@"`, on a continuation line of a quoted scalar, # in a `run: |` script or in a comment is therefore no site this sweep -# rewrites; a denylisted SHA there is still seen by line_has_denied_sha on +# rewrites; a denied SHA or version there is still seen by line_has_denied_token on # the raw line, and flagged for the owner. A quote never closed before the # end of the file keeps its lines masked: whatever they hold is flagged, # never repaired. @@ -308,10 +308,25 @@ replace_site() { # }' } -# A denylisted SHA token in any position (a lock record key, a fixture): a -# finding in either spelling, but not a `uses:` site this sweep rewrites. -line_has_denied_sha() { - [ -n "$SHA_ALTERNATION" ] && printf '%s\n' "$1" | grep -qiE "@(${SHA_ALTERNATION})(${REF_END})" +# A denied pin token in any position (a lock record key, a fixture, a block +# scalar), not a `uses:` site this sweep rewrites: a denylisted SHA in either +# spelling, or a blocked version written after ITS action +# (`github/codeql-action/init@v4.38.1` in a `run: |` script) — phase 1 found +# the candidate by that token, so phase 2 must see it too, never call the +# file clean. A blocked version is blocked for its own action only, as at a +# site (`denylisted_pair`): `some/action@v4.38.1` is no finding. +line_has_denied_token() { + local tok t + if [ -n "$SHA_ALTERNATION" ] && printf '%s\n' "$1" | grep -qiE "@(${SHA_ALTERNATION})(${REF_END})"; then + return 0 + fi + [ ${#DENY_VERSIONS[@]} -gt 0 ] || return 1 + while IFS= read -r tok; do + [ -n "$tok" ] || continue + t=$(printf '%s' "$tok" | sed -E "s/${REF_END_CLASS}\$//") + denylisted_pair "$(printf '%s' "${t%@*}" | awk -F/ '{print $1"/"$2}')" "${t##*@}" && return 0 + done < <(printf '%s\n' "$1" | grep -oiE "[A-Za-z0-9_.-]+/[A-Za-z0-9_./-]*@(${REF_ALTERNATION})(${REF_END})" || true) + return 1 } # Lines carrying a denied pin: a `uses:` site whose action/ref pair the @@ -321,7 +336,7 @@ count_denied_lines() { masked=$(mktemp) mask_file "$1" > "$masked" while IFS= read -r line <&3 && IFS= read -r mline <&4; do - if line_has_denied_sha "$line"; then n=$((n + 1)); continue; fi + if line_has_denied_token "$line"; then n=$((n + 1)); continue; fi while IFS= read -r site; do [ -n "$site" ] || continue site_parts "$site" @@ -602,7 +617,7 @@ while IFS=$'\t' read -r org repo path; do # A denylisted SHA left on the line that is not at a `uses:` site — the # line had none, or the SHA sits in a quoted scalar or a comment the # mask blanked — is a finding for the owner, never part of a repair. - if line_has_denied_sha "$out" && { [ "$site_seen" -eq 0 ] || ! line_has_denied_sha "$mout"; }; then + if line_has_denied_token "$out" && { [ "$site_seen" -eq 0 ] || ! line_has_denied_token "$mout"; }; then outside_uses=$((outside_uses + 1)); fixable=0 fi # The comment is relabelled only when the line has ONE `uses:` site: on a diff --git a/test/rules/pin_integrity_test.exs b/test/rules/pin_integrity_test.exs index 778ad22b..4d26bc14 100644 --- a/test/rules/pin_integrity_test.exs +++ b/test/rules/pin_integrity_test.exs @@ -246,6 +246,30 @@ defmodule Hypatia.Rules.PinIntegrityTest do ) == [] end + test "a uses: value carrying an anchor or tag is the pin all the same" do + # YAML node properties name the node; the scalar after them is the + # action reference, so it is a site — reported and repaired. + content = + " - uses: &init github/codeql-action/init@#{@poison}\n" <> + " - uses: !!str \"github/codeql-action/analyze@#{@poison}\"\n" <> + " - { name: x, uses: !!str github/codeql-action/upload-sarif@#{@poison} }\n" <> + " - uses: !!str\n github/codeql-action/autobuild@#{@poison}\n" + + assert [%{line: 1}, %{line: 2}, %{line: 3}, %{line: 5}] = + PI.pi001_denylisted_pin(content, @policy) + + assert {:ok, new_content, [_, _, _, _]} = PI.substitute_denylisted_pins(content, @policy) + refute new_content =~ @poison + assert new_content =~ " - uses: &init github/codeql-action/init@#{@good}\n" + assert new_content =~ " - uses: !!str \"github/codeql-action/analyze@#{@good}\"\n" + + # Control: `&` or `!` inside a quoted name is text, not a property. + assert PI.pi001_denylisted_pin( + " - name: \"uses: !!str github/codeql-action/init@#{@poison}\"\n", + @policy + ) == [] + end + test "line_states/1 says which lines are text and which a cut-off quote leaves open" do states = Hypatia.Rules.PinSites.line_states( diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs index 1e521658..acbb2a7a 100644 --- a/test/rules/workflow_audit_pins_test.exs +++ b/test/rules/workflow_audit_pins_test.exs @@ -746,6 +746,24 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do assert [] == WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => script}) end + test "a uses: value carrying an anchor or tag is the step's pin; routed to review" do + wf = """ + jobs: + check: + steps: + - uses: &rust dtolnay/rust-toolchain@#{@toolchain_sha} + - uses: !!str dtolnay/rust-toolchain@#{@toolchain_sha} + """ + + assert [ + %{line: 4, auto_fixable: false, fix_script: nil} = a, + %{line: 5, auto_fixable: false} = b + ] = WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + + assert a.reason =~ "anchor or tag" + assert b.reason =~ "anchor or tag" + end + test "a toolchain: key with no value is no input: null and ~ included" do wf = """ jobs: @@ -914,6 +932,30 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do assert WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => env}) == [] end + test "a job's uses: with an anchor or tag, or a flow-style job, is the call; routed to review" do + # The fleet fixer refuses all three (exit 2), so none is handed to it. + wf = + "jobs:\n a:\n uses: !!str #{@rust_ci}@#{@broken_standards}\n" <> + " b:\n uses: &call #{@rust_ci}@#{@broken_standards}\n" <> + " c: { uses: #{@rust_ci}@#{@broken_standards} }\n" + + assert [ + %{line: 3, auto_fixable: false, fix_script: nil} = a, + %{line: 5, auto_fixable: false} = b, + %{line: 6, auto_fixable: false} = c + ] = WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => wf}) + + assert a.reason =~ "anchor or tag" + assert b.reason =~ "anchor or tag" + assert c.reason =~ "flow mapping" + + # Control: the plain block call is still handed to the fixer. + plain = "jobs:\n a:\n uses: #{@rust_ci}@#{@broken_standards}\n" + + assert [%{auto_fixable: true, fix_script: "fix-stale-reusable-pin.sh"}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => plain}) + end + test "silent for another reusable at the same revision: the window is per reusable" do wf = "jobs:\n mirror:\n uses: hyperpolymath/standards/.github/workflows/mirror-reusable.yml@#{@broken_standards}\n" From 9660f0d6ef58c44673f9d81d19ad09e88d5e1c81 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 05:53:42 +0000 Subject: [PATCH 39/47] fix(pins): continued uses: values inside flow mappings; detached flow steps routed to review; flow-step aliases by column - PinSites.continued_site/2 read only a block `uses:` key, so a flow step written `steps: [ { uses:` with its reference on the next line (`github/codeql-action/init@ } ]`) held no site: PI001 and the substitution repair, WF026 and WF027 all missed it. A flow-mapping entry in a step or job position (the structural test sites/1 applies) now continues too; its value runs to the delimiter that ends it. - WF026 took a step for block-style when its `{` stood alone on a line above (`steps: [`, `{`, `uses: ...`) and handed it to fix-rust-toolchain-input.sh, which refuses it. PinSites.line_depths/1 gives the flow depth at each line's start; a step inside an open flow collection routes to review. (WF027 is unchanged: fix-stale-reusable-pin rewrites a flow job's `uses:` key that stands on its own line.) - A flow step's `with: *alias` was resolved from the end of its line, so an anchor redefined later on that line won. The `with` value's own line and column are now passed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- lib/rules/pin_sites.ex | 62 +++++++++++++++++---- lib/rules/workflow_audit.ex | 74 ++++++++++++++++++++++--- test/rules/pin_integrity_test.exs | 17 ++++++ test/rules/workflow_audit_pins_test.exs | 56 +++++++++++++++++++ 4 files changed, 190 insertions(+), 19 deletions(-) diff --git a/lib/rules/pin_sites.ex b/lib/rules/pin_sites.ex index f0a35bde..0628c3ed 100644 --- a/lib/rules/pin_sites.ex +++ b/lib/rules/pin_sites.ex @@ -127,21 +127,38 @@ defmodule Hypatia.Rules.PinSites do line with content, where YAML continues its scalar (`- uses:` then ` github/codeql-action/init@`; `continuations/1` steps over the blank and comment-only lines between), as `sites/1` returns one, its - offsets into `next_line`; nil unless `next_line` sits deeper than the key - and holds nothing but the reference, quoted or plain, and a comment. Both - lines are as `mask_lines/1` returns them. + offsets into `next_line`; nil otherwise. Both lines are as `mask_lines/1` + returns them. + + The key is a block key (at the line's start or after its list dash), or an + entry of a flow mapping in a step or job position as `sites/1` reads one + (`steps: [ { uses:`, `- { name: x, uses:`). A block key's value sits deeper + than the key and holds nothing but the reference, quoted or plain, and a + comment; a flow entry's value is the reference up to the delimiter that + ends it (`,`, `}` or `]`), wherever the line is indented — inside a flow + collection the scalar continues on any line. """ @spec continued_site(String.t(), String.t()) :: map() | nil def continued_site(key_line, next_line) when is_binary(key_line) and is_binary(next_line) do - with [_, col] <- - Regex.run(~r/^(\s*(?:-\s+)?)["']?uses["']?\s*:(?:\s+[&!]\S*)*\s*$/, head(key_line)), - true <- indent_of(next_line) > byte_size(col), - [_, {a_at, a_len}, {r_at, r_len}] <- - Regex.run( - ~r/^\s*(?:[&!]\S*\s+)*["']?([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)["']?\s*(?:#.*)?$/, - next_line, - return: :index - ) do + key = head(key_line) + + value_re = + cond do + m = Regex.run(~r/^(\s*(?:-\s+)?)["']?uses["']?\s*:(?:\s+[&!]\S*)*\s*$/, key) -> + if indent_of(next_line) > byte_size(Enum.at(m, 1)), + do: + ~r/^\s*(?:[&!]\S*\s+)*["']?([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)["']?\s*(?:#.*)?$/ + + Regex.match?(~r/[{,]\s*["']?uses["']?\s*:(?:\s+[&!]\S*)*\s*$/, key) and + structural?(String.trim_trailing(key) <> " ") -> + ~r/^\s*(?:[&!]\S*\s+)*["']?([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)["']?\s*(?:[,}\]]|#|$)/ + + true -> + nil + end + + with %Regex{} <- value_re, + [_, {a_at, a_len}, {r_at, r_len}] <- Regex.run(value_re, next_line, return: :index) do %{ action: binary_part(next_line, a_at, a_len), ref: binary_part(next_line, r_at, r_len), @@ -153,6 +170,27 @@ defmodule Hypatia.Rules.PinSites do end end + @doc """ + The flow depth at the start of each line of `content` — how many `[` and + `{` the lines above left open, as the walk of `line_states/1` counts them + (outside quoted scalars, comments and block-scalar content). A line at + depth 0 is block structure; one deeper sits inside a flow collection, even + when nothing on the line itself shows it (`steps: [`, then `{` alone, then + `uses: ...`). + """ + @spec line_depths(String.t()) :: [non_neg_integer()] + def line_depths(content) when is_binary(content) do + content + |> String.split("\n") + |> Enum.with_index() + |> Enum.map_reduce(%{q: nil, opened: nil, block: nil, depth: 0, pending: false}, fn + {line, n}, st -> + {_, next} = mask_file_line(line, n, st) + {st.depth, next} + end) + |> elem(0) + end + @doc """ Every `uses:` key in `lines` (as `mask_lines/1` returns them) whose reference YAML continues below it, as `{key_line, value_line, site}`: diff --git a/lib/rules/workflow_audit.ex b/lib/rules/workflow_audit.ex index ba8b20de..49d21d97 100644 --- a/lib/rules/workflow_audit.ex +++ b/lib/rules/workflow_audit.ex @@ -1198,6 +1198,9 @@ defmodule Hypatia.Rules.WorkflowAudit do kyaml? = normalised != content lines = String.split(normalised, "\n") continued = wf_continued_uses(lines) + # A step inside an open flow collection is a flow-style step even when + # its `{` stands alone on a line above (`steps: [`, `{`, `uses: ...`). + depths = Hypatia.Rules.PinSites.line_depths(normalised) lines |> Enum.with_index(1) @@ -1223,7 +1226,8 @@ defmodule Hypatia.Rules.WorkflowAudit do true -> nil end - wf026_block_step_finding(filename, lines, line, number, sha, review) + in_flow? = Enum.at(depths, number - 1, 0) > 0 + wf026_block_step_finding(filename, lines, line, number, sha, review, in_flow?) _ -> # Several flow-style steps may share one physical line @@ -1240,10 +1244,12 @@ defmodule Hypatia.Rules.WorkflowAudit do end) end - # The block-style path: `uses:` is a key on its own line. - defp wf026_block_step_finding(filename, lines, line, number, sha, review) do + # The block-style path: `uses:` is a key on its own line. A step whose + # opener is `- {`, or which sits inside a flow collection opened on an + # earlier line (`in_flow?`), is a flow-style step: the fixer refuses it. + defp wf026_block_step_finding(filename, lines, line, number, sha, review, in_flow?) do {step, key_indent, opener} = wf026_step(lines, number) - flow_item? = Regex.match?(~r/^\s*-\s+\{/, Enum.at(lines, opener - 1)) + flow_item? = in_flow? or Regex.match?(~r/^\s*-\s+\{/, Enum.at(lines, opener - 1)) cond do # `uses:` nested under `env:`, inside a `run: |` script or under @@ -1283,7 +1289,15 @@ defmodule Hypatia.Rules.WorkflowAudit do with {:ok, open_col} <- wf_flow_opener_before(line, uses_at), true <- wf026_inline_step_at?(lines, line, number, open_col), {:ok, interior, _} <- wf_flow_interior(lines, number, open_col) do - case wf026_with_entry(wf_flow_entries(interior), lines, {number, :eol}) do + # Where the step's `with` value stands, line and column: an alias + # there names the latest anchor before it, not one later on its line. + pos = + case wf_flow_entry_value_at(interior, "with") do + nil -> {number, :eol} + at -> wf_interior_pos(interior, at, number, open_col) + end + + case wf026_with_entry(wf_flow_entries(interior), lines, pos) do {:toolchain, _} -> [] verdict -> [wf026_finding(filename, number, sha, verdict, :flow_step, review)] end @@ -1633,8 +1647,8 @@ defmodule Hypatia.Rules.WorkflowAudit do # order: `&inputs`, `&inputs !!map`, `!!map &inputs`. defp wf026_anchored?(value), do: Regex.match?(~r/^(?:!\S*\s+)*&/, value) - # The `with` entry of a flow-style step's own entries; `pos` is the step's - # line, where an alias among them stands. + # The `with` entry of a flow-style step's own entries; `pos` is where its + # value stands, which decides the anchor an alias there names. defp wf026_with_entry(entries, lines, pos) do case List.keyfind(entries, "with", 0) do nil -> @@ -2069,6 +2083,52 @@ defmodule Hypatia.Rules.WorkflowAudit do do: wf_flow_split(rest, sep, depth, q, [c | cur], out) # Drop a trailing ` # comment` that sits outside quotes. + # The byte offset in a flow mapping's interior at which the value of its + # top-level `key` entry starts, or nil. Entries are split at commas outside + # quotes and nested brackets. + defp wf_flow_entry_value_at(interior, key) do + re = ~r/^\s*["']?#{Regex.escape(key)}["']?\s*:\s*/ + + interior + |> wf_flow_entry_starts() + |> Enum.find_value(fn at -> + case Regex.run(re, binary_part(interior, at, byte_size(interior) - at), return: :index) do + [{0, len}] -> at + len + _ -> nil + end + end) + end + + defp wf_flow_entry_starts(interior) do + interior + |> :binary.bin_to_list() + |> Enum.with_index() + |> Enum.reduce({[0], 0, nil}, fn + {c, _}, {acc, d, nil} when c in [?", ?'] -> {acc, d, c} + {c, _}, {acc, d, nil} when c in [?{, ?[] -> {acc, d + 1, nil} + {c, _}, {acc, d, nil} when c in [?}, ?]] -> {acc, d - 1, nil} + {?,, i}, {acc, 0, nil} -> {[i + 1 | acc], 0, nil} + {_, _}, {acc, d, nil} -> {acc, d, nil} + {q, _}, {acc, d, q} -> {acc, d, nil} + {_, _}, {acc, d, q} -> {acc, d, q} + end) + |> elem(0) + |> Enum.reverse() + end + + # Offset `at` of a flow interior that `wf_flow_interior/3` read from the + # `{` at `open_col` of line `number`, as `{line, column}` in the file. + defp wf_interior_pos(interior, at, number, open_col) do + case :binary.matches(binary_part(interior, 0, at), "\n") do + [] -> + {number, open_col + 1 + at} + + nls -> + {nl, _} = List.last(nls) + {number + length(nls), at - nl - 1} + end + end + # Does the line's own `uses:` value carry YAML node properties — an anchor # or tag before the reference (`uses: &rust a/b@x`, `uses: !!str a/b@x`)? defp wf_node_properties?(line), diff --git a/test/rules/pin_integrity_test.exs b/test/rules/pin_integrity_test.exs index 4d26bc14..224bf084 100644 --- a/test/rules/pin_integrity_test.exs +++ b/test/rules/pin_integrity_test.exs @@ -270,6 +270,23 @@ defmodule Hypatia.Rules.PinIntegrityTest do ) == [] end + test "a uses: entry of a flow step whose reference continues on the next line is a site" do + content = + " - { name: x, uses:\n github/codeql-action/init@#{@poison} }\n" <> + " steps: [ { uses:\n github/codeql-action/analyze@#{@poison}, name: y } ]\n" + + assert [%{line: 2}, %{line: 4}] = PI.pi001_denylisted_pin(content, @policy) + assert {:ok, new_content, [_, _]} = PI.substitute_denylisted_pins(content, @policy) + assert new_content =~ " github/codeql-action/init@#{@good} }\n" + assert new_content =~ " github/codeql-action/analyze@#{@good}, name: y } ]\n" + + # Control: an input's mapping is not a step — `with: { uses:` is data. + assert PI.pi001_denylisted_pin( + " with: { uses:\n github/codeql-action/init@#{@poison} }\n", + @policy + ) == [] + end + test "line_states/1 says which lines are text and which a cut-off quote leaves open" do states = Hypatia.Rules.PinSites.line_states( diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs index acbb2a7a..29d69a6e 100644 --- a/test/rules/workflow_audit_pins_test.exs +++ b/test/rules/workflow_audit_pins_test.exs @@ -764,6 +764,55 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do assert b.reason =~ "anchor or tag" end + test "a flow step whose `{` stands on its own line is a flow step; routed to review" do + wf = """ + jobs: + check: + steps: [ + { + uses: dtolnay/rust-toolchain@#{@toolchain_sha} + } + ] + """ + + assert [%{line: 5, auto_fixable: false, fix_script: nil} = f] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + + assert f.reason =~ "flow style" + end + + test "a flow step's uses: continued on the next line is the step's pin" do + wf = """ + jobs: + check: + steps: [ { uses: + dtolnay/rust-toolchain@#{@toolchain_sha} } ] + """ + + assert [%{line: 3, auto_fixable: false}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "a flow step's alias names the anchor before it, not one later on its line" do + # `*inputs` is the empty mapping defined first; the configured + # `&inputs` after it on the same line is not what it names. + wf = """ + x: &inputs {} + jobs: + check: + steps: [ { uses: dtolnay/rust-toolchain@#{@toolchain_sha}, with: *inputs }, { run: echo, env: &inputs {toolchain: nightly} } ] + """ + + assert [%{line: 4}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + + # Control: with the configured anchor first, the alias names it. + configured = String.replace(wf, "x: &inputs {}", "x: &inputs {toolchain: stable}") + + assert [] == + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => configured}) + end + test "a toolchain: key with no value is no input: null and ~ included" do wf = """ jobs: @@ -956,6 +1005,13 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => plain}) end + test "a flow-style job's uses: continued on the next line is the call" do + wf = "jobs: { rust: { uses:\n #{@rust_ci}@#{@broken_standards} } }\n" + + assert [%{line: 1, auto_fixable: false, current_sha: @broken_standards}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => wf}) + end + test "silent for another reusable at the same revision: the window is per reusable" do wf = "jobs:\n mirror:\n uses: hyperpolymath/standards/.github/workflows/mirror-reusable.yml@#{@broken_standards}\n" From e61358987bf278d842a3cef00845277ae99b27a4 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 06:41:57 +0000 Subject: [PATCH 40/47] fix(pins): aliased uses: resolved to the anchor's site; continued pins seen in PR diffs; flow ancestry without indentation - PinSites.alias_sites/1: a `uses: *name` (block key or step/job flow entry) resolves to the latest `&name` before it. PI001 reports and substitutes the pin on the anchor's line, once however many aliases name it; WF026/WF027 read the aliased reference and route it to review (the fleet fixers do not rewrite anchored values). - PrAutomerge: a re-pinned value under an unchanged `uses:` key is the site it continues, on either side of a hunk, so a poisoned continued pin is no longer invisible to poison_sites/2 and pin_lines_only?/1. - WF026/WF027: inside a flow collection opened above, a line's ancestry is the brackets still open (wf_flow_context/4, now readable up to a column of the line, with each bracket's kind), not its indentation. `jobs: {` / `build: {` / `steps: [` / `{` / `uses:` all in column 0, and the inline forms on such lines, are steps and jobs again. A column-0 own-line job key the fleet fixer cannot place (it reads indentation) routes to review; an indented KYAML one it rewrites stays auto-fixable. - wf_flow_context moved below the wf_walk_up/4 clauses (CI compiles with --warnings-as-errors); Tuple.append/2 avoided. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- lib/rules/pin_integrity.ex | 17 +- lib/rules/pin_sites.ex | 86 ++++++++++ lib/rules/pr_automerge.ex | 98 +++++++++--- lib/rules/workflow_audit.ex | 202 ++++++++++++++++++++---- test/rules/pin_integrity_test.exs | 35 ++++ test/rules/pr_automerge_test.exs | 19 +++ test/rules/workflow_audit_pins_test.exs | 164 +++++++++++++++++++ 7 files changed, 559 insertions(+), 62 deletions(-) diff --git a/lib/rules/pin_integrity.ex b/lib/rules/pin_integrity.ex index a394178e..c8a64643 100644 --- a/lib/rules/pin_integrity.ex +++ b/lib/rules/pin_integrity.ex @@ -68,7 +68,9 @@ defmodule Hypatia.Rules.PinIntegrity do read through `PinSites.mask_lines/1`, which carries that state from line to line. A `uses:` whose reference YAML continues below it (`- uses:` then ` a/b@x`, blank and comment-only lines allowed between) is a site - on the value's line (`PinSites.continuations/1`). + on the value's line (`PinSites.continuations/1`), and a `uses: *name` is a + site on the line of the anchor it names, where its reference stands + (`PinSites.alias_sites/1`). Returns a list of maps with `:line`, `:action` (the bare action path as written, e.g. `github/codeql-action/init`), `:action_base` (the first @@ -90,9 +92,22 @@ defmodule Hypatia.Rules.PinIntegrity do site_entry(site, number, next |> PinSites.comment() |> String.trim()) end + # A `uses: *name` runs the reference the anchor holds: its site is the + # anchor's value, where a repair rewrites it (every alias with it). + aliased = + for %{anchor_line: number, site: site} <- PinSites.alias_sites(masked) do + site_entry( + site, + number, + masked |> Enum.at(number - 1) |> PinSites.comment() |> String.trim() + ) + end + numbered |> Enum.flat_map(fn {line, number} -> line_sites(line, number) end) |> Kernel.++(continued) + |> Kernel.++(aliased) + |> Enum.uniq_by(&{&1.line, &1.ref_at}) |> Enum.sort_by(&{&1.line, &1.ref_at}) end diff --git a/lib/rules/pin_sites.ex b/lib/rules/pin_sites.ex index 0628c3ed..51922697 100644 --- a/lib/rules/pin_sites.ex +++ b/lib/rules/pin_sites.ex @@ -170,6 +170,92 @@ defmodule Hypatia.Rules.PinSites do end end + @doc """ + Every `uses:` whose value is an alias (`- uses: *checkout`, a block key or a + step/job flow entry as `sites/1` reads one), resolved to the site the + anchor it names holds: `%{line, at, len, anchor_line, site}` — the alias' + line, its byte offset and length, the anchor's line, and the site as + `sites/1` returns one, its offsets into the anchor's line. YAML resolves an + alias to the latest `&name` before it, so a later definition does not + count and an earlier one is shadowed — by any value, including one that is + no `action@ref` scalar, which names no site. Lines as `mask_lines/1` + returns them. + """ + @spec alias_sites([String.t()]) :: [map()] + def alias_sites(lines) when is_list(lines) do + numbered = Enum.with_index(lines, 1) + defs = for {line, n} <- numbered, d <- anchor_defs(line), do: Map.put(d, :line, n) + + for {line, n} <- numbered, + {name, at, len} <- uses_aliases(line), + %{site: %{} = site, line: anchor_line} <- [latest_anchor(defs, name, n, at)] do + %{line: n, at: at, len: len, anchor_line: anchor_line, site: site} + end + end + + # The anchors `line` defines where a node starts (outside quotes and the + # comment), each `%{name, at, site}`: the site its scalar value is, or nil. + defp anchor_defs(line) do + masked = mask(line) + + ~r/&([\w.-]+)/ + |> Regex.scan(masked, return: :index) + |> Enum.filter(fn [{at, _}, _] -> + Regex.match?( + ~r/(?:^\s*(?:-\s+)*|:\s+|[\[{,]\s*)(?:![^\s]*\s+)*$/, + binary_part(masked, 0, at) + ) + end) + |> Enum.map(fn [{at, len}, {n_at, n_len}] -> + rest_at = at + len + rest = binary_part(line, rest_at, byte_size(line) - rest_at) + + site = + case Regex.run( + ~r/^\s+(?:![^\s]*\s+)*["']?([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)["']?(?=[\s,}\]#]|$)/, + rest, + return: :index + ) do + [_, {a_at, a_len}, {r_at, r_len}] -> + %{ + action: binary_part(rest, a_at, a_len), + ref: binary_part(rest, r_at, r_len), + action_at: rest_at + a_at, + ref_at: rest_at + r_at + } + + _ -> + nil + end + + %{name: binary_part(masked, n_at, n_len), at: at, site: site} + end) + end + + # The `uses:` keys of `line` whose value is an alias, at a site position, + # as `{name, offset of the "*", length of "*name"}`. + defp uses_aliases(line) do + masked = mask(line) + + ~r/["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*\*([\w.-]+)/ + |> Regex.scan(masked, return: :index) + |> Enum.flat_map(fn [{at, len}, {n_at, n_len}] -> + star = n_at - 1 + prefix = binary_part(masked, 0, at) + + if structural?(binary_part(masked, 0, star)) and + not Regex.match?(~r/[A-Za-z0-9_.\/-]$/, prefix), + do: [{binary_part(masked, n_at, n_len), star, at + len - star}], + else: [] + end) + end + + defp latest_anchor(defs, name, line, at) do + defs + |> Enum.filter(&(&1.name == name and (&1.line < line or (&1.line == line and &1.at < at)))) + |> Enum.max_by(&{&1.line, &1.at}, fn -> nil end) + end + @doc """ The flow depth at the start of each line of `content` — how many `[` and `{` the lines above left open, as the walk of `line_states/1` counts them diff --git a/lib/rules/pr_automerge.ex b/lib/rules/pr_automerge.ex index bfe74d8f..e9a80f63 100644 --- a/lib/rules/pr_automerge.ex +++ b/lib/rules/pr_automerge.ex @@ -123,11 +123,11 @@ defmodule Hypatia.Rules.PrAutomerge do end defp poison_only_file?(file, policy) do - added = added_lines(file) + added = added_entries(file) pin_lines_only?(file) and added != [] and - Enum.all?(added, fn line -> - pins = parse_pins(line) + Enum.all?(added, fn {line, _certain?, continued} -> + pins = parse_pins(line) ++ continued pins != [] and Enum.all?(pins, &denylisted_pin?(&1, policy)) end) end @@ -159,14 +159,17 @@ defmodule Hypatia.Rules.PrAutomerge do |> Enum.flat_map(fn file -> file |> added_entries() - |> Enum.flat_map(fn {line, certain?} -> + |> Enum.flat_map(fn {line, certain?, continued} -> # Every site on the line: a flow sequence can put several steps on # one changed line, and a pin judged by its first site alone lets a - # denylisted second one through to `arm_auto`. The action is - # compared case-folded — GitHub resolves `GitHub/CodeQL-Action` and - # `github/codeql-action` to one repository. + # denylisted second one through to `arm_auto`. A line that is the + # value of a `uses:` key on the line above it (the key unchanged + # context, the reference re-pinned) holds that site too. The action + # is compared case-folded — GitHub resolves `GitHub/CodeQL-Action` + # and `github/codeql-action` to one repository. line |> parse_pins() + |> Kernel.++(continued) |> Enum.filter(fn %{base: base, ref: ref} -> String.downcase(base) == String.downcase(Map.get(entry, "action", "")) and denylisted_ref?(entry, ref) @@ -231,14 +234,22 @@ defmodule Hypatia.Rules.PrAutomerge do # a new step (`+ { uses: a/b@main },`) or a swapped action is a change to # what the workflow runs, not a re-pin of what it already ran. lines != [] and - Enum.all?(lines, fn {line, scalar?, certain?} -> - certain? and not scalar? and pin_line?(line) + Enum.all?(lines, fn {line, scalar?, certain?, continued} -> + certain? and not scalar? and (pin_line?(line) or continued_value_line?(line, continued)) end) and - action_multiset(removed_lines(file)) == action_multiset(added_lines(file)) + action_multiset(removed_pins(file)) == action_multiset(added_pins(file)) end - defp action_multiset(lines) do - lines |> Enum.flat_map(&parse_pins/1) |> Enum.map(&String.downcase(&1.action)) |> Enum.sort() + # A changed line that is nothing but the reference a `uses:` key above it + # holds (`- a/b@x` -> `+ a/b@y`): a re-pin as much as a + # one-line `uses:` is. + defp continued_value_line?(line, [_ | _]), + do: Regex.match?(~r/^\s*\S+\s*(?:#.*)?$/, String.replace(line, ~r/^[-+]/, "")) + + defp continued_value_line?(_line, []), do: false + + defp action_multiset(pins) do + pins |> Enum.map(&String.downcase(&1.action)) |> Enum.sort() end # ─── PA002/PA003/PA004 · version deltas ─────────────────────────────── @@ -270,8 +281,8 @@ defmodule Hypatia.Rules.PrAutomerge do # several steps on one line (`steps: [ { uses: a@x }, { uses: b@y } ]`), # and each of them is a delta of its own — including a major bump on a # line that does not start with `uses:`. - removed = file |> removed_lines() |> Enum.flat_map(&parse_pins/1) - added = file |> added_lines() |> Enum.flat_map(&parse_pins/1) + removed = removed_pins(file) + added = added_pins(file) # Each added site pairs with ONE removed site of the same action, in # order: a line that keeps one `actions/checkout` at v4 and moves a @@ -585,7 +596,13 @@ defmodule Hypatia.Rules.PrAutomerge do defp pin_line?(line), do: line |> String.replace(~r/^[-+]/, "") |> PinSites.pin_line?() - # Every changed line of a patch as `{line, scalar?, certain?}`. + # Every changed line of a patch as `{line, scalar?, certain?, continued}`. + # + # `continued`: the pins the line holds as the value of a `uses:` key whose + # reference YAML continues below it (`- uses:`, then the reference on the + # line under it, past any blank or comment lines) — read, like + # `PinSites.continuations/1` reads a file, from the line's own side of the + # hunk, so a re-pinned value under an unchanged key is still a pin site. # # `scalar?`: the line sits inside a scalar an earlier line of its own side # opened, as far as the hunk shows — a block scalar (`run: |`, @@ -624,18 +641,37 @@ defmodule Hypatia.Rules.PrAutomerge do hunk = Enum.reject(hunk, &Regex.match?(~r/^[-+]{3}/, &1)) old = side_view(hunk, "-") new = side_view(hunk, "+") + old_cont = side_continuations(old) + new_cont = side_continuations(new) hunk |> Enum.with_index() |> Enum.flat_map(fn {line, i} -> cond do - String.starts_with?(line, "+") -> [line_status(line, i, new)] - String.starts_with?(line, "-") -> [line_status(line, i, old)] - true -> [] + String.starts_with?(line, "+") -> + [line_status(line, i, new, Map.get(new_cont, i, []))] + + String.starts_with?(line, "-") -> + [line_status(line, i, old, Map.get(old_cont, i, []))] + + true -> + [] end end) end + # The continued `uses:` values of one side of a hunk, as + # `%{hunk_index => [pin]}` on the value's line. + defp side_continuations(view) do + texts = for {_, text, state} <- view, do: if(state == :text, do: "", else: text) + index = view |> Enum.map(&elem(&1, 0)) |> List.to_tuple() + + for {_key, value, site} <- PinSites.continuations(texts), into: %{} do + {elem(index, value - 1), + [%{action: site.action, base: action_base(site.action), ref: site.ref}]} + end + end + # One side of a hunk (context lines plus the changed lines of `mark`), in # order, as `{hunk_index, text, state}` — `state` as # `PinSites.line_states/1` reads the side: `:yaml`, `:text` or `:open`. @@ -653,7 +689,7 @@ defmodule Hypatia.Rules.PrAutomerge do Enum.zip_with(entries, states, fn {i, text}, {_, state} -> {i, text, state} end) end - defp line_status(line, i, view) do + defp line_status(line, i, view, continued) do {before, [{_, text, state} | rest]} = Enum.split_while(view, fn {j, _, _} -> j != i end) indent = indent_of(text) @@ -664,7 +700,7 @@ defmodule Hypatia.Rules.PrAutomerge do |> Enum.filter(fn {_, t, _} -> indent_of(t) <= indent end) |> Enum.all?(fn {_, t, _} -> yaml_like?(t) end) - {line, state == :text, certain?} + {line, state == :text, certain?, continued} end defp indent_of(text), do: String.length(text) - String.length(String.trim_leading(text, " ")) @@ -678,21 +714,31 @@ defmodule Hypatia.Rules.PrAutomerge do Regex.match?(~r/^-(?:\s|$)/, t) or Regex.match?(~r/^[{}\[\],]/, t) end - defp added_lines(file), do: for({line, _} <- added_entries(file), do: line) + defp added_pins(file), + do: + Enum.flat_map(added_entries(file), fn {line, _, continued} -> + parse_pins(line) ++ continued + end) + + defp removed_pins(file), + do: + for( + {line, false, _, continued} <- changed_lines(file), + String.starts_with?(line, "-"), + pin <- parse_pins(line) ++ continued, + do: pin + ) # Added lines outside block and continued quoted scalars, each with its # certainty. defp added_entries(file), do: for( - {line, false, certain?} <- changed_lines(file), + {line, false, certain?, continued} <- changed_lines(file), String.starts_with?(line, "+"), - do: {line, certain?} + do: {line, certain?, continued} ) - defp removed_lines(file), - do: for({line, false, _} <- changed_lines(file), String.starts_with?(line, "-"), do: line) - defp patch_of(file), do: Map.get(file, :patch) || Map.get(file, "patch") || "" defp filename_of(file), do: Map.get(file, :filename) || Map.get(file, "filename") || "" diff --git a/lib/rules/workflow_audit.ex b/lib/rules/workflow_audit.ex index 49d21d97..98103b4e 100644 --- a/lib/rules/workflow_audit.ex +++ b/lib/rules/workflow_audit.ex @@ -1205,11 +1205,12 @@ defmodule Hypatia.Rules.WorkflowAudit do lines |> Enum.with_index(1) |> Enum.flat_map(fn {line, number} -> - # `- uses:` with the reference below it is read as the one-line key. - {line, continued?} = + # `- uses:` with the reference below it, or `- uses: *name` naming an + # anchored one, is read as the one-line key it stands for. + {line, resolved} = case Map.fetch(continued, number) do - {:ok, joined} -> {joined, true} - :error -> {line, false} + {:ok, {joined, how}} -> {joined, how} + :error -> {line, nil} end case Regex.run(@toolchain_sha_pin, line) do @@ -1220,14 +1221,15 @@ defmodule Hypatia.Rules.WorkflowAudit do review = cond do kyaml? -> :kyaml - continued? -> :continued + resolved != nil -> resolved wf_node_properties?(line) -> :properties not String.contains?(line, "dtolnay/rust-toolchain@") -> :spelling true -> nil end - in_flow? = Enum.at(depths, number - 1, 0) > 0 - wf026_block_step_finding(filename, lines, line, number, sha, review, in_flow?) + if Enum.at(depths, number - 1, 0) > 0, + do: wf026_flow_line_finding(filename, lines, depths, number, sha, review), + else: wf026_block_step_finding(filename, lines, line, number, sha, review) _ -> # Several flow-style steps may share one physical line @@ -1237,19 +1239,27 @@ defmodule Hypatia.Rules.WorkflowAudit do |> Enum.flat_map(fn [{at, _}, {sha_at, 40}] -> sha = binary_part(line, sha_at, 40) review = if kyaml?, do: :kyaml - wf026_inline_step_finding(filename, lines, line, number, at + 1, sha, review) + + # On a line inside a flow collection opened above, the step's + # ancestry is in the brackets, not the line's indentation. + if Enum.at(depths, number - 1, 0) > 0 do + prefix = binary_part(line, 0, at + 1) + wf026_flow_line_finding(filename, lines, depths, number, sha, review, prefix) + else + wf026_inline_step_finding(filename, lines, line, number, at + 1, sha, review) + end end) end end) end) end - # The block-style path: `uses:` is a key on its own line. A step whose - # opener is `- {`, or which sits inside a flow collection opened on an - # earlier line (`in_flow?`), is a flow-style step: the fixer refuses it. - defp wf026_block_step_finding(filename, lines, line, number, sha, review, in_flow?) do + # The block-style path: `uses:` is a key on its own line, outside any flow + # collection (one inside is read by `wf026_flow_line_finding/7`). A step + # whose opener is `- {` is a flow-style step: the fixer refuses it. + defp wf026_block_step_finding(filename, lines, line, number, sha, review) do {step, key_indent, opener} = wf026_step(lines, number) - flow_item? = in_flow? or Regex.match?(~r/^\s*-\s+\{/, Enum.at(lines, opener - 1)) + flow_item? = Regex.match?(~r/^\s*-\s+\{/, Enum.at(lines, opener - 1)) cond do # `uses:` nested under `env:`, inside a `run: |` script or under @@ -1306,6 +1316,36 @@ defmodule Hypatia.Rules.WorkflowAudit do end end + # A `uses:` key inside a flow collection opened on an earlier line — + # KYAML-like layout, indented or not (`steps: [`, `{`, `uses: ...`) — at + # the line's start, or after `prefix` on it (`{ uses: ...`, `name: x, + # uses: ...`). Its ancestry is the brackets still open before it + # (`wf_flow_context/4`), not its indentation: it is a step's own key when + # the innermost one is a mapping that is an item of the `[` a `steps:` in + # a job opens, and the step is read through that mapping's braces. + defp wf026_flow_line_finding(filename, lines, depths, number, sha, review, prefix \\ "") do + with {:ok, keys, kinds, {brace_line, brace_col}} <- + wf_flow_context(lines, depths, number, prefix), + true <- match?([nil, "steps", job, "jobs" | _] when is_binary(job), keys), + # The `{` is an item of the `[` that `steps:` opens, or of the + # block sequence under it (`- {` on a line of its own). + true <- match?([:map, :seq | _], kinds) or kinds == [:map], + {:ok, interior, _} <- wf_flow_interior(lines, brace_line, brace_col) do + pos = + case wf_flow_entry_value_at(interior, "with") do + nil -> {brace_line, :eol} + at -> wf_interior_pos(interior, at, brace_line, brace_col) + end + + case wf026_with_entry(wf_flow_entries(interior), lines, pos) do + {:toolchain, _} -> [] + verdict -> [wf026_finding(filename, number, sha, verdict, :flow_step, review)] + end + else + _ -> [] + end + end + # Is the flow mapping opening at `open_col` a step? Either it sits in a # sequence position of a block document (below), or the line itself # writes the path to it in flow style. @@ -1410,6 +1450,10 @@ defmodule Hypatia.Rules.WorkflowAudit do " The `uses:` value is continued below its key; the repair script reads a " <> "reference on the key's own line only, so this one routes to review." + _ when review == :alias -> + " The `uses:` value is an alias of an anchored reference; the repair script " <> + "does not rewrite anchored values, so this one routes to review." + _ when review == :spelling -> " The action is not spelled `dtolnay/rust-toolchain`; GitHub resolves the name in " <> "either case, but the repair script matches it as written in lower case, so this " <> @@ -1561,6 +1605,38 @@ defmodule Hypatia.Rules.WorkflowAudit do end end + # For line `number` inside a flow collection — its depth at the start + # (`PinSites.line_depths/1`) above 0 — read up to `prefix`, the part of the + # line before the position judged: `{:ok, keys, kinds, {line, col}}`, the + # keys opening the brackets still open there, innermost first (nil for one + # no key opens, a `{` item of a sequence), then the block ancestors of the + # line that opened the outermost; each bracket's kind (:map for `{`, :seq + # for `[`); and where the innermost one stands. Read from the text since + # that line, comments cut, so a collection spread over lines is one however + # its lines are indented — column 0 included. nil at depth 0 or when that + # text cannot be read. + defp wf_flow_context(lines, depths, number, prefix \\ "") do + start = + if number > 1, + do: Enum.find((number - 1)..1//-1, &(Enum.at(depths, &1 - 1, 0) == 0)) + + with true <- Enum.at(depths, number - 1, 0) > 0, + s when is_integer(s) <- start, + text = + lines + |> Enum.slice(s - 1, number - s) + |> Enum.map_join("\n", &Hypatia.Rules.PinSites.head/1) + |> Kernel.<>("\n" <> prefix), + {:ok, [inner | _] = stack} <- wf_flow_opener_stack(text, byte_size(text)) do + keys = wf_flow_opener_keys(text, stack) ++ (wf_ancestor_keys(lines, s) || []) + kinds = Enum.map(stack, &if(is_integer(&1), do: :map, else: :seq)) + at = if is_integer(inner), do: inner, else: elem(inner, 1) + {:ok, keys, kinds, wf_interior_pos(text, at, s, -1)} + else + _ -> nil + end + end + defp wf_key_name(line) do line |> String.replace(~r/^\s*(?:-\s+)?/, "") @@ -2134,21 +2210,33 @@ defmodule Hypatia.Rules.WorkflowAudit do defp wf_node_properties?(line), do: Regex.match?(~r/^\s*(?:-\s+)?["']?uses["']?\s*:\s*[&!]/, line) - # The `uses:` keys whose reference YAML continues below them (`- uses:`, - # then — past any blank or comment-only lines — the reference on a deeper - # line), as `%{key_line => joined}`: the key's line with the reference - # written after it, which the rules read as they read a one-line key. - # Found on the masked lines (`PinSites.mask_lines/1`), so a `uses:` in a - # `run: |` script or a quoted string continues nothing. + # The `uses:` keys whose reference is not written on their own line, as + # `%{key_line => {joined, how}}`: the key's line with the reference in + # place, which the rules read as they read a one-line key. `how` is + # :continued for a reference YAML continues below the key (`- uses:`, then + # — past any blank or comment-only lines — the reference on a deeper line) + # and :alias for `uses: *name`, the reference the latest `&name` before it + # anchors (`PinSites.alias_sites/1`). Found on the masked lines + # (`PinSites.mask_lines/1`), so a `uses:` in a `run: |` script or a quoted + # string stands for nothing. defp wf_continued_uses(lines) do masked = lines |> Enum.join("\n") |> Hypatia.Rules.PinSites.mask_lines() - for {key, _value, site} <- Hypatia.Rules.PinSites.continuations(masked), into: %{} do - line = Enum.at(lines, key - 1) + continued = + for {key, _value, site} <- Hypatia.Rules.PinSites.continuations(masked), into: %{} do + line = Enum.at(lines, key - 1) - {key, - String.trim_trailing(Hypatia.Rules.PinSites.head(line)) <> - " " <> site.action <> "@" <> site.ref} + {key, + {String.trim_trailing(Hypatia.Rules.PinSites.head(line)) <> + " " <> site.action <> "@" <> site.ref, :continued}} + end + + for %{line: n, at: at, len: len, site: site} <- Hypatia.Rules.PinSites.alias_sites(masked), + not Map.has_key?(continued, n), + into: continued do + line = Enum.at(lines, n - 1) + tail = binary_part(line, at + len, byte_size(line) - at - len) + {n, {binary_part(line, 0, at) <> site.action <> "@" <> site.ref <> tail, :alias}} end end @@ -2446,6 +2534,7 @@ defmodule Hypatia.Rules.WorkflowAudit do Enum.flat_map(workflow_contents, fn {filename, content} -> lines = String.split(content, "\n") continued = wf_continued_uses(lines) + depths = Hypatia.Rules.PinSites.line_depths(content) Enum.flat_map(@broken_reusable_revisions, fn rev -> # Anchored to a `uses:` key at the start of a line (quoted or not) that @@ -2467,11 +2556,12 @@ defmodule Hypatia.Rules.WorkflowAudit do lines |> Enum.with_index(1) |> Enum.flat_map(fn {line, number} -> - # A job's `uses:` with the reference below it is the one-line key. - {line, continued?} = + # A job's `uses:` with the reference below it, or naming an anchored + # one (`uses: *call`), is the one-line key it stands for. + {line, resolved} = case Map.fetch(continued, number) do - {:ok, joined} -> {joined, true} - :error -> {line, false} + {:ok, {joined, how}} -> {joined, how} + :error -> {line, nil} end # Each call with the layout the fleet fixer would meet: it rewrites @@ -2481,21 +2571,22 @@ defmodule Hypatia.Rules.WorkflowAudit do [_full, sha] -> review = cond do - continued? -> :continued + resolved != nil -> resolved wf_node_properties?(line) -> :properties true -> nil end - if match?([_job, "jobs" | _], wf_ancestor_keys(lines, number)) and - not wf_in_open_quote?(lines, wf_parent_line(lines, number), number), - do: [{sha, review}], - else: [] + case wf027_job_key(lines, depths, number) do + :block -> [{sha, review}] + :flow -> [{sha, review || :flow}] + nil -> [] + end nil -> inline_re |> Regex.scan(line, return: :index) |> Enum.filter(fn [{at, _} | _] -> - wf027_inline_job?(lines, line, number, at + 1) + wf027_inline_job?(lines, depths, line, number, at + 1) end) |> Enum.map(fn [_, {sha_at, 40}] -> {binary_part(line, sha_at, 40), :flow} end) end @@ -2512,6 +2603,34 @@ defmodule Hypatia.Rules.WorkflowAudit do end) end + # Is the `uses:` that starts line `number` a key of a job under `jobs:`? + # :block when its indentation says so, outside any quoted scalar the job + # opened above it — the reading the fleet fixer makes. A line inside a flow + # collection opened on an earlier line is judged by the brackets still open + # before it (`wf_flow_context/4`), however it is indented: the innermost + # must be the mapping a job key opens in `jobs`. Such a call is :block too + # when its indentation agrees (KYAML-style `rust: {` / ` uses: ...`, which + # the fixer rewrites), and :flow when only the brackets say so (`jobs: {`, + # `rust: {`, `uses: ...` all in column 0, which the fixer cannot place). + # nil when it is no job's key. + defp wf027_job_key(lines, depths, number) do + block? = + match?([_job, "jobs" | _], wf_ancestor_keys(lines, number)) and + not wf_in_open_quote?(lines, wf_parent_line(lines, number), number) + + cond do + Enum.at(depths, number - 1, 0) == 0 -> if block?, do: :block + not wf027_flow_job?(wf_flow_context(lines, depths, number)) -> nil + block? -> :block + true -> :flow + end + end + + # Is the innermost bracket of a `wf_flow_context/4` the mapping a job key + # opens in `jobs`? + defp wf027_flow_job?({:ok, [job, "jobs" | _], [:map | _], _}) when is_binary(job), do: true + defp wf027_flow_job?(_), do: false + # The reusable's identity as a regex source. GitHub resolves the owner and # the repository name in either case (`Hyperpolymath/Standards` is # `hyperpolymath/standards`); the workflow path inside the repository is a @@ -2532,7 +2651,15 @@ defmodule Hypatia.Rules.WorkflowAudit do # must read a job key, then `jobs`. Each key is the last entry before its # brace, split quote-aware, so a sibling's `name: "CI, build"` separates # nothing. - defp wf027_inline_job?(lines, line, number, uses_at) do + defp wf027_inline_job?(lines, depths, line, number, uses_at) do + # On a line inside a flow collection opened above, the brackets before + # `uses` are read from where the collection began. + if Enum.at(depths, number - 1, 0) > 0, + do: wf027_flow_job?(wf_flow_context(lines, depths, number, binary_part(line, 0, uses_at))), + else: wf027_line_job?(lines, line, number, uses_at) + end + + defp wf027_line_job?(lines, line, number, uses_at) do case {wf_flow_opener_stack(line, uses_at), wf_ancestor_keys(lines, number)} do {{:ok, [open | _] = stack}, ancestors} when is_integer(open) and is_list(ancestors) -> keys = wf_flow_opener_keys(line, stack) ++ ancestors @@ -2592,6 +2719,11 @@ defmodule Hypatia.Rules.WorkflowAudit do " The `uses:` value is continued below its key; the repair script rewrites " <> "a reference on the key's own line only, so this one routes to review." + defp wf027_review_note(:alias), + do: + " The `uses:` value is an alias of an anchored reference; the repair script " <> + "does not rewrite anchored values, so this one routes to review." + defp wf027_review_note(:properties), do: " The `uses:` value carries a YAML anchor or tag; the repair script rewrites " <> diff --git a/test/rules/pin_integrity_test.exs b/test/rules/pin_integrity_test.exs index 224bf084..1f8b6cd6 100644 --- a/test/rules/pin_integrity_test.exs +++ b/test/rules/pin_integrity_test.exs @@ -297,6 +297,41 @@ defmodule Hypatia.Rules.PinIntegrityTest do assert List.last(states) == {" uses: e/f@z", :open} end + test "a uses: alias runs the reference its anchor holds; the site is the anchor's value" do + content = + "env:\n BAD: &bad github/codeql-action/init@#{@poison} # v4.38.0\n" <> + "jobs:\n a:\n steps:\n - uses: *bad\n - { name: y, uses: *bad }\n" + + # One site however many aliases name it: the anchor's line is where the + # reference stands, and where a repair rewrites it. + assert [%{line: 2, detail: %{ref: @poison, action: "github/codeql-action/init"}}] = + PI.pi001_denylisted_pin(content, @policy) + + assert {:ok, new_content, [_]} = PI.substitute_denylisted_pins(content, @policy) + assert new_content =~ " BAD: &bad github/codeql-action/init@#{@good} # v4.38.0\n" + assert new_content =~ " - uses: *bad\n" + + # Controls: an alias names the latest anchor before it, so a poisoned + # `&bad` defined after it is not what runs; an anchor whose value is no + # action reference names no site; nor does an alias that is not a + # `uses:` value. + later = + "env:\n OK: &bad github/codeql-action/init@#{@good}\n" <> + "jobs:\n a:\n steps:\n - uses: *bad\n" <> + "x:\n BAD: &bad github/codeql-action/init@#{@poison}\n" + + assert PI.pi001_denylisted_pin(later, @policy) == [] + + not_action = "x: &bad stable\njobs:\n a:\n steps:\n - uses: *bad\n" + assert PI.pi001_denylisted_pin(not_action, @policy) == [] + + not_uses = + "env:\n BAD: &bad github/codeql-action/init@#{@poison}\n ALSO: *bad\n" <> + " - name: \"uses: *bad\"\n" + + assert PI.pi001_denylisted_pin(not_uses, @policy) == [] + end + test "a KYAML flow step on its own line is still a site" do content = "steps: [\n { name: \"x\", run: \"echo 'a\" },\n" <> diff --git a/test/rules/pr_automerge_test.exs b/test/rules/pr_automerge_test.exs index 3f9630ae..86752c65 100644 --- a/test/rules/pr_automerge_test.exs +++ b/test/rules/pr_automerge_test.exs @@ -425,6 +425,25 @@ defmodule Hypatia.Rules.PrAutomergeTest do assert PA.poison_sites([file(".github/workflows/codeql.yml", patch)], @policy) == [] end + test "a re-pinned value under an unchanged `uses:` key is the site it continues" do + # The key is context; only the reference on the line below it changes. + patch = """ + @@ -1,3 +1,3 @@ + - uses: + # pinned + - github/codeql-action/init@#{@good} + + github/codeql-action/init@#{@poison} + """ + + f = file(".github/workflows/codeql.yml", patch) + assert [%{denylist_id: "PIN-001"}] = PA.poison_sites([f], @policy) + assert PA.pin_lines_only?(f) + + # Control: the same value line under a key that is not `uses:` is data. + data = String.replace(patch, " - uses:", " - run:") + assert PA.poison_sites([file(".github/workflows/codeql.yml", data)], @policy) == [] + end + test "every site of a flow-sequence line is inspected, not only the first" do # The first step is an ordinary resolvable bump; the second is the # poison. Judging the line by its first pin armed automerge. diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs index 29d69a6e..fe1808ac 100644 --- a/test/rules/workflow_audit_pins_test.exs +++ b/test/rules/workflow_audit_pins_test.exs @@ -793,6 +793,120 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) end + test "a uses: alias of an anchored toolchain reference is the step's pin; routed to review" do + wf = """ + x: &rust dtolnay/rust-toolchain@#{@toolchain_sha} + jobs: + check: + steps: + - uses: *rust + - run: cargo test + """ + + assert [%{line: 5, auto_fixable: false, fix_script: nil} = f] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + + assert f.current_sha == @toolchain_sha + assert f.reason =~ "alias of an anchored reference" + + # Controls: with its input the aliased step is configured; an alias + # of another action, or of an anchor defined after it, is not this pin. + with_input = + String.replace(wf, " - run:", " with: { toolchain: stable }\n - run:") + + assert [] == + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => with_input}) + + other = String.replace(wf, "dtolnay/rust-toolchain", "actions/checkout") + assert [] == WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => other}) + + later = """ + jobs: + check: + steps: + - uses: *rust + x: &rust dtolnay/rust-toolchain@#{@toolchain_sha} + """ + + assert [] == WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => later}) + end + + test "a flow step written without indentation, opener by opener, is still a step" do + # Flow content may start in column 0 on every line: indentation says + # nothing, so the ancestry is the openers the lines above left open. + wf = """ + jobs: { + build: { + steps: [ + { + uses: dtolnay/rust-toolchain@#{@toolchain_sha} + } + ] + } + } + """ + + assert [%{line: 5, auto_fixable: false, fix_script: nil} = f] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + + assert f.reason =~ "flow style" + + # Controls: the step's own input configures it, and the same layout + # under `env:` or a job's non-step key is data, not a step. + with_input = + String.replace( + wf, + "rust-toolchain@#{@toolchain_sha}\n", + "rust-toolchain@#{@toolchain_sha},\nwith: { toolchain: stable }\n" + ) + + assert [] == + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => with_input}) + + env = String.replace(wf, "jobs: {", "env: {") + assert [] == WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => env}) + + fixture = String.replace(wf, "steps: [", "fixture: [") + + assert [] == + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => fixture}) + + # The step's `uses:` after its `{`, or after a sibling entry, on a + # column-0 line is the same step. + for {step, line} <- [ + {"{ uses: dtolnay/rust-toolchain@#{@toolchain_sha} }\n", 4}, + {"{\nname: x, uses: dtolnay/rust-toolchain@#{@toolchain_sha}\n}\n", 5} + ] do + inline = "jobs: {\nbuild: {\nsteps: [\n" <> step <> "]\n}\n}\n" + + assert [%{line: ^line, auto_fixable: false}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => inline}) + + configured = + String.replace( + inline, + "@#{@toolchain_sha}", + "@#{@toolchain_sha}, with: { toolchain: stable }" + ) + + assert [] == + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{ + "rust.yml" => configured + }) + end + + shared = + "jobs: {\nbuild: {\nsteps: [ { uses: dtolnay/rust-toolchain@#{@toolchain_sha} } ]\n}\n}\n" + + assert [%{line: 3, auto_fixable: false}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => shared}) + + assert [] == + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{ + "rust.yml" => String.replace(shared, "steps: [", "fixture: [") + }) + end + test "a flow step's alias names the anchor before it, not one later on its line" do # `*inputs` is the empty mapping defined first; the configured # `&inputs` after it on the same line is not what it names. @@ -1005,6 +1119,56 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => plain}) end + test "a job's uses: alias of an anchored reusable reference is the call; routed to review" do + wf = + "x: &call #{@rust_ci}@#{@broken_standards}\n" <> + "jobs:\n rust-ci:\n uses: *call\n" + + assert [%{line: 4, auto_fixable: false, fix_script: nil} = f] = + WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => wf}) + + assert f.current_sha == @broken_standards + assert f.reason =~ "alias of an anchored reference" + + # Control: an alias of the replacement revision is silent. + good = String.replace(wf, @broken_standards, @good_standards) + assert WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => good}) == [] + end + + test "a flow-style job written without indentation is still a job; routed to review" do + # Every line in column 0: the brackets, not the indentation, say the + # `uses:` is a job's key. The fleet fixer places a key by indentation + # and cannot place these, so none is handed to it. + own_line = "jobs: {\nrust: {\nuses: #{@rust_ci}@#{@broken_standards}\n}\n}\n" + inline = "jobs: {\nrust: { uses: #{@rust_ci}@#{@broken_standards} }\n}\n" + + sibling = + "jobs: {\na: { runs-on: x }, rust: { uses: #{@rust_ci}@#{@broken_standards} }\n}\n" + + for {wf, line} <- [{own_line, 3}, {inline, 2}, {sibling, 2}] do + assert [%{line: ^line, current_sha: @broken_standards, auto_fixable: false} = f] = + WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => wf}) + + assert f.reason =~ "flow mapping" + end + + # Controls: indented as its nesting reads, the own-line key is one the + # fixer rewrites; under `env:`, or inside a job's `with: { }`, the + # same layout is data, not a call. + indented = "jobs: {\n rust: {\n uses: #{@rust_ci}@#{@broken_standards}\n }\n}\n" + + assert [%{auto_fixable: true, fix_script: "fix-stale-reusable-pin.sh"}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => indented}) + + for wf <- [own_line, inline], + data <- [ + String.replace(wf, "jobs: {", "env: {"), + String.replace(wf, "rust: {", "rust: {\nwith: {") <> "}\n" + ] do + assert WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => data}) == [] + end + end + test "a flow-style job's uses: continued on the next line is the call" do wf = "jobs: { rust: { uses:\n #{@rust_ci}@#{@broken_standards} } }\n" From d2a99d8ffe4a13431a8f9f6500672e2c198324d0 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 07:12:46 +0000 Subject: [PATCH 41/47] fix(pins): flow ancestry across quoted lines; node properties before step mappings; continued anchor values; scalar toolchain aliases - PinSites.structure_lines/1: each line as its structure reads (quoted scalars blanked across lines, comments cut, a quote the line leaves open blanked too); flow_prefixes/1 gives a line inside a multi-line flow collection the structure since it opened, and sites/2 judges a `uses:` there against the brackets still open. PI001 reads every line so: a step's `uses:` after a sibling (`name: "hello` / `world", uses: ...`) is a site, one in an input's mapping is not. - WF026/WF027 build their flow context from the same structure lines, so a quoted sibling continued past a `# world",` line no longer hides the step or the call; a `uses:` blanked as string text is no key. - Node properties before a bracket name the collection, not its key: `steps: [ &step {`, `- &step {` and `rust: &job {` are a step and a job. - An anchor with nothing after it on its line names the scalar continued on the next deeper line; its alias's site is there. - `toolchain: *name` is the scalar its anchor holds: empty or null is no input; an anchor this reader cannot read routes the finding to review. - A key or dash followed only by node properties (`- run: &cmd`) leaves its value pending, so the quote below it opens the string; mirrored in the sweep's mask. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- lib/rules/pin_integrity.ex | 17 +- lib/rules/pin_sites.ex | 133 ++++++++++++--- lib/rules/workflow_audit.ex | 206 +++++++++++++++++------- scripts/sweeps/estate-pin-integrity.sh | 4 +- test/rules/pin_integrity_test.exs | 78 +++++++++ test/rules/workflow_audit_pins_test.exs | 85 ++++++++++ 6 files changed, 440 insertions(+), 83 deletions(-) diff --git a/lib/rules/pin_integrity.ex b/lib/rules/pin_integrity.ex index c8a64643..d68592cb 100644 --- a/lib/rules/pin_integrity.ex +++ b/lib/rules/pin_integrity.ex @@ -66,7 +66,10 @@ defmodule Hypatia.Rules.PinIntegrity do comment or script text is not one, nor is one on a line of a `run: |` block or of a quoted scalar continued from an earlier line: the file is read through `PinSites.mask_lines/1`, which carries that state from line - to line. A `uses:` whose reference YAML continues below it (`- uses:` + to line. A line inside a flow collection opened on an earlier line is + judged against the brackets still open (`PinSites.flow_prefixes/1`), so a + step's `uses:` after a sibling (`name: "hello` then `world", uses: a/b@x + }]`) is a site and one in an input's mapping is not. A `uses:` whose reference YAML continues below it (`- uses:` then ` a/b@x`, blank and comment-only lines allowed between) is a site on the value's line (`PinSites.continuations/1`), and a `uses: *name` is a site on the line of the anchor it names, where its reference stands @@ -82,7 +85,7 @@ defmodule Hypatia.Rules.PinIntegrity do @spec pin_sites(String.t()) :: [map()] def pin_sites(content) when is_binary(content) do masked = PinSites.mask_lines(content) - numbered = Enum.with_index(masked, 1) + numbered = masked |> Enum.zip(PinSites.flow_prefixes(content)) |> Enum.with_index(1) # A `uses:` whose reference continues below it — past any blank or # comment-only lines — is a site on the value's line. @@ -104,7 +107,7 @@ defmodule Hypatia.Rules.PinIntegrity do end numbered - |> Enum.flat_map(fn {line, number} -> line_sites(line, number) end) + |> Enum.flat_map(fn {{line, prefix}, number} -> line_sites(line, number, prefix) end) |> Kernel.++(continued) |> Kernel.++(aliased) |> Enum.uniq_by(&{&1.line, &1.ref_at}) @@ -113,8 +116,8 @@ defmodule Hypatia.Rules.PinIntegrity do def pin_sites(_), do: [] - defp line_sites(line, number) do - sites = PinSites.sites(line) + defp line_sites(line, number, flow_prefix) do + sites = PinSites.sites(line, flow_prefix) comment = if length(sites) == 1, do: line |> PinSites.comment() |> String.trim(), else: "" for site <- sites, do: site_entry(site, number, comment) @@ -694,8 +697,8 @@ defmodule Hypatia.Rules.PinIntegrity do """ @spec pin_only_edit?([String.t()], [String.t()], map()) :: boolean() def pin_only_edit?(removed, added, policy) do - removed_sites = Enum.flat_map(removed, &line_sites(&1, 0)) - added_sites = Enum.flat_map(added, &line_sites(&1, 0)) + removed_sites = Enum.flat_map(removed, &line_sites(&1, 0, "")) + added_sites = Enum.flat_map(added, &line_sites(&1, 0, "")) pinned_removed = removed_sites |> Enum.map(& &1.ref) |> Enum.sort() pinned_added = added_sites |> Enum.map(& &1.ref) |> Enum.sort() diff --git a/lib/rules/pin_sites.ex b/lib/rules/pin_sites.ex index 51922697..fe6a0c09 100644 --- a/lib/rules/pin_sites.ex +++ b/lib/rules/pin_sites.ex @@ -36,13 +36,20 @@ defmodule Hypatia.Rules.PinSites do @doc """ The structural sites of `line`, in order, as `%{action: a, ref: r, action_at: byte, ref_at: byte}`. + + `flow_prefix` is, for a line inside a flow collection opened on an earlier + line, the structure of the lines since it opened (`flow_prefixes/1`): a + `uses:` there is a site when the brackets still open before it make it a + step's or a job's own key, however the line itself begins (`name: + "hello` then `world", uses: a/b@x }]`). Without it the line is judged on + its own. """ - @spec sites(String.t()) :: [map()] - def sites(line) when is_binary(line) do + @spec sites(String.t(), String.t()) :: [map()] + def sites(line, flow_prefix \\ "") when is_binary(line) and is_binary(flow_prefix) do masked = mask(line) masked - |> scan() + |> scan(flow_prefix) |> Enum.map(fn [_full, {a_at, a_len}, {r_at, r_len}] -> %{ action: binary_part(masked, a_at, a_len), @@ -58,13 +65,47 @@ defmodule Hypatia.Rules.PinSites do `[full, action, ref]` `{byte, length}` pairs (`Regex.scan/3` with `return: :index`). """ - @spec scan(String.t()) :: [[{non_neg_integer(), non_neg_integer()}]] - def scan(masked) when is_binary(masked) do + @spec scan(String.t(), String.t()) :: [[{non_neg_integer(), non_neg_integer()}]] + def scan(masked, flow_prefix \\ "") when is_binary(masked) do @site_re |> Regex.scan(masked, return: :index, capture: :all) |> Enum.filter(fn [_full, {action_at, _} | _] -> - structural?(binary_part(masked, 0, action_at)) + prefix = binary_part(masked, 0, action_at) + + if flow_prefix == "", + do: structural?(prefix), + else: structural_in_flow?(flow_prefix <> prefix) + end) + end + + @doc """ + The lines of a whole file as their structure reads: `mask_lines/1`, each + then masked (`mask/1`, so cut at its comment), with a quoted scalar the + line leaves open blanked from its opening quote. Joined, they read as one + text of keys, brackets and commas whatever scalars span them: a `#` that + continues a quoted scalar (`name: "hello` then `# world",`) cuts nothing, + and no quote is left open. + """ + @spec structure_lines(String.t()) :: [String.t()] + def structure_lines(content) when is_binary(content), + do: content |> mask_lines() |> Enum.map(&do_mask(&1, "", true)) + + @doc """ + For each line of `content`, the structure of the lines since the flow + collection it sits in was opened (`structure_lines/1`, joined, ending in a + newline), or `""` for a line at flow depth 0 (`line_depths/1`): what + `sites/2` reads a line inside a multi-line flow collection against. + """ + @spec flow_prefixes(String.t()) :: [String.t()] + def flow_prefixes(content) when is_binary(content) do + content + |> structure_lines() + |> Enum.zip(line_depths(content)) + |> Enum.map_reduce("", fn {text, depth}, carry -> + prefix = if depth > 0, do: carry <> "\n", else: "" + {prefix, if(depth > 0, do: carry <> "\n" <> text, else: text)} end) + |> elem(0) end @doc """ @@ -184,15 +225,48 @@ defmodule Hypatia.Rules.PinSites do @spec alias_sites([String.t()]) :: [map()] def alias_sites(lines) when is_list(lines) do numbered = Enum.with_index(lines, 1) - defs = for {line, n} <- numbered, d <- anchor_defs(line), do: Map.put(d, :line, n) + + defs = + for {line, n} <- numbered, d <- anchor_defs(line) do + d |> Map.put(:line, n) |> continued_anchor(line, Enum.drop(numbered, n)) + end for {line, n} <- numbered, {name, at, len} <- uses_aliases(line), - %{site: %{} = site, line: anchor_line} <- [latest_anchor(defs, name, n, at)] do - %{line: n, at: at, len: len, anchor_line: anchor_line, site: site} + %{site: %{} = site} = d <- [latest_anchor(defs, name, n, at)] do + %{line: n, at: at, len: len, anchor_line: Map.get(d, :site_line, d.line), site: site} end end + # An anchor with nothing after it on its line but a tag or a comment + # (`BAD: &bad`) names the scalar YAML continues on the next line with + # content, deeper than the key (` github/codeql-action/init@`): its + # site is there, `:site_line` that line. + defp continued_anchor(%{site: nil, rest: rest} = d, line, below) do + with true <- Regex.match?(~r/^\s*(?:![^\s]*\s*)*(?:#.*)?$/, rest), + {next, n} <- Enum.find(below, fn {l, _} -> String.trim(head(l)) != "" end), + true <- indent_of(next) > indent_of(line), + [_, {a_at, a_len}, {r_at, r_len}] <- + Regex.run( + ~r/^\s*(?:![^\s]*\s+)*["']?([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)["']?\s*(?:#.*)?$/, + next, + return: :index + ) do + site = %{ + action: binary_part(next, a_at, a_len), + ref: binary_part(next, r_at, r_len), + action_at: a_at, + ref_at: r_at + } + + %{d | site: site} |> Map.put(:site_line, n) + else + _ -> d + end + end + + defp continued_anchor(d, _line, _below), do: d + # The anchors `line` defines where a node starts (outside quotes and the # comment), each `%{name, at, site}`: the site its scalar value is, or nil. defp anchor_defs(line) do @@ -228,7 +302,7 @@ defmodule Hypatia.Rules.PinSites do nil end - %{name: binary_part(masked, n_at, n_len), at: at, site: site} + %{name: binary_part(masked, n_at, n_len), at: at, site: site, rest: rest} end) end @@ -408,8 +482,10 @@ defmodule Hypatia.Rules.PinSites do %{st | block: block_parent(seen), pending: false} true -> + # A key or dash followed only by node properties (`- run: &cmd`, + # `key: !!str`) leaves its value pending as surely as a bare one. pending = - Regex.match?(~r/(?::|^\s*-)$/, trimmed) or + Regex.match?(~r/(?::|^\s*-)(?:\s+[&!]\S*)*$/, trimmed) or (st.depth > 0 and Regex.match?(~r/[\[{,]$/, trimmed)) %{st | pending: pending} @@ -485,6 +561,15 @@ defmodule Hypatia.Rules.PinSites do end end + # A `uses:` read against the flow collections the lines above opened + # (`text` is their structure and the line up to the action): the key must + # stand at an entry position — after `{`, `,` or at the line's start — + # and the brackets still open must make it a step's or a job's own. + defp structural_in_flow?(text) do + Regex.match?(~r/(?:[{,]|\n)\s*(?:-\s+)?["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*["']?$/, text) and + text |> flow_stack() |> structural_flow_stack?() + end + # The brackets still open in `text`, innermost first, each with what opens # it: the key before it (`steps: [`), :none (nothing but indentation or a # list dash), or :text (anything else — `echo {` is a script, not YAML). @@ -500,7 +585,11 @@ defmodule Hypatia.Rules.PinSites do |> elem(0) end + # Node properties before a bracket name the collection, not its key: + # `steps: [ &step {` opens an anonymous step, `rust: &job {` a job. defp opener_key(seg) do + seg = Regex.replace(~r/(?:[&!]\S*\s+)+$/, seg, "") + cond do Regex.match?(~r/^\s*(?:-\s*)?$/, seg) -> :none m = Regex.run(~r/^\s*(?:-\s+)?["']?([\w.-]+)["']?\s*:\s*$/, seg) -> Enum.at(m, 1) @@ -518,9 +607,13 @@ defmodule Hypatia.Rules.PinSites do end end - defp do_mask("", out), do: out + # `open?` blanks a quoted scalar the line leaves open, its quote too + # (`structure_lines/1`); `mask/1` keeps that quote. + defp do_mask(line, out, open? \\ false) + + defp do_mask("", out, _open?), do: out - defp do_mask(<>, out) when q in [?", ?'] do + defp do_mask(<>, out, open?) when q in [?", ?'] do {body, close, tail} = take_quoted(rest, q, "") kept = @@ -528,17 +621,21 @@ defmodule Hypatia.Rules.PinSites do do: body, else: String.duplicate("x", byte_size(body)) - do_mask(tail, out <> <> <> kept <> close) + if open? and close == "", + do: out <> String.duplicate("x", 1 + byte_size(body)), + else: do_mask(tail, out <> <> <> kept <> close, open?) end - defp do_mask(<>, out) do + defp do_mask(<>, out, open?) do if out == "" or String.ends_with?(out, [" ", "\t"]), do: out, - else: do_mask(rest, out <> "#") + else: do_mask(rest, out <> "#", open?) end - defp do_mask(<>, out), do: do_mask(rest, out <> <>) - defp do_mask(<>, out), do: do_mask(rest, out <> <>) + defp do_mask(<>, out, open?), + do: do_mask(rest, out <> <>, open?) + + defp do_mask(<>, out, open?), do: do_mask(rest, out <> <>, open?) # The body of a quoted scalar up to its closing quote; a backslash escapes # the next character inside double quotes. An unterminated scalar runs to diff --git a/lib/rules/workflow_audit.ex b/lib/rules/workflow_audit.ex index 98103b4e..53fa8942 100644 --- a/lib/rules/workflow_audit.ex +++ b/lib/rules/workflow_audit.ex @@ -1200,7 +1200,7 @@ defmodule Hypatia.Rules.WorkflowAudit do continued = wf_continued_uses(lines) # A step inside an open flow collection is a flow-style step even when # its `{` stands alone on a line above (`steps: [`, `{`, `uses: ...`). - depths = Hypatia.Rules.PinSites.line_depths(normalised) + flow = wf_flow_lines(normalised) lines |> Enum.with_index(1) @@ -1227,8 +1227,8 @@ defmodule Hypatia.Rules.WorkflowAudit do true -> nil end - if Enum.at(depths, number - 1, 0) > 0, - do: wf026_flow_line_finding(filename, lines, depths, number, sha, review), + if wf_in_flow?(flow, number), + do: wf026_flow_line_finding(filename, lines, flow, number, sha, review), else: wf026_block_step_finding(filename, lines, line, number, sha, review) _ -> @@ -1242,9 +1242,8 @@ defmodule Hypatia.Rules.WorkflowAudit do # On a line inside a flow collection opened above, the step's # ancestry is in the brackets, not the line's indentation. - if Enum.at(depths, number - 1, 0) > 0 do - prefix = binary_part(line, 0, at + 1) - wf026_flow_line_finding(filename, lines, depths, number, sha, review, prefix) + if wf_in_flow?(flow, number) do + wf026_flow_line_finding(filename, lines, flow, number, sha, review, at + 1) else wf026_inline_step_finding(filename, lines, line, number, at + 1, sha, review) end @@ -1318,14 +1317,13 @@ defmodule Hypatia.Rules.WorkflowAudit do # A `uses:` key inside a flow collection opened on an earlier line — # KYAML-like layout, indented or not (`steps: [`, `{`, `uses: ...`) — at - # the line's start, or after `prefix` on it (`{ uses: ...`, `name: x, + # the line's start, or at byte `at` of it (`{ uses: ...`, `name: x, # uses: ...`). Its ancestry is the brackets still open before it # (`wf_flow_context/4`), not its indentation: it is a step's own key when # the innermost one is a mapping that is an item of the `[` a `steps:` in # a job opens, and the step is read through that mapping's braces. - defp wf026_flow_line_finding(filename, lines, depths, number, sha, review, prefix \\ "") do - with {:ok, keys, kinds, {brace_line, brace_col}} <- - wf_flow_context(lines, depths, number, prefix), + defp wf026_flow_line_finding(filename, lines, flow, number, sha, review, at \\ 0) do + with {:ok, keys, kinds, {brace_line, brace_col}} <- wf_flow_context(lines, flow, number, at), true <- match?([nil, "steps", job, "jobs" | _] when is_binary(job), keys), # The `{` is an item of the `[` that `steps:` opens, or of the # block sequence under it (`- {` on a line of its own). @@ -1392,8 +1390,15 @@ defmodule Hypatia.Rules.WorkflowAudit do # Anything else (`run: "echo { uses: ... }"`) is a scalar carrying braces. # An earlier sibling on the same line (`[ { uses: a }, { uses: b } ]`) is # a closed mapping before this one; it is removed before the test. + # Node properties may stand before the mapping or the sequence it sits in + # (`- &step {`, `steps: &s [ &step {`): they name the node, not a key. defp wf026_inline_item?(line, open_col) do - prefix = line |> binary_part(0, open_col) |> wf_drop_closed_mappings() + prefix = + line + |> binary_part(0, open_col) + |> wf_drop_closed_mappings() + |> String.replace(~r/(^|[\s\[{,:-])[&!]\S*(?=\s)/, "\\1") + Regex.match?(~r/^\s*(?:-\s*)?(?:["']?steps["']?\s*:\s*)?[\[,\s]*$/, prefix) end @@ -1430,6 +1435,11 @@ defmodule Hypatia.Rules.WorkflowAudit do " Its `with:` is the alias `*#{name}` and no `&#{name}` anchor is defined before " <> "it in the file, so the inputs cannot be read; the alias must be resolved by hand." + {{:unreadable, :toolchain_alias, name}, _} -> + " Its `toolchain:` input is the alias `*#{name}`, and no `&#{name}` scalar this " <> + "reader can read is defined before it, so whether the action receives a channel " <> + "cannot be told; the alias must be checked by hand." + {_, :flow_step} -> " The step is written in flow style; the repair script rewrites block-style " <> "steps only, so this one routes to review." @@ -1605,28 +1615,53 @@ defmodule Hypatia.Rules.WorkflowAudit do end end + # The flow depth and the structure of each line of a file + # (`PinSites.line_depths/1`, `PinSites.structure_lines/1`), read once. + defp wf_flow_lines(content) do + %{ + depths: content |> Hypatia.Rules.PinSites.line_depths() |> List.to_tuple(), + text: content |> Hypatia.Rules.PinSites.structure_lines() |> List.to_tuple() + } + end + + defp wf_in_flow?(flow, number), do: wf_flow_at(flow.depths, number, 0) > 0 + + defp wf_flow_at(tuple, number, default) do + if number >= 1 and number <= tuple_size(tuple), do: elem(tuple, number - 1), else: default + end + # For line `number` inside a flow collection — its depth at the start - # (`PinSites.line_depths/1`) above 0 — read up to `prefix`, the part of the - # line before the position judged: `{:ok, keys, kinds, {line, col}}`, the - # keys opening the brackets still open there, innermost first (nil for one - # no key opens, a `{` item of a sequence), then the block ancestors of the - # line that opened the outermost; each bracket's kind (:map for `{`, :seq - # for `[`); and where the innermost one stands. Read from the text since - # that line, comments cut, so a collection spread over lines is one however - # its lines are indented — column 0 included. nil at depth 0 or when that - # text cannot be read. - defp wf_flow_context(lines, depths, number, prefix \\ "") do + # (`PinSites.line_depths/1`) above 0 — read up to byte `at` of it, the + # position judged: `{:ok, keys, kinds, {line, col}}`, the keys opening the + # brackets still open there, innermost first (nil for one no key opens, a + # `{` item of a sequence), then the block ancestors of the line that + # opened the outermost; each bracket's kind (:map for `{`, :seq for `[`); + # and where the innermost one stands. Read from the structure of the lines + # since that one (`PinSites.structure_lines/1`: comments cut and quoted + # scalars blanked across lines, so a `# world"` continuing a string is + # text), so a collection spread over lines is one however its lines are + # indented — column 0 included. nil at depth 0, past the line's comment, + # or when that text cannot be read. + defp wf_flow_context(lines, flow, number, at \\ 0) do start = if number > 1, - do: Enum.find((number - 1)..1//-1, &(Enum.at(depths, &1 - 1, 0) == 0)) + do: Enum.find((number - 1)..1//-1, &(wf_flow_at(flow.depths, &1, 0) == 0)) - with true <- Enum.at(depths, number - 1, 0) > 0, + here = wf_flow_at(flow.text, number, "") + + with true <- wf_in_flow?(flow, number), s when is_integer(s) <- start, + true <- at <= byte_size(here), + # The key itself must be structure: a `uses:` the structure view + # blanks is inside a quoted scalar continued from above. + true <- + Regex.match?( + ~r/^\s*(?:-\s+)?["']?uses["']?\s*:/, + binary_part(here, at, byte_size(here) - at) + ), text = - lines - |> Enum.slice(s - 1, number - s) - |> Enum.map_join("\n", &Hypatia.Rules.PinSites.head/1) - |> Kernel.<>("\n" <> prefix), + Enum.map_join(s..(number - 1), "\n", &wf_flow_at(flow.text, &1, "")) <> + "\n" <> binary_part(here, 0, at), {:ok, [inner | _] = stack} <- wf_flow_opener_stack(text, byte_size(text)) do keys = wf_flow_opener_keys(text, stack) ++ (wf_ancestor_keys(lines, s) || []) kinds = Enum.map(stack, &if(is_integer(&1), do: :map, else: :seq)) @@ -1787,7 +1822,10 @@ defmodule Hypatia.Rules.WorkflowAudit do defp wf026_toolchain_in(entries, lines, seen, pos) do case Enum.find(entries, fn {k, _} -> k == "toolchain" end) do {_, v} when is_binary(v) -> - if wf026_null_value?(v), do: :null, else: {:set, :own} + case Regex.run(~r/^\s*\*([\w.-]+)\s*$/, v) do + [_, name] -> wf026_scalar_alias(name, lines, pos) + nil -> if wf026_null_value?(v), do: :null, else: {:set, :own} + end _ -> entries @@ -1844,6 +1882,18 @@ defmodule Hypatia.Rules.WorkflowAudit do end end + # `toolchain: *name` is the scalar the latest `&name` before it anchors: + # null or empty there is no input, exactly as if written in place + # (`x: &empty ""`). An anchor this reader cannot find, or one that names a + # mapping, a sequence or a value continued on the lines below, leaves the + # input unreadable, and the finding routes to review. + defp wf026_scalar_alias(name, lines, pos) do + case wf_anchor_scalar(lines, name, pos) do + {:ok, value} -> if wf026_null_value?(value), do: :null, else: {:set, :own} + :error -> {:unreadable, :toolchain_alias, name} + end + end + # YAML's null: nothing, `~`, or `null` in the three spellings the core # schema admits, unquoted. `"null"` is the string null, an input the # action receives; `""` and `''` are the empty string, which it does not. @@ -1884,7 +1934,64 @@ defmodule Hypatia.Rules.WorkflowAudit do # when a `{` follows the anchor, else as the block mapping under its line. # Returns `{:ok, entries, pos}` with the definition's own position (where # an alias inside the anchored mapping stands), or :error. - defp wf_anchor_entries(lines, name, {pos_line, pos_col}) do + defp wf_anchor_entries(lines, name, pos) do + case wf_anchor_def(lines, name, pos) do + nil -> + :error + + {line, masked, number, at, len} -> + after_anchor = binary_part(masked, at + len, byte_size(masked) - at - len) + + case Regex.run(~r/\{/, after_anchor, return: :index) do + [{brace, _}] -> + case wf_flow_interior(lines, number, at + len + brace) do + {:ok, interior, _} -> {:ok, wf_flow_entries(interior), {number, :eol}} + :error -> :error + end + + nil -> + children = wf_block_children(Enum.drop(lines, number), wf026_key_indent(line)) + {:ok, children, {number, :eol}} + end + end + end + + # The scalar the anchor `&name` an alias at `pos` names holds, as written + # (quoted or plain, "" for none): `{:ok, value}`, or :error when there is + # no such anchor, or it names a mapping or sequence, or its value is + # continued on the lines below. + defp wf_anchor_scalar(lines, name, pos) do + with {line, masked, number, at, len} <- wf_anchor_def(lines, name, pos), + rest = binary_part(line, at + len, byte_size(line) - at - len), + false <- Regex.match?(~r/^\s*(?:![^\s]*\s+)*[\[{*&|>]/, rest) do + cond do + m = + Regex.run( + ~r/^\s+(?:![^\s]*\s+)*("(?:[^"\\]|\\.)*"|'(?:[^']|'')*'|[^\s,}\]#][^,}\]#]*?)\s*(?:[,}\]]|\s#|$)/, + rest + ) -> + {:ok, Enum.at(m, 1)} + + # Nothing after the anchor but a tag or a comment: null, unless a + # deeper line continues the scalar. + Regex.match?( + ~r/^\s*(?:![^\s]*\s*)*(?:#.*)?$/, + binary_part(masked, at + len, byte_size(masked) - at - len) + ) -> + next = lines |> Enum.drop(number) |> Enum.find(&wf026_plain?/1) + if next && wf026_indent(next) > wf026_indent(line), do: :error, else: {:ok, ""} + + true -> + :error + end + else + _ -> :error + end + end + + # The latest definition of `&name` before `pos`, as + # `{line, masked line, number, offset, length}`, or nil. + defp wf_anchor_def(lines, name, {pos_line, pos_col}) do re = ~r/&#{Regex.escape(name)}(?=[\s,{}\[\]]|$)/ states = lines |> Enum.join("\n") |> Hypatia.Rules.PinSites.line_states() @@ -1906,25 +2013,6 @@ defmodule Hypatia.Rules.WorkflowAudit do end end) |> List.last() - |> case do - nil -> - :error - - {line, masked, number, at, len} -> - after_anchor = binary_part(masked, at + len, byte_size(masked) - at - len) - - case Regex.run(~r/\{/, after_anchor, return: :index) do - [{brace, _}] -> - case wf_flow_interior(lines, number, at + len + brace) do - {:ok, interior, _} -> {:ok, wf_flow_entries(interior), {number, :eol}} - :error -> :error - end - - nil -> - children = wf_block_children(Enum.drop(lines, number), wf026_key_indent(line)) - {:ok, children, {number, :eol}} - end - end end # The direct children of a block mapping whose key sits at `key_indent`, @@ -2534,7 +2622,7 @@ defmodule Hypatia.Rules.WorkflowAudit do Enum.flat_map(workflow_contents, fn {filename, content} -> lines = String.split(content, "\n") continued = wf_continued_uses(lines) - depths = Hypatia.Rules.PinSites.line_depths(content) + flow = wf_flow_lines(content) Enum.flat_map(@broken_reusable_revisions, fn rev -> # Anchored to a `uses:` key at the start of a line (quoted or not) that @@ -2576,7 +2664,7 @@ defmodule Hypatia.Rules.WorkflowAudit do true -> nil end - case wf027_job_key(lines, depths, number) do + case wf027_job_key(lines, flow, number) do :block -> [{sha, review}] :flow -> [{sha, review || :flow}] nil -> [] @@ -2586,7 +2674,7 @@ defmodule Hypatia.Rules.WorkflowAudit do inline_re |> Regex.scan(line, return: :index) |> Enum.filter(fn [{at, _} | _] -> - wf027_inline_job?(lines, depths, line, number, at + 1) + wf027_inline_job?(lines, flow, line, number, at + 1) end) |> Enum.map(fn [_, {sha_at, 40}] -> {binary_part(line, sha_at, 40), :flow} end) end @@ -2613,14 +2701,14 @@ defmodule Hypatia.Rules.WorkflowAudit do # the fixer rewrites), and :flow when only the brackets say so (`jobs: {`, # `rust: {`, `uses: ...` all in column 0, which the fixer cannot place). # nil when it is no job's key. - defp wf027_job_key(lines, depths, number) do + defp wf027_job_key(lines, flow, number) do block? = match?([_job, "jobs" | _], wf_ancestor_keys(lines, number)) and not wf_in_open_quote?(lines, wf_parent_line(lines, number), number) cond do - Enum.at(depths, number - 1, 0) == 0 -> if block?, do: :block - not wf027_flow_job?(wf_flow_context(lines, depths, number)) -> nil + not wf_in_flow?(flow, number) -> if block?, do: :block + not wf027_flow_job?(wf_flow_context(lines, flow, number)) -> nil block? -> :block true -> :flow end @@ -2651,11 +2739,11 @@ defmodule Hypatia.Rules.WorkflowAudit do # must read a job key, then `jobs`. Each key is the last entry before its # brace, split quote-aware, so a sibling's `name: "CI, build"` separates # nothing. - defp wf027_inline_job?(lines, depths, line, number, uses_at) do + defp wf027_inline_job?(lines, flow, line, number, uses_at) do # On a line inside a flow collection opened above, the brackets before # `uses` are read from where the collection began. - if Enum.at(depths, number - 1, 0) > 0, - do: wf027_flow_job?(wf_flow_context(lines, depths, number, binary_part(line, 0, uses_at))), + if wf_in_flow?(flow, number), + do: wf027_flow_job?(wf_flow_context(lines, flow, number, uses_at)), else: wf027_line_job?(lines, line, number, uses_at) end @@ -2673,12 +2761,16 @@ defmodule Hypatia.Rules.WorkflowAudit do # The key whose value is the flow mapping opening at byte `open`: the last # top-level entry of `line[from, open)` must read `key:` and nothing else. defp wf027_key_before(line, from, open) do + # Node properties before the bracket name the collection, not its key: + # `rust: &job {` is the job `rust`, `[ &step {` an anonymous step. entry = line |> binary_part(from, open - from) |> wf_flow_split(?,) |> List.last("") |> String.trim() + |> String.replace(~r/(?:^|\s+)(?:[&!]\S*)(?:\s+[&!]\S*)*$/, "") + |> String.trim() case Regex.run(~r/^["']?([\w.-]+)["']?\s*:$/, entry) do [_, key] -> key diff --git a/scripts/sweeps/estate-pin-integrity.sh b/scripts/sweeps/estate-pin-integrity.sh index 23c84ae3..144421e3 100755 --- a/scripts/sweeps/estate-pin-integrity.sh +++ b/scripts/sweeps/estate-pin-integrity.sh @@ -243,7 +243,9 @@ function walk(s, i, seen, n, c, j, t) { seen = seen c; i++ } t = seen; sub(/[[:space:]]+$/, "", t) - if (t !~ /^[[:space:]]*$/) PENDING = (t ~ /:$/ || t ~ /^[[:space:]]*-$/ || (DEPTH > 0 && t ~ /[{,[]$/)) + # A key or dash followed only by node properties (`- run: &cmd`, `key: + # !!str`) leaves its value pending as a bare one does. + if (t !~ /^[[:space:]]*$/) PENDING = (t ~ /(:|^[[:space:]]*-)([[:space:]]+[&!][^[:space:]]*)*$/ || (DEPTH > 0 && t ~ /[{,[]$/)) return "" } BEGIN { USES_TAIL = "(^|[-{,[:space:]])[\"" sq "]?uses[\"" sq "]?[[:space:]]*:[[:space:]]*$"; in_scalar = 0; inq = ""; DEPTH = 0; PENDING = 0 } diff --git a/test/rules/pin_integrity_test.exs b/test/rules/pin_integrity_test.exs index 1f8b6cd6..1a2aefca 100644 --- a/test/rules/pin_integrity_test.exs +++ b/test/rules/pin_integrity_test.exs @@ -332,6 +332,84 @@ defmodule Hypatia.Rules.PinIntegrityTest do assert PI.pi001_denylisted_pin(not_uses, @policy) == [] end + test "node properties before a step's own mapping leave it a step" do + content = + " steps: [ &step { uses: github/codeql-action/init@#{@poison} } ]\n" <> + " - &other { name: x, uses: github/codeql-action/analyze@#{@poison} }\n" + + assert [%{line: 1}, %{line: 2}] = PI.pi001_denylisted_pin(content, @policy) + assert {:ok, new_content, [_, _]} = PI.substitute_denylisted_pins(content, @policy) + refute new_content =~ @poison + + # Control: an anchored input mapping is still data. + assert PI.pi001_denylisted_pin( + " with: &w { uses: github/codeql-action/init@#{@poison} }\n", + @policy + ) == [] + end + + test "an anchor whose scalar continues on the next line holds that reference" do + content = + "env:\n BAD: &bad\n github/codeql-action/init@#{@poison}\n" <> + "jobs:\n a:\n steps:\n - uses: *bad\n" + + assert [%{line: 3, detail: %{ref: @poison}}] = PI.pi001_denylisted_pin(content, @policy) + assert {:ok, new_content, [_]} = PI.substitute_denylisted_pins(content, @policy) + assert new_content =~ " BAD: &bad\n github/codeql-action/init@#{@good}\n" + + # Control: a line that is not deeper is no continuation — `&bad` is + # null, and the alias names no reference. + null = + "env:\n BAD: &bad\n NEXT: github/codeql-action/init@#{@poison}\n" <> + "jobs:\n a:\n steps:\n - uses: *bad\n" + + assert PI.pi001_denylisted_pin(null, @policy) == [] + end + + test "inside a flow collection opened above, a step's uses: after a sibling is a site" do + # The quoted sibling closes on the line the `uses:` stands on; the + # brackets the lines above opened still make it the step's own key. + quoted = + "jobs:\n a:\n steps: [{ name: \"hello\n world\", " <> + "uses: github/codeql-action/init@#{@poison} }]\n" + + sibling = + "jobs:\n a:\n steps: [\n {\n name: x, " <> + "uses: github/codeql-action/init@#{@poison}\n }\n ]\n" + + for {content, line} <- [{quoted, 4}, {sibling, 5}] do + assert [%{line: ^line}] = PI.pi001_denylisted_pin(content, @policy) + assert {:ok, new_content, [_]} = PI.substitute_denylisted_pins(content, @policy) + assert new_content =~ "github/codeql-action/init@#{@good}" + end + + # Controls: the same layout inside a step input's mapping is data, and + # an own-line `uses:` there is too. + for content <- [ + "jobs:\n a:\n steps: [{ with: { note: \"x\n y\", " <> + "uses: github/codeql-action/init@#{@poison} } }]\n", + "jobs:\n a:\n steps:\n - with: {\n " <> + "uses: github/codeql-action/init@#{@poison}\n }\n" + ] do + assert PI.pi001_denylisted_pin(content, @policy) == [] + end + end + + test "a value pending after node properties opens the quoted scalar below it" do + # `run: &cmd` leaves its value pending: the quote on the next line + # opens the string, and the `uses:` text inside it is no site. + content = + " - run: &cmd\n \"echo start\n" <> + " uses: github/codeql-action/init@#{@poison}\n echo end\"\n" + + assert PI.pi001_denylisted_pin(content, @policy) == [] + assert {:ok, ^content, []} = PI.substitute_denylisted_pins(content, @policy) + + # Control: a value written on the property's own line pends nothing. + live = " - run: &cmd echo\n - uses: github/codeql-action/init@#{@poison}\n" + assert [%{line: 2}] = PI.pi001_denylisted_pin(live, @policy) + end + test "a KYAML flow step on its own line is still a site" do content = "steps: [\n { name: \"x\", run: \"echo 'a\" },\n" <> diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs index fe1808ac..1d85b9ac 100644 --- a/test/rules/workflow_audit_pins_test.exs +++ b/test/rules/workflow_audit_pins_test.exs @@ -907,6 +907,72 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do }) end + test "node properties before a step's mapping leave it a flow step" do + for {wf, line} <- [ + {"jobs:\n b:\n steps: [ &step { uses: dtolnay/rust-toolchain@#{@toolchain_sha} } ]\n", + 3}, + {"jobs:\n b:\n steps:\n - &step { uses: dtolnay/rust-toolchain@#{@toolchain_sha} }\n", + 4} + ] do + assert [%{line: ^line, auto_fixable: false}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + end + + test "a quoted sibling continued past a `#` line does not hide the step" do + # `# world",` continues the string; it is no comment, and its quote + # closes the scalar before the step's own `uses:`. + wf = + "jobs: {\nb: {\nsteps: [\n{\nname: \"hello\n# world\",\n" <> + "uses: dtolnay/rust-toolchain@#{@toolchain_sha}\n}\n]\n}\n}\n" + + assert [%{line: 7, auto_fixable: false}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + + # The same quote closing on the `uses:` line's own start. + inline = + "jobs:\n b:\n steps: [{ name: \"hello\n world\", " <> + "uses: dtolnay/rust-toolchain@#{@toolchain_sha} }]\n" + + assert [%{line: 4, auto_fixable: false}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => inline}) + end + + test "toolchain: *alias is the scalar its anchor holds" do + step = fn tc -> + "x: &empty \"\"\ny: &chan stable\njobs:\n b:\n steps:\n" <> + " - uses: dtolnay/rust-toolchain@#{@toolchain_sha}\n with:\n" <> + " toolchain: #{tc}\n" + end + + # An empty anchored value is no input, as `toolchain: ""` is not. + assert [%{line: 6, auto_fixable: true}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{ + "rust.yml" => step.("*empty") + }) + + # An alias of no anchor cannot be read: reported, routed to review. + assert [%{line: 6, auto_fixable: false, fix_script: nil} = f] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{ + "rust.yml" => step.("*nope") + }) + + assert f.reason =~ "`toolchain:` input is the alias `*nope`" + + # Control: an anchored channel is the input. + assert [] == + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{ + "rust.yml" => step.("*chan") + }) + + flow = + "x: &empty ~\njobs:\n b:\n steps:\n" <> + " - { uses: dtolnay/rust-toolchain@#{@toolchain_sha}, with: { toolchain: *empty } }\n" + + assert [%{line: 5}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => flow}) + end + test "a flow step's alias names the anchor before it, not one later on its line" do # `*inputs` is the empty mapping defined first; the configured # `&inputs` after it on the same line is not what it names. @@ -1169,6 +1235,25 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do end end + test "a job's mapping carrying node properties, or past a `#` string line, is the call" do + anchored = "jobs: { rust: &job { uses: #{@rust_ci}@#{@broken_standards} } }\n" + + assert [%{line: 1, auto_fixable: false}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => anchored}) + + quoted = + "jobs: {\nrust: {\nname: \"hello\n# world\",\nuses: #{@rust_ci}@#{@broken_standards}\n}\n}\n" + + assert [%{line: 5, auto_fixable: false}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => quoted}) + + # Control: a `uses:` line inside that string is text. + text = + "jobs: {\nrust: {\nname: \"hello\nuses: #{@rust_ci}@#{@broken_standards}\n\",\n}\n}\n" + + assert WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => text}) == [] + end + test "a flow-style job's uses: continued on the next line is the call" do wf = "jobs: { rust: { uses:\n #{@rust_ci}@#{@broken_standards} } }\n" From d2a3e738af74bae9803e3d838012facf59a015e1 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 07:46:39 +0000 Subject: [PATCH 42/47] fix(pins): aliased sites judged by their brackets; steps sequences need a job; anchors only where nodes start; block values resolved in place - PinSites.alias_sites/2 takes the file's flow prefixes, so a `uses: *name` inside a flow collection opened above is judged by the brackets still open: one in a step's `env: {` is data, and PI001 no longer substitutes the anchored env string it names. - An anonymous mapping in a `steps` sequence is a step only when that is the whole stack the text shows (a block job's line) or the stack goes on through a job key into `jobs`; `fixture: { steps: [ { uses: ... } ] }` in a matrix is data. - An anchor counts only where YAML lets a node start: the line's start or after its dashes, after `key: `, or after `[`, `{`, `,` inside a flow collection. `NOTE: foo&channel x` and `NOTE: a, &bad x` are plain scalars (PinSites and WF026's anchor lookup alike). - WF026: a bare sequence item's anchor (`- &inputs`) holds the mapping under its dash; a block input's alias resolves at its own child line (`source: &channel nightly` then `toolchain: *channel`); an empty block scalar (`toolchain: >-`) is the empty string. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- lib/rules/pin_integrity.ex | 5 +- lib/rules/pin_sites.ex | 54 +++++++++----- lib/rules/workflow_audit.ex | 94 ++++++++++++++++++++++--- test/rules/pin_integrity_test.exs | 51 ++++++++++++++ test/rules/workflow_audit_pins_test.exs | 44 ++++++++++++ 5 files changed, 219 insertions(+), 29 deletions(-) diff --git a/lib/rules/pin_integrity.ex b/lib/rules/pin_integrity.ex index d68592cb..beb1d075 100644 --- a/lib/rules/pin_integrity.ex +++ b/lib/rules/pin_integrity.ex @@ -85,7 +85,8 @@ defmodule Hypatia.Rules.PinIntegrity do @spec pin_sites(String.t()) :: [map()] def pin_sites(content) when is_binary(content) do masked = PinSites.mask_lines(content) - numbered = masked |> Enum.zip(PinSites.flow_prefixes(content)) |> Enum.with_index(1) + prefixes = PinSites.flow_prefixes(content) + numbered = masked |> Enum.zip(prefixes) |> Enum.with_index(1) # A `uses:` whose reference continues below it — past any blank or # comment-only lines — is a site on the value's line. @@ -98,7 +99,7 @@ defmodule Hypatia.Rules.PinIntegrity do # A `uses: *name` runs the reference the anchor holds: its site is the # anchor's value, where a repair rewrites it (every alias with it). aliased = - for %{anchor_line: number, site: site} <- PinSites.alias_sites(masked) do + for %{anchor_line: number, site: site} <- PinSites.alias_sites(masked, prefixes) do site_entry( site, number, diff --git a/lib/rules/pin_sites.ex b/lib/rules/pin_sites.ex index fe6a0c09..d7f761b2 100644 --- a/lib/rules/pin_sites.ex +++ b/lib/rules/pin_sites.ex @@ -220,19 +220,24 @@ defmodule Hypatia.Rules.PinSites do alias to the latest `&name` before it, so a later definition does not count and an earlier one is shadowed — by any value, including one that is no `action@ref` scalar, which names no site. Lines as `mask_lines/1` - returns them. + returns them; `flow_prefixes` (`flow_prefixes/1` of the same file) lets a + line inside a multi-line flow collection be judged against the brackets + the lines above opened, as `sites/2` judges a literal site: a `uses: + *name` in an `env: {` spread over lines is data, not a step's action. """ - @spec alias_sites([String.t()]) :: [map()] - def alias_sites(lines) when is_list(lines) do - numbered = Enum.with_index(lines, 1) + @spec alias_sites([String.t()], [String.t()]) :: [map()] + def alias_sites(lines, flow_prefixes \\ []) when is_list(lines) do + prefixes = flow_prefixes ++ List.duplicate("", max(length(lines) - length(flow_prefixes), 0)) + numbered = lines |> Enum.zip(prefixes) |> Enum.with_index(1) defs = - for {line, n} <- numbered, d <- anchor_defs(line) do - d |> Map.put(:line, n) |> continued_anchor(line, Enum.drop(numbered, n)) + for {{line, prefix}, n} <- numbered, d <- anchor_defs(line, prefix) do + below = for {{l, _}, m} <- Enum.drop(numbered, n), do: {l, m} + d |> Map.put(:line, n) |> continued_anchor(line, below) end - for {line, n} <- numbered, - {name, at, len} <- uses_aliases(line), + for {{line, prefix}, n} <- numbered, + {name, at, len} <- uses_aliases(line, prefix), %{site: %{} = site} = d <- [latest_anchor(defs, name, n, at)] do %{line: n, at: at, len: len, anchor_line: Map.get(d, :site_line, d.line), site: site} end @@ -269,16 +274,21 @@ defmodule Hypatia.Rules.PinSites do # The anchors `line` defines where a node starts (outside quotes and the # comment), each `%{name, at, site}`: the site its scalar value is, or nil. - defp anchor_defs(line) do + # A node starts at the line's start or after its list dashes, after + # `key: `, and — inside a flow collection only (one the line opens, or + # `flow_prefix` says it sits in) — after `[`, `{` or `,`, each past any + # tag. `NOTE: a, &x b` in block context is one plain scalar: no anchor. + defp anchor_defs(line, flow_prefix) do masked = mask(line) ~r/&([\w.-]+)/ |> Regex.scan(masked, return: :index) |> Enum.filter(fn [{at, _}, _] -> - Regex.match?( - ~r/(?:^\s*(?:-\s+)*|:\s+|[\[{,]\s*)(?:![^\s]*\s+)*$/, - binary_part(masked, 0, at) - ) + before = binary_part(masked, 0, at) + + Regex.match?(~r/(?:^\s*(?:-\s+)*|:\s+)(?:![^\s]*\s+)*$/, before) or + ((flow_prefix != "" or String.contains?(before, ["[", "{"])) and + Regex.match?(~r/[\[{,]\s*(?:![^\s]*\s+)*$/, before)) end) |> Enum.map(fn [{at, len}, {n_at, n_len}] -> rest_at = at + len @@ -308,7 +318,7 @@ defmodule Hypatia.Rules.PinSites do # The `uses:` keys of `line` whose value is an alias, at a site position, # as `{name, offset of the "*", length of "*name"}`. - defp uses_aliases(line) do + defp uses_aliases(line, flow_prefix) do masked = mask(line) ~r/["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*\*([\w.-]+)/ @@ -316,8 +326,14 @@ defmodule Hypatia.Rules.PinSites do |> Enum.flat_map(fn [{at, len}, {n_at, n_len}] -> star = n_at - 1 prefix = binary_part(masked, 0, at) + head = binary_part(masked, 0, star) + + structural? = + if flow_prefix == "", + do: structural?(head), + else: structural_in_flow?(flow_prefix <> head) - if structural?(binary_part(masked, 0, star)) and + if structural? and not Regex.match?(~r/[A-Za-z0-9_.\/-]$/, prefix), do: [{binary_part(masked, n_at, n_len), star, at + len - star}], else: [] @@ -597,11 +613,17 @@ defmodule Hypatia.Rules.PinSites do end end + # A `{` item of a sequence is a step when that sequence is `steps:` of a + # job: the whole stack the text shows (`steps: [ {` on a block job's + # line, which this reader cannot see past), or one that goes on through a + # job key into `jobs` (`jobs: { b: { steps: [ {`). The same `steps:` + # inside any other mapping (`fixture: { steps: [ {` in a matrix) is data. defp structural_flow_stack?(stack) do Enum.all?(stack, fn {_, key} -> key != :text end) and case stack do [{"{", :none}] -> true - [{"{", :none}, {"[", key} | _] when key in ["steps", :none] -> true + [{"{", :none}, {"[", key}] when key in ["steps", :none] -> true + [{"{", :none}, {"[", "steps"}, {"{", job}, {"{", "jobs"} | _] when is_binary(job) -> true [{"{", job}, {"{", "jobs"} | _] when is_binary(job) -> true _ -> false end diff --git a/lib/rules/workflow_audit.ex b/lib/rules/workflow_audit.ex index 53fa8942..db19655b 100644 --- a/lib/rules/workflow_audit.ex +++ b/lib/rules/workflow_audit.ex @@ -1750,7 +1750,8 @@ defmodule Hypatia.Rules.WorkflowAudit do # (`components: |` carrying `toolchain: stable`) sit deeper and are text. true -> form = if wf026_anchored?(value), do: :anchor, else: :block - wf026_entries_verdict(wf_block_children(rest, key_indent), lines, form, {number, :eol}) + pos = {:block, number, wf_block_child_lines(rest, key_indent, number)} + wf026_entries_verdict(wf_block_children(rest, key_indent), lines, form, pos) end end @@ -1823,7 +1824,7 @@ defmodule Hypatia.Rules.WorkflowAudit do case Enum.find(entries, fn {k, _} -> k == "toolchain" end) do {_, v} when is_binary(v) -> case Regex.run(~r/^\s*\*([\w.-]+)\s*$/, v) do - [_, name] -> wf026_scalar_alias(name, lines, pos) + [_, name] -> wf026_scalar_alias(name, lines, wf_entry_pos(pos, "toolchain")) nil -> if wf026_null_value?(v), do: :null, else: {:set, :own} end @@ -1834,7 +1835,7 @@ defmodule Hypatia.Rules.WorkflowAudit do case wf026_merge_aliases(v) do {:ok, names} -> Enum.find_value(names, fn name -> - case wf026_alias_lookup(name, lines, seen, pos) do + case wf026_alias_lookup(name, lines, seen, wf_entry_pos(pos, "<<")) do :absent -> nil {:set, _} -> {:set, :merge} other -> other @@ -1882,6 +1883,15 @@ defmodule Hypatia.Rules.WorkflowAudit do end end + # Where an entry's value stands, for an alias in it: a block mapping's + # position (`{:block, line, %{key => line}}`) is each child's own line — + # `source: &channel nightly` then `toolchain: *channel` names the anchor + # one line above it — any other position is the mapping's. + defp wf_entry_pos({:block, number, child_lines}, key), + do: {Map.get(child_lines, key, number), :eol} + + defp wf_entry_pos(pos, _key), do: pos + # `toolchain: *name` is the scalar the latest `&name` before it anchors: # null or empty there is no input, exactly as if written in place # (`x: &empty ""`). An anchor this reader cannot find, or one that names a @@ -1950,8 +1960,17 @@ defmodule Hypatia.Rules.WorkflowAudit do end nil -> - children = wf_block_children(Enum.drop(lines, number), wf026_key_indent(line)) - {:ok, children, {number, :eol}} + # A bare sequence item carrying the anchor (`- &inputs`, its keys + # on the lines below) holds the mapping under the dash, not under + # the column its first key would take. + parent = + if Regex.match?(~r/^\s*-\s+(?:[&!]\S*\s*)+$/, wf_strip_comment(line)), + do: wf026_indent(line), + else: wf026_key_indent(line) + + rest = Enum.drop(lines, number) + children = wf_block_children(rest, parent) + {:ok, children, {:block, number, wf_block_child_lines(rest, parent, number)}} end end end @@ -1991,22 +2010,29 @@ defmodule Hypatia.Rules.WorkflowAudit do # The latest definition of `&name` before `pos`, as # `{line, masked line, number, offset, length}`, or nil. + # An anchor is `&name` where YAML lets a node start: at the line's start or + # after its list dashes, after `key: `, or — inside a flow collection (one + # the line opens, or the line sits in) — after `[`, `{` or `,`, each past + # any tag. `NOTE: foo&channel nightly` is one plain scalar, no anchor. defp wf_anchor_def(lines, name, {pos_line, pos_col}) do re = ~r/&#{Regex.escape(name)}(?=[\s,{}\[\]]|$)/ - states = lines |> Enum.join("\n") |> Hypatia.Rules.PinSites.line_states() + content = Enum.join(lines, "\n") + states = Hypatia.Rules.PinSites.line_states(content) + depths = Hypatia.Rules.PinSites.line_depths(content) lines - |> Enum.zip(states) + |> Enum.zip(Enum.zip(states, depths)) |> Enum.with_index(1) |> Enum.take(pos_line) - |> Enum.flat_map(fn {{l, {_, state}}, number} -> + |> Enum.flat_map(fn {{l, {{_, state}, depth}}, number} -> if state != :text and wf026_plain?(l) do masked = wf_mask_quoted(l) re |> Regex.scan(masked, return: :index) |> Enum.filter(fn [{at, _}] -> number < pos_line or pos_col == :eol or at < pos_col end) + |> Enum.filter(fn [{at, _}] -> wf_node_start?(binary_part(masked, 0, at), depth) end) |> Enum.map(fn [{at, len}] -> {l, masked, number, at, len} end) else [] @@ -2015,6 +2041,44 @@ defmodule Hypatia.Rules.WorkflowAudit do |> List.last() end + defp wf_node_start?(before, depth) do + Regex.match?(~r/(?:^\s*(?:-\s+)*|:\s+)(?:![^\s]*\s+)*$/, before) or + ((depth > 0 or String.contains?(before, ["[", "{"])) and + Regex.match?(~r/[\[{,]\s*(?:![^\s]*\s+)*$/, before)) + end + + # The line of each direct child of the block mapping whose key sits at + # `key_indent`, `rest` being the lines after the key's line `number`, as + # `%{key => line}` — the children `wf_block_children/2` reads, the first + # of a key spelled twice. + defp wf_block_child_lines(rest, key_indent, number) do + block = + rest + |> Enum.with_index(number + 1) + |> Enum.take_while(fn {l, _} -> + trimmed = String.trim(l) + trimmed == "" or String.starts_with?(trimmed, "#") or wf026_indent(l) > key_indent + end) + |> Enum.filter(fn {l, _} -> wf026_plain?(l) end) + + case block do + [] -> + %{} + + [{first, _} | _] -> + child_indent = wf026_indent(first) + + block + |> Enum.filter(fn {l, _} -> wf026_indent(l) == child_indent end) + |> Enum.reduce(%{}, fn {l, n}, acc -> + case Regex.run(~r/^\s*(?:-\s+)?["']?([^"':\s]+)["']?\s*:/, l) do + [_, k] -> Map.put_new(acc, k, n) + nil -> acc + end + end) + end + end + # The direct children of a block mapping whose key sits at `key_indent`, # as `{key, value}` entries: the plain lines deeper than the key, at the # first child's indentation. Deeper lines are nested mappings or scalar @@ -2043,7 +2107,12 @@ defmodule Hypatia.Rules.WorkflowAudit do case Regex.run(~r/^\s*(?:-\s+)?["']?([^"':\s]+)["']?\s*:(.*)$/, l) do [_, k, v] -> v = v |> wf_strip_comment() |> String.trim() - [{k, if(v == "", do: wf_continued_value(children, i, child_indent), else: v)}] + + # A value on the lines below — nothing on the key's line, or a + # block scalar's indicator (`toolchain: >-`) — is what those + # lines hold: an empty block scalar is the empty string. + continued? = v == "" or Regex.match?(~r/^(?:[&!]\S*\s+)*[|>][-+0-9]*$/, v) + [{k, if(continued?, do: wf_continued_value(children, i, child_indent), else: v)}] nil -> [] @@ -2308,7 +2377,9 @@ defmodule Hypatia.Rules.WorkflowAudit do # (`PinSites.mask_lines/1`), so a `uses:` in a `run: |` script or a quoted # string stands for nothing. defp wf_continued_uses(lines) do - masked = lines |> Enum.join("\n") |> Hypatia.Rules.PinSites.mask_lines() + content = Enum.join(lines, "\n") + masked = Hypatia.Rules.PinSites.mask_lines(content) + prefixes = Hypatia.Rules.PinSites.flow_prefixes(content) continued = for {key, _value, site} <- Hypatia.Rules.PinSites.continuations(masked), into: %{} do @@ -2319,7 +2390,8 @@ defmodule Hypatia.Rules.WorkflowAudit do " " <> site.action <> "@" <> site.ref, :continued}} end - for %{line: n, at: at, len: len, site: site} <- Hypatia.Rules.PinSites.alias_sites(masked), + for %{line: n, at: at, len: len, site: site} <- + Hypatia.Rules.PinSites.alias_sites(masked, prefixes), not Map.has_key?(continued, n), into: continued do line = Enum.at(lines, n - 1) diff --git a/test/rules/pin_integrity_test.exs b/test/rules/pin_integrity_test.exs index 1a2aefca..d0b97db0 100644 --- a/test/rules/pin_integrity_test.exs +++ b/test/rules/pin_integrity_test.exs @@ -410,6 +410,57 @@ defmodule Hypatia.Rules.PinIntegrityTest do assert [%{line: 2}] = PI.pi001_denylisted_pin(live, @policy) end + test "an aliased uses: inside a flow collection opened above is judged by its brackets" do + # `uses: *bad` in a step's `env: {` spread over lines is an env var. + anchor = "x: &bad github/codeql-action/init@#{@poison}\n" + + env = + anchor <> + "jobs:\n a:\n steps: [\n { run: echo, env: {\n uses: *bad\n } }\n ]\n" + + assert PI.pi001_denylisted_pin(env, @policy) == [] + assert {:ok, ^env, []} = PI.substitute_denylisted_pins(env, @policy) + + # Control: the step's own `uses:` after a sibling is the step's action. + step = + anchor <> "jobs:\n a:\n steps: [\n { run: echo,\n uses: *bad }\n ]\n" + + assert [%{line: 1}] = PI.pi001_denylisted_pin(step, @policy) + end + + test "a steps: sequence is a job's only when the brackets say so" do + # A matrix value holding `steps: [ { uses: ... } ]` is data. + data = + " include: [ { fixture: { steps: [ { uses: github/codeql-action/init@#{@poison} } ] } } ]\n" + + assert PI.pi001_denylisted_pin(data, @policy) == [] + + # Controls: through a job key into `jobs`, or the whole stack a block + # job's own line shows. + for line <- [ + "jobs: { b: { steps: [ { uses: github/codeql-action/init@#{@poison} } ] } }\n", + " steps: [ { uses: github/codeql-action/init@#{@poison} } ]\n" + ] do + assert [%{line: 1}] = PI.pi001_denylisted_pin(line, @policy) + end + end + + test "an anchor is a node property only where a node starts" do + # In block context `a, &bad b` is one plain scalar: no anchor. + text = + "env:\n NOTE: a, &bad github/codeql-action/init@#{@poison}\n" <> + "jobs:\n a:\n steps:\n - uses: *bad\n" + + assert PI.pi001_denylisted_pin(text, @policy) == [] + + # Control: after `[` in a flow sequence a node starts. + flow = + "env:\n LIST: [ &bad github/codeql-action/init@#{@poison} ]\n" <> + "jobs:\n a:\n steps:\n - uses: *bad\n" + + assert [%{line: 2}] = PI.pi001_denylisted_pin(flow, @policy) + end + test "a KYAML flow step on its own line is still a site" do content = "steps: [\n { name: \"x\", run: \"echo 'a\" },\n" <> diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs index 1d85b9ac..8ee49bbd 100644 --- a/test/rules/workflow_audit_pins_test.exs +++ b/test/rules/workflow_audit_pins_test.exs @@ -12,6 +12,11 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do @good_standards "75bd5994470661a099df562d6d5ad2fe744dbd66" @rust_ci "hyperpolymath/standards/.github/workflows/rust-ci-reusable.yml" + # A SHA-pinned toolchain step with no inputs, for a `with:` to follow. + @rust_step "jobs:\n b:\n steps:\n - uses: dtolnay/rust-toolchain@#{@toolchain_sha}\n" + + defp wf026(wf), do: WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + describe "check_sha_pinned_toolchain_without_input/1 (WF026)" do test "fires on a SHA-pinned step with no with: block at all" do wf = """ @@ -938,6 +943,45 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => inline}) end + test "a bare sequence item's anchor holds the mapping under its dash" do + item = + "x:\n - &inputs\n toolchain: nightly\n" <> @rust_step <> " with: *inputs\n" + + assert [] == wf026(item) + + unconfigured = + "x:\n - &inputs\n components: clippy\n" <> @rust_step <> " with: *inputs\n" + + assert [%{line: 7}] = wf026(unconfigured) + end + + test "an & inside a plain scalar is text, not the anchor an alias names" do + # `*channel` is the empty value anchored before `foo&channel`. + text = + "x: &channel \"\"\ny: foo&channel nightly\n" <> + @rust_step <> " with:\n toolchain: *channel\n" + + assert [%{line: 6}] = wf026(text) + end + + test "a block input's alias names the anchor on a child line above it" do + sibling = + @rust_step <> + " with:\n source: &channel nightly\n toolchain: *channel\n" + + assert [] == wf026(sibling) + end + + test "an empty block scalar toolchain is the empty string; one with content is set" do + empty = + @rust_step <> " with:\n toolchain: >-\n components: clippy\n" + + assert [%{line: 4}] = wf026(empty) + + folded = @rust_step <> " with:\n toolchain: >-\n stable\n" + assert [] == wf026(folded) + end + test "toolchain: *alias is the scalar its anchor holds" do step = fn tc -> "x: &empty \"\"\ny: &chan stable\njobs:\n b:\n steps:\n" <> From 3b78d550c773354a450dfc490706cc7fc0cf2af2 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 08:26:29 +0000 Subject: [PATCH 43/47] fix(pins): block uses: keys judged by their ancestry; WF026 reads values past node properties; flow entries honour escaped quotes - PinSites.data_uses_lines/1: a block `uses:` key the indentation above proves is neither a step's nor a job's own (an input named `uses` under `with:`, an item of a sequence another key holds, a `steps:` inside a step's input) holds data, as `with: { uses: ... }` already did. PI001, the mechanical repair and the PR scanner skip it, on its line, below its key and through its alias. A line whose ancestry is not visible (a hunk cut short, a fragment) is judged on its own, as before. - WF026: `toolchain: &channel ""`, `!!str ""`, `&channel null` and a bare `toolchain: &channel` are no input; `!!str null` is the string null, an input, directly or through an anchor. - WF026: the flow-entry reducer skips `\"` inside double quotes, so a step name like `"a\"b, c"` no longer hides `with: *inputs` and points the alias at a later anchor. - tests: 8 new, each failing on d2a3e73. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- lib/rules/pin_integrity.ex | 11 +- lib/rules/pin_sites.ex | 156 ++++++++++++++++++++++++ lib/rules/pr_automerge.ex | 22 +++- lib/rules/workflow_audit.ex | 83 +++++++++---- test/rules/pin_integrity_test.exs | 53 ++++++++ test/rules/pr_automerge_test.exs | 26 ++++ test/rules/workflow_audit_pins_test.exs | 59 +++++++++ 7 files changed, 382 insertions(+), 28 deletions(-) diff --git a/lib/rules/pin_integrity.ex b/lib/rules/pin_integrity.ex index beb1d075..f4a1dac3 100644 --- a/lib/rules/pin_integrity.ex +++ b/lib/rules/pin_integrity.ex @@ -87,11 +87,15 @@ defmodule Hypatia.Rules.PinIntegrity do masked = PinSites.mask_lines(content) prefixes = PinSites.flow_prefixes(content) numbered = masked |> Enum.zip(prefixes) |> Enum.with_index(1) + # A block `uses:` key the indentation above it puts under no step or job + # (an input of that name, `with:` then `uses: ...`) holds data: no site + # on its line, below it, or through its alias. + data = PinSites.data_uses_lines(content) # A `uses:` whose reference continues below it — past any blank or # comment-only lines — is a site on the value's line. continued = - for {_key, number, site} <- PinSites.continuations(masked) do + for {key, number, site} <- PinSites.continuations(masked), key not in data do next = Enum.at(masked, number - 1) site_entry(site, number, next |> PinSites.comment() |> String.trim()) end @@ -99,7 +103,9 @@ defmodule Hypatia.Rules.PinIntegrity do # A `uses: *name` runs the reference the anchor holds: its site is the # anchor's value, where a repair rewrites it (every alias with it). aliased = - for %{anchor_line: number, site: site} <- PinSites.alias_sites(masked, prefixes) do + for %{line: alias_line, anchor_line: number, site: site} <- + PinSites.alias_sites(masked, prefixes), + alias_line not in data do site_entry( site, number, @@ -108,6 +114,7 @@ defmodule Hypatia.Rules.PinIntegrity do end numbered + |> Enum.reject(fn {_, number} -> number in data end) |> Enum.flat_map(fn {{line, prefix}, number} -> line_sites(line, number, prefix) end) |> Kernel.++(continued) |> Kernel.++(aliased) diff --git a/lib/rules/pin_sites.ex b/lib/rules/pin_sites.ex index d7f761b2..34241383 100644 --- a/lib/rules/pin_sites.ex +++ b/lib/rules/pin_sites.ex @@ -346,6 +346,162 @@ defmodule Hypatia.Rules.PinSites do |> Enum.max_by(&{&1.line, &1.at}, fn -> nil end) end + @doc """ + The one-based numbers of the lines of `content` whose leading block `uses:` + key the indentation above it proves is neither a step's nor a job's own: + an input of that name (`with:` then `uses: a/b@x`), an entry of a sequence + some key other than `steps:` holds, or a key of a `steps:` that is no + job's. Such a key holds data, not a pin. A line whose ancestry the lines + above do not show far enough — a diff hunk cut short, a fragment with no + `jobs:` — is not listed: it is judged on its own line, as before. + """ + @spec data_uses_lines(String.t()) :: MapSet.t(pos_integer()) + def data_uses_lines(content) when is_binary(content) do + rows = + content + |> line_states() + |> Enum.zip_with(line_depths(content), &block_row/2) + |> List.to_tuple() + + for i <- 0..(tuple_size(rows) - 1)//1, + match?(%{key: "uses"}, elem(rows, i)), + uses_owner(rows, i) == :data, + into: MapSet.new(), + do: i + 1 + end + + # One line as block structure: its indentation, the column of its last + # list dash (nil without one), the column and name of the key after the + # dashes, whether that key's value is on the lines below (`key:` alone, + # past any node properties), and whether the line is a bare dash. Blank, + # comment-only and scalar lines, and lines inside a flow collection, are + # :skip: nothing there is a block parent. + defp block_row({text, :yaml}, 0) do + h = head(text) + + if String.trim(h) == "" do + :skip + else + [_, ind, dashes, rest] = Regex.run(~r/^( *)((?:-(?:\s+|$))*)(.*)$/, h) + + {key, open?} = + case Regex.run(~r/^["']?([^"':\s][^"':]*?)["']?\s*:(?:\s+(.*))?$/, rest) do + [_, k | value] -> + {k, Regex.match?(~r/^(?:[&!]\S*\s*)*$/, String.trim(Enum.join(value)))} + + nil -> + {nil, false} + end + + dash = + case :binary.matches(dashes, "-") do + [] -> nil + found -> byte_size(ind) + elem(List.last(found), 0) + end + + %{ + indent: byte_size(ind), + dash: dash, + key_col: byte_size(ind) + byte_size(dashes), + key: key, + open?: open?, + bare?: String.trim(rest) == "" + } + end + end + + defp block_row(_, _), do: :skip + + # :site, :data or :unknown for the `uses` key on row i: a step's when its + # mapping is an item of a job's `steps:`, a job's when its mapping is the + # value of a key of `jobs:`. + defp uses_owner(rows, i) do + case mapping_of(rows, i) do + {:item, j, d} -> sequence_owner(rows, seq_owner(rows, j, d)) + {:value, j, _key} -> job_key(rows, j) + _ -> :unknown + end + end + + defp sequence_owner(rows, {:value, s, "steps"}), do: steps_owner(rows, s) + defp sequence_owner(_rows, {:value, _, _}), do: :data + defp sequence_owner(_rows, _), do: :unknown + + # `steps:` is a job's, or a composite action's under `runs:`; inside a + # list item or another key's mapping it is data (`with:` then `steps:`). + defp steps_owner(rows, s) do + case mapping_of(rows, s) do + {:value, _p, "runs"} -> :site + {:value, p, _job} -> job_key(rows, p) + {:item, _, _} -> :data + _ -> :unknown + end + end + + # Is the key on row j a job: its mapping the value of `jobs:`? A key + # inside a list item (a step's `with:`) or under any other key is not. + defp job_key(rows, j) do + case mapping_of(rows, j) do + {:value, _, "jobs"} -> :site + {:value, _, _} -> :data + {:item, _, _} -> :data + _ -> :unknown + end + end + + # The mapping row j's key sits in: the list item its own dash opens, or + # the one the rows above give it. + defp mapping_of(rows, j) do + case elem(rows, j) do + %{dash: d} when d != nil -> {:item, j, d} + %{key_col: col} -> container(rows, j, col) + end + end + + # The mapping a key at column `col` of row i belongs to, from the nearest + # row above it that sits shallower: a list item whose first key is at + # `col` (`- name: x` above ` uses:`) or a bare dash, `{:item, row, dash}`; + # a key whose value is below it, `{:value, row, key}`. Rows as deep or + # deeper are its siblings and their contents. :root when no row is + # shallower; :unknown when the shallower row is neither. + defp container(rows, i, col) do + Enum.reduce_while((i - 1)..0//-1, :root, fn j, acc -> + case elem(rows, j) do + :skip -> {:cont, acc} + %{indent: ind} when ind >= col -> {:cont, acc} + %{dash: d, key_col: ^col} when d != nil -> {:halt, {:item, j, d}} + %{dash: d, bare?: true} when d != nil -> {:halt, {:item, j, d}} + %{key: k, open?: true, key_col: kc} when k != nil and kc < col -> {:halt, {:value, j, k}} + _ -> {:halt, :unknown} + end + end) + end + + # The key that holds the sequence whose item's dash is at column `d` of + # row j: the nearest row above that is not a sibling item or deeper — + # `steps:` above it, or at its own column (`steps:` then `- uses:`). + # :unknown when no row above shows it. + defp seq_owner(rows, j, d) do + Enum.reduce_while((j - 1)..0//-1, :unknown, fn k, acc -> + case elem(rows, k) do + :skip -> + {:cont, acc} + + %{indent: ind} when ind > d -> + {:cont, acc} + + %{indent: ^d, dash: dd} when dd != nil -> + {:cont, acc} + + %{key: key, open?: true, key_col: kc} when key != nil and kc <= d -> + {:halt, {:value, k, key}} + + _ -> + {:halt, :unknown} + end + end) + end + @doc """ The flow depth at the start of each line of `content` — how many `[` and `{` the lines above left open, as the walk of `line_states/1` counts them diff --git a/lib/rules/pr_automerge.ex b/lib/rules/pr_automerge.ex index e9a80f63..930dc6ef 100644 --- a/lib/rules/pr_automerge.ex +++ b/lib/rules/pr_automerge.ex @@ -609,7 +609,9 @@ defmodule Hypatia.Rules.PrAutomerge do # `script: >-`) or a quoted one continued over lines (`- run: "echo` ... # `done"`). Each side's context and changed lines are read in order by # `PinSites.line_states/1`, the reader PI001 reads a file with. A pin - # written inside a script is text. + # written inside a script is text. So is a block `uses:` key the side's + # indentation shows under no step or job — an input of that name (`with:` + # then `uses: ...`, `PinSites.data_uses_lines/1`): data, not a pin line. # # `certain?`: the hunk does not show the line among script text. A line # after an opening quote the hunk never shows closed is not certain: the @@ -661,7 +663,8 @@ defmodule Hypatia.Rules.PrAutomerge do end # The continued `uses:` values of one side of a hunk, as - # `%{hunk_index => [pin]}` on the value's line. + # `%{hunk_index => [pin]}` on the value's line. A `:text` line, a key that + # holds data among them, continues no pin. defp side_continuations(view) do texts = for {_, text, state} <- view, do: if(state == :text, do: "", else: text) index = view |> Enum.map(&elem(&1, 0)) |> List.to_tuple() @@ -674,7 +677,9 @@ defmodule Hypatia.Rules.PrAutomerge do # One side of a hunk (context lines plus the changed lines of `mark`), in # order, as `{hunk_index, text, state}` — `state` as - # `PinSites.line_states/1` reads the side: `:yaml`, `:text` or `:open`. + # `PinSites.line_states/1` reads the side: `:yaml`, `:text` or `:open`; a + # block `uses:` key the side shows under no step or job + # (`PinSites.data_uses_lines/1`) is `:text` too. defp side_view(hunk, mark) do entries = hunk @@ -684,9 +689,16 @@ defmodule Hypatia.Rules.PrAutomerge do end) |> Enum.map(fn {line, i} -> {i, String.slice(line, 1..-1//1)} end) - states = entries |> Enum.map_join("\n", &elem(&1, 1)) |> PinSites.line_states() + side = Enum.map_join(entries, "\n", &elem(&1, 1)) + states = PinSites.line_states(side) + data = PinSites.data_uses_lines(side) - Enum.zip_with(entries, states, fn {i, text}, {_, state} -> {i, text, state} end) + entries + |> Enum.zip(states) + |> Enum.with_index(1) + |> Enum.map(fn {{{i, text}, {_, state}}, n} -> + {i, text, if(n in data, do: :text, else: state)} + end) end defp line_status(line, i, view, continued) do diff --git a/lib/rules/workflow_audit.ex b/lib/rules/workflow_audit.ex index db19655b..def1c637 100644 --- a/lib/rules/workflow_audit.ex +++ b/lib/rules/workflow_audit.ex @@ -1825,7 +1825,7 @@ defmodule Hypatia.Rules.WorkflowAudit do {_, v} when is_binary(v) -> case Regex.run(~r/^\s*\*([\w.-]+)\s*$/, v) do [_, name] -> wf026_scalar_alias(name, lines, wf_entry_pos(pos, "toolchain")) - nil -> if wf026_null_value?(v), do: :null, else: {:set, :own} + nil -> if wf026_null_input?(v), do: :null, else: {:set, :own} end _ -> @@ -1899,11 +1899,28 @@ defmodule Hypatia.Rules.WorkflowAudit do # input unreadable, and the finding routes to review. defp wf026_scalar_alias(name, lines, pos) do case wf_anchor_scalar(lines, name, pos) do - {:ok, value} -> if wf026_null_value?(value), do: :null, else: {:set, :own} + {:ok, value} -> if wf026_null_input?(value), do: :null, else: {:set, :own} :error -> {:unreadable, :toolchain_alias, name} end end + # Is a `toolchain` value, as written, no input the action can read? Its + # node properties come off first: an anchor names the node, not its value + # (`&channel ""` is the empty string, `&channel null` null), and a tag + # says how the scalar resolves — `!!null` always to null, `!!str` to the + # string it spells, so `!!str ""` is still empty but `!!str null` is the + # string null, an input. + defp wf026_null_input?(value) do + [_, props, scalar] = Regex.run(~r/^\s*((?:[&!]\S*(?:\s+|$))*)(.*)$/s, value) + tags = Regex.scan(~r/!\S*/, props) |> List.flatten() + + cond do + "!!null" in tags -> true + "!!str" in tags -> String.trim(scalar) in ["", ~s(""), "''"] + true -> wf026_null_value?(scalar) + end + end + # YAML's null: nothing, `~`, or `null` in the three spellings the core # schema admits, unquoted. `"null"` is the string null, an input the # action receives; `""` and `''` are the empty string, which it does not. @@ -1986,10 +2003,11 @@ defmodule Hypatia.Rules.WorkflowAudit do cond do m = Regex.run( - ~r/^\s+(?:![^\s]*\s+)*("(?:[^"\\]|\\.)*"|'(?:[^']|'')*'|[^\s,}\]#][^,}\]#]*?)\s*(?:[,}\]]|\s#|$)/, + ~r/^\s+((?:![^\s]*\s+)*)("(?:[^"\\]|\\.)*"|'(?:[^']|'')*'|[^\s,}\]#][^,}\]#]*?)\s*(?:[,}\]]|\s#|$)/, rest ) -> - {:ok, Enum.at(m, 1)} + # The scalar with its tag (`&x !!str null` is the string null). + {:ok, Enum.at(m, 1) <> Enum.at(m, 2)} # Nothing after the anchor but a tag or a comment: null, unless a # deeper line continues the scalar. @@ -2111,8 +2129,16 @@ defmodule Hypatia.Rules.WorkflowAudit do # A value on the lines below — nothing on the key's line, or a # block scalar's indicator (`toolchain: >-`) — is what those # lines hold: an empty block scalar is the empty string. + # So is one with nothing on the line but node properties + # (`toolchain: &channel`), kept in front of what is below. + props = Regex.run(~r/^(?:[&!]\S*(?:\s+|$))*/, v) |> hd() continued? = v == "" or Regex.match?(~r/^(?:[&!]\S*\s+)*[|>][-+0-9]*$/, v) - [{k, if(continued?, do: wf_continued_value(children, i, child_indent), else: v)}] + + cond do + continued? -> [{k, wf_continued_value(children, i, child_indent)}] + props == v -> [{k, props <> " " <> wf_continued_value(children, i, child_indent)}] + true -> [{k, v}] + end nil -> [] @@ -2332,22 +2358,37 @@ defmodule Hypatia.Rules.WorkflowAudit do end) end - defp wf_flow_entry_starts(interior) do - interior - |> :binary.bin_to_list() - |> Enum.with_index() - |> Enum.reduce({[0], 0, nil}, fn - {c, _}, {acc, d, nil} when c in [?", ?'] -> {acc, d, c} - {c, _}, {acc, d, nil} when c in [?{, ?[] -> {acc, d + 1, nil} - {c, _}, {acc, d, nil} when c in [?}, ?]] -> {acc, d - 1, nil} - {?,, i}, {acc, 0, nil} -> {[i + 1 | acc], 0, nil} - {_, _}, {acc, d, nil} -> {acc, d, nil} - {q, _}, {acc, d, q} -> {acc, d, nil} - {_, _}, {acc, d, q} -> {acc, d, q} - end) - |> elem(0) - |> Enum.reverse() - end + # Where each top-level entry of a flow interior starts: byte 0, and past + # every comma outside quotes and nested brackets. Inside double quotes a + # `\` escapes the byte after it (`"say \"hi\", then"` is one scalar); a + # doubled `''` inside single quotes closes and reopens with nothing + # between, so it hides no comma either. + defp wf_flow_entry_starts(interior), + do: interior |> :binary.bin_to_list() |> wf_entry_starts(0, 0, nil, [0]) |> Enum.reverse() + + defp wf_entry_starts([], _i, _d, _q, acc), do: acc + + defp wf_entry_starts([?\\, _ | rest], i, d, ?", acc), + do: wf_entry_starts(rest, i + 2, d, ?", acc) + + defp wf_entry_starts([q | rest], i, d, q, acc), do: wf_entry_starts(rest, i + 1, d, nil, acc) + + defp wf_entry_starts([_ | rest], i, d, q, acc) when q != nil, + do: wf_entry_starts(rest, i + 1, d, q, acc) + + defp wf_entry_starts([c | rest], i, d, nil, acc) when c in [?", ?'], + do: wf_entry_starts(rest, i + 1, d, c, acc) + + defp wf_entry_starts([c | rest], i, d, nil, acc) when c in [?{, ?[], + do: wf_entry_starts(rest, i + 1, d + 1, nil, acc) + + defp wf_entry_starts([c | rest], i, d, nil, acc) when c in [?}, ?]], + do: wf_entry_starts(rest, i + 1, d - 1, nil, acc) + + defp wf_entry_starts([?, | rest], i, 0, nil, acc), + do: wf_entry_starts(rest, i + 1, 0, nil, [i + 1 | acc]) + + defp wf_entry_starts([_ | rest], i, d, nil, acc), do: wf_entry_starts(rest, i + 1, d, nil, acc) # Offset `at` of a flow interior that `wf_flow_interior/3` read from the # `{` at `open_col` of line `number`, as `{line, column}` in the file. diff --git a/test/rules/pin_integrity_test.exs b/test/rules/pin_integrity_test.exs index d0b97db0..17072f58 100644 --- a/test/rules/pin_integrity_test.exs +++ b/test/rules/pin_integrity_test.exs @@ -461,6 +461,59 @@ defmodule Hypatia.Rules.PinIntegrityTest do assert [%{line: 2}] = PI.pi001_denylisted_pin(flow, @policy) end + test "a block uses: key under no step or job is data, not a site" do + # An input named `uses` is the step's data, as `with: { uses: ... }` + # is: PI001 does not report it and the repair leaves it as is. + input = + "jobs:\n b:\n steps:\n - uses: some/wrapper@v1\n with:\n" <> + " uses: github/codeql-action/init@#{@poison}\n" + + assert PI.pi001_denylisted_pin(input, @policy) == [] + assert {:ok, ^input, []} = PI.substitute_denylisted_pins(input, @policy) + + # Controls: a step's own key, a job's own key, a composite action's + # step, and a line whose ancestry the file does not show (judged on + # its own, as before). + step = + "jobs:\n b:\n steps:\n - name: init\n uses: github/codeql-action/init@#{@poison}\n" + + job = "jobs:\n b:\n uses: github/codeql-action/init@#{@poison}\n" + + composite = + "runs:\n using: composite\n steps:\n - uses: github/codeql-action/init@#{@poison}\n" + + fragment = " - uses: github/codeql-action/init@#{@poison}\n" + + for {text, line} <- [{step, 5}, {job, 3}, {composite, 4}, {fragment, 1}] do + assert [%{line: ^line}] = PI.pi001_denylisted_pin(text, @policy), text + end + end + + test "an input named uses holds no site below its key or through its alias" do + input = + "jobs:\n b:\n steps:\n - uses: some/wrapper@v1\n with:\n" <> + " uses: github/codeql-action/init@#{@poison}\n" + + continued = String.replace(input, "uses: github/", "uses:\n github/") + + aliased = + "env:\n BAD: &bad github/codeql-action/init@#{@poison}\n" <> + String.replace(input, "uses: github/codeql-action/init@#{@poison}", "uses: *bad") + + assert PI.pi001_denylisted_pin(continued, @policy) == [] + assert PI.pi001_denylisted_pin(aliased, @policy) == [] + end + + test "a list item is a step only in a job's steps:" do + # A sequence another key holds, and a `steps:` inside a step's input. + listed = + "jobs:\n b:\n steps:\n - uses: some/runner@v1\n with:\n" <> + " list:\n - uses: github/codeql-action/init@#{@poison}\n" + + assert PI.pi001_denylisted_pin(listed, @policy) == [] + assert PI.pi001_denylisted_pin(String.replace(listed, "list:", "steps:"), @policy) == [] + end + test "a KYAML flow step on its own line is still a site" do content = "steps: [\n { name: \"x\", run: \"echo 'a\" },\n" <> diff --git a/test/rules/pr_automerge_test.exs b/test/rules/pr_automerge_test.exs index 86752c65..7c496b91 100644 --- a/test/rules/pr_automerge_test.exs +++ b/test/rules/pr_automerge_test.exs @@ -636,6 +636,32 @@ defmodule Hypatia.Rules.PrAutomergeTest do assert live_decision.disposition == :close_poison_only end + test "an input named uses under a step is data: no poison site, no pin-only change" do + patch = """ + @@ -12,3 +12,3 @@ + - uses: some/wrapper@v1 + with: + - uses: github/codeql-action/init@#{@good} + + uses: github/codeql-action/init@#{@poison} + """ + + f = file(".github/workflows/ci.yml", patch) + assert PA.poison_sites([f], @policy) == [] + refute PA.pin_lines_only?(f) + refute PA.classify(pr([f]), @policy).disposition == :close_poison_only + + # Control: the step's own key at the same depth is the pin it was. + live = """ + @@ -12,3 +12,3 @@ + - name: init + - uses: github/codeql-action/init@#{@good} + + uses: github/codeql-action/init@#{@poison} + """ + + assert PA.classify(pr([file(".github/workflows/ci.yml", live)]), @policy).disposition == + :close_poison_only + end + test "a pin inside a multi-line quoted run: is text; one the hunk never closes is uncertain" do # The quote opens on a context line and closes inside the hunk: every # line between is string text, never a site. diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs index 8ee49bbd..2a98a883 100644 --- a/test/rules/workflow_audit_pins_test.exs +++ b/test/rules/workflow_audit_pins_test.exs @@ -1037,6 +1037,65 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => configured}) end + test "node properties on a toolchain value are not the value" do + # An anchor names the node and `!!str ""` is still the empty string: + # none of these is an input. `!!str null` is the string null, one. + wf = """ + jobs: + check: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + toolchain: &channel "" + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + toolchain: !!str "" + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: { toolchain: &channel null } + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + with: + toolchain: !!str null + """ + + assert [%{line: 4}, %{line: 7}, %{line: 10}] = + WorkflowAudit.check_sha_pinned_toolchain_without_input(%{"rust.yml" => wf}) + end + + test "a toolchain value of nothing but an anchor is null, or the value below it" do + bare = @rust_step <> " with:\n toolchain: &channel\n" + assert [%{line: 4}] = wf026(bare) + + # Control: the anchored scalar continued on the line below is the input. + assert [] == wf026(bare <> " nightly\n") + end + + test "an anchored toolchain value keeps its tag through an alias" do + aliased = fn value -> + "x: &e #{value}\n" <> @rust_step <> " with:\n toolchain: *e\n" + end + + # `!!str null` is the string null, an input; `!!str ""` is still empty. + assert [] == wf026(aliased.("!!str null")) + assert [%{line: 5}] = wf026(aliased.(~s(!!str ""))) + end + + test "an escaped quote in a flow step's name hides none of its entries" do + # `"a\"b, c"` is one scalar: `with: *inputs` is the step's entry and + # names the empty mapping before it, not the configured `&inputs` + # defined later on the same line. + wf = """ + x: &inputs {} + jobs: + check: + steps: [ { name: "a\\"b, c", uses: dtolnay/rust-toolchain@#{@toolchain_sha}, with: *inputs }, { run: echo, env: &inputs {toolchain: nightly} } ] + """ + + assert [%{line: 4}] = wf026(wf) + + # Control: with the configured anchor first, the alias names it. + assert [] == wf026(String.replace(wf, "x: &inputs {}", "x: &inputs {toolchain: stable}")) + end + test "a toolchain: key with no value is no input: null and ~ included" do wf = """ jobs: From a6180b19376cc124ede6f60c7bebc4a600d014b0 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 08:56:01 +0000 Subject: [PATCH 44/47] fix(pins): whole steps and jobs given as aliases; anchors after JSON colons, continued in flow, read to their full name; WF026 steps need a job; sweep confirms bare denied SHAs - PinSites.node_alias_sites/1: a step or job given whole as an alias (`- *step`, `steps: [*step]`, `rust: *job`) runs the `uses:` of the mapping its anchor holds (flow on the anchor's line, or block below), a site even where that mapping sits in data. PI001, the repair and the PR scanner read it. - PinSites anchors: a name runs to whitespace or a flow indicator (`&bad/x`); inside flow a node starts right after a quoted key's colon (`{"BAD":&bad ...}`); an anchored scalar continued inside flow may sit at any indentation. - PR scanner: each hunk side resolves `uses: *alias` and whole-node aliases to the anchor's line, so a re-pinned anchor is a poison site. - WF026: a list item is a step only under a job's `steps:` (or a composite `runs:`); alias names and JSON-colon anchors read as above. - estate-pin-integrity.sh: phase 2 confirms a denylisted SHA as a bounded token anywhere (a lock's `commit: 'sha1-'`), not only after `@`. - tests: 10 new, each failing on 3b78d55. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- lib/rules/pin_integrity.ex | 13 ++ lib/rules/pin_sites.ex | 225 ++++++++++++++++++++++-- lib/rules/pr_automerge.ex | 31 +++- lib/rules/workflow_audit.ex | 34 ++-- scripts/sweeps/estate-pin-integrity.sh | 9 +- test/rules/pin_integrity_test.exs | 68 +++++++ test/rules/pr_automerge_test.exs | 41 +++++ test/rules/workflow_audit_pins_test.exs | 36 ++++ 8 files changed, 421 insertions(+), 36 deletions(-) diff --git a/lib/rules/pin_integrity.ex b/lib/rules/pin_integrity.ex index f4a1dac3..e262c39b 100644 --- a/lib/rules/pin_integrity.ex +++ b/lib/rules/pin_integrity.ex @@ -113,11 +113,24 @@ defmodule Hypatia.Rules.PinIntegrity do ) end + # A step or job given whole as an alias (`- *step`) runs the `uses:` of + # the mapping its anchor holds — a site even where that mapping sits in + # data (a matrix entry), where a repair rewrites it. + node_aliased = + for %{anchor_line: number, site: site} <- PinSites.node_alias_sites(content) do + site_entry( + site, + number, + masked |> Enum.at(number - 1) |> PinSites.comment() |> String.trim() + ) + end + numbered |> Enum.reject(fn {_, number} -> number in data end) |> Enum.flat_map(fn {{line, prefix}, number} -> line_sites(line, number, prefix) end) |> Kernel.++(continued) |> Kernel.++(aliased) + |> Kernel.++(node_aliased) |> Enum.uniq_by(&{&1.line, &1.ref_at}) |> Enum.sort_by(&{&1.line, &1.ref_at}) end diff --git a/lib/rules/pin_sites.ex b/lib/rules/pin_sites.ex index 34241383..f25e1890 100644 --- a/lib/rules/pin_sites.ex +++ b/lib/rules/pin_sites.ex @@ -245,18 +245,23 @@ defmodule Hypatia.Rules.PinSites do # An anchor with nothing after it on its line but a tag or a comment # (`BAD: &bad`) names the scalar YAML continues on the next line with - # content, deeper than the key (` github/codeql-action/init@`): its - # site is there, `:site_line` that line. + # content (` github/codeql-action/init@`): its site is there, + # `:site_line` that line. In block context the scalar must sit deeper than + # the key, and is all the line holds; inside a flow collection it may + # stand at any indentation, even column 0, up to the `,`, `}` or `]` that + # ends it. defp continued_anchor(%{site: nil, rest: rest} = d, line, below) do + value_re = + if d.flow, + do: + ~r/^\s*(?:![^\s]*\s+)*["']?([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)["']?\s*(?:[,}\]]|#|$)/, + else: + ~r/^\s*(?:![^\s]*\s+)*["']?([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)["']?\s*(?:#.*)?$/ + with true <- Regex.match?(~r/^\s*(?:![^\s]*\s*)*(?:#.*)?$/, rest), {next, n} <- Enum.find(below, fn {l, _} -> String.trim(head(l)) != "" end), - true <- indent_of(next) > indent_of(line), - [_, {a_at, a_len}, {r_at, r_len}] <- - Regex.run( - ~r/^\s*(?:![^\s]*\s+)*["']?([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)["']?\s*(?:#.*)?$/, - next, - return: :index - ) do + true <- d.flow or indent_of(next) > indent_of(line), + [_, {a_at, a_len}, {r_at, r_len}] <- Regex.run(value_re, next, return: :index) do site = %{ action: binary_part(next, a_at, a_len), ref: binary_part(next, r_at, r_len), @@ -273,22 +278,25 @@ defmodule Hypatia.Rules.PinSites do defp continued_anchor(d, _line, _below), do: d # The anchors `line` defines where a node starts (outside quotes and the - # comment), each `%{name, at, site}`: the site its scalar value is, or nil. - # A node starts at the line's start or after its list dashes, after - # `key: `, and — inside a flow collection only (one the line opens, or - # `flow_prefix` says it sits in) — after `[`, `{` or `,`, each past any - # tag. `NOTE: a, &x b` in block context is one plain scalar: no anchor. + # comment), each `%{name, at, site, rest, flow}`: the site its scalar value + # is, or nil; the text after it; whether it sits inside a flow collection. + # A name runs to whitespace or a flow indicator (`&rust/tool`). A node + # starts at the line's start or after its list dashes, after `key: `, and + # — inside a flow collection only (one the line opens, or `flow_prefix` + # says it sits in) — after `[`, `{` or `,`, or directly after the colon of + # a quoted key (`{"BAD":&bad ...}`, JSON-style), each past any tag. + # `NOTE: a, &x b` in block context is one plain scalar: no anchor. defp anchor_defs(line, flow_prefix) do masked = mask(line) - ~r/&([\w.-]+)/ + ~r/&([^\s,\[\]{}]+)/ |> Regex.scan(masked, return: :index) |> Enum.filter(fn [{at, _}, _] -> before = binary_part(masked, 0, at) Regex.match?(~r/(?:^\s*(?:-\s+)*|:\s+)(?:![^\s]*\s+)*$/, before) or ((flow_prefix != "" or String.contains?(before, ["[", "{"])) and - Regex.match?(~r/[\[{,]\s*(?:![^\s]*\s+)*$/, before)) + Regex.match?(~r/(?:[\[{,]|["']:)\s*(?:![^\s]*\s+)*$/, before)) end) |> Enum.map(fn [{at, len}, {n_at, n_len}] -> rest_at = at + len @@ -312,7 +320,16 @@ defmodule Hypatia.Rules.PinSites do nil end - %{name: binary_part(masked, n_at, n_len), at: at, site: site, rest: rest} + before = binary_part(masked, 0, at) + open = length(Regex.scan(~r/[\[{]/, before)) - length(Regex.scan(~r/[\]}]/, before)) + + %{ + name: binary_part(masked, n_at, n_len), + at: at, + site: site, + rest: rest, + flow: flow_prefix != "" or open > 0 + } end) end @@ -321,7 +338,7 @@ defmodule Hypatia.Rules.PinSites do defp uses_aliases(line, flow_prefix) do masked = mask(line) - ~r/["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*\*([\w.-]+)/ + ~r/["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*\*([^\s,\[\]{}]+)/ |> Regex.scan(masked, return: :index) |> Enum.flat_map(fn [{at, len}, {n_at, n_len}] -> star = n_at - 1 @@ -346,6 +363,178 @@ defmodule Hypatia.Rules.PinSites do |> Enum.max_by(&{&1.line, &1.at}, fn -> nil end) end + @doc """ + Every step or job given whole as an alias — `- *step` in a job's + `steps:`, `*step` as an entry of a `steps: [ ... ]`, `rust: *job` under + `jobs:` — resolved to the `uses:` of the mapping its anchor holds: + `%{line, anchor_line, site}`, the alias' line, the line of that `uses:` + key, and its site as `sites/1` returns one (offsets into that line). + YAML resolves the alias to that very node, so its `uses:` is the step's + or the job's action even where the anchor sits in data (`include: - + &step { uses: ... }` in a matrix). The anchored mapping is read as a flow + mapping on the anchor's own line (`&step { uses: ... }`) or in block form + (`- &step`, `key: &step`, its keys on the deeper lines below); any other + shape names no site here. A `- *step` whose list owner the lines above + do not show is judged a step, as a lone `- uses:` line is. + """ + @spec node_alias_sites(String.t()) :: [map()] + def node_alias_sites(content) when is_binary(content) do + lines = mask_lines(content) + numbered = lines |> Enum.zip(flow_prefixes(content)) |> Enum.with_index(1) + texts = List.to_tuple(lines) + + rows = + content + |> line_states() + |> Enum.zip_with(line_depths(content), &block_row/2) + |> List.to_tuple() + + defs = + for {{line, prefix}, n} <- numbered, + d <- anchor_defs(line, prefix), + do: Map.put(d, :line, n) + + for {{line, prefix}, n} <- numbered, + {name, at} <- node_aliases(line, prefix, rows, n - 1), + %{} = d <- [latest_anchor(defs, name, n, at)], + {uses_line, site} <- anchored_uses(d, texts, rows) do + %{line: n, anchor_line: uses_line, site: site} + end + end + + # The aliases on `line` that stand for a whole step or job, as `{name, + # offset of the "*"}`: a block list item that is nothing but the alias, in + # a sequence `steps:` holds (or one whose owner is not shown); a block key + # of `jobs:` whose value is the alias; or a flow entry whose brackets make + # it an item of a job's step sequence, or a job's value in `jobs: { ... }`. + defp node_aliases(line, prefix, rows, i) do + masked = mask(line) + row = elem(rows, i) + + item = Regex.run(~r/^(\s*-\s+)\*([^\s,\[\]{}]+)\s*$/, masked) + value = Regex.run(~r/^(\s*["']?[^"':\s]+["']?\s*:\s+)\*([^\s,\[\]{}]+)\s*$/, masked) + + block = + case {row, item, value} do + {%{dash: d}, [_, lead, name], _} when d != nil -> + if sequence_owner(rows, seq_owner(rows, i, d)) != :data, + do: [{name, byte_size(lead)}], + else: [] + + {%{dash: nil}, _, [_, lead, name]} -> + if match?({:value, _, "jobs"}, mapping_of(rows, i)), + do: [{name, byte_size(lead)}], + else: [] + + _ -> + [] + end + + flow = + ~r/([\[{,:])\s*\*([^\s,\[\]{}]+)/ + |> Regex.scan(masked, return: :index) + |> Enum.flat_map(fn [_, {c_at, _}, {n_at, n_len}] -> + star = n_at - 1 + stack = flow_stack(prefix <> binary_part(masked, 0, star)) + + node? = + if binary_part(masked, c_at, 1) == ":", + do: match?([{"{", "jobs"} | _], stack), + else: alias_step_stack?(stack) + + if node?, do: [{binary_part(masked, n_at, n_len), star}], else: [] + end) + + block ++ flow + end + + # An entry directly in a `[` that `steps:` opens: the whole stack the text + # shows (a block job's `steps: [*step]`), or one that goes on through a + # job key into `jobs`. + defp alias_step_stack?([{"[", "steps"}]), do: true + + defp alias_step_stack?([{"[", "steps"}, {"{", job}, {"{", "jobs"} | _]) when is_binary(job), + do: true + + defp alias_step_stack?(_), do: false + + # The `uses:` of the mapping the anchor `d` holds, as `[{line, site}]`: a + # flow mapping opening on the anchor's own line, read to its top-level + # `uses` entry, or a block mapping on the lines below, read to its `uses:` + # key. + defp anchored_uses(d, texts, rows) do + line = elem(texts, d.line - 1) + rest_at = byte_size(line) - byte_size(d.rest) + + cond do + m = Regex.run(~r/^\s*(?:![^\s]*\s+)*\{/, d.rest, return: :index) -> + [{0, len}] = m + + case top_uses(mask(line), rest_at + len - 1) do + nil -> [] + site -> [{d.line, site}] + end + + Regex.match?(~r/^\s*(?:![^\s]*\s*)*(?:#.*)?$/, d.rest) -> + block_uses(d.line, texts, rows) + + true -> + [] + end + end + + # The site of the top-level `uses` entry of the flow mapping whose `{` is + # at byte `open` of masked line `m`, or nil when the line names none: an + # entry after that `{` or a `,` with no other bracket open between. + defp top_uses(m, open) do + ~r/[{,]\s*["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*["']?([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)/ + |> Regex.scan(m, return: :index, offset: open) + |> Enum.find_value(fn [{at, _}, {a_at, a_len}, {r_at, r_len}] -> + between = binary_part(m, open, at - open + 1) + depth = length(Regex.scan(~r/[\[{]/, between)) - length(Regex.scan(~r/[\]}]/, between)) + + if depth == 1 do + %{ + action: binary_part(m, a_at, a_len), + ref: binary_part(m, r_at, r_len), + action_at: a_at, + ref_at: r_at + } + end + end) + end + + # A block mapping anchored with nothing after the anchor (`- &step`, + # `step: &step`) on row `n`: its keys are the rows below deeper than the + # dash or the key, the first of them setting their column; the one named + # `uses` holds the site(s). + defp block_uses(n, texts, rows) do + case elem(rows, n - 1) do + %{} = row -> + parent = if row.key == nil, do: row.dash || row.indent, else: row.key_col + + children = + n..(tuple_size(rows) - 1)//1 + |> Enum.map(&{&1, elem(rows, &1)}) + |> Enum.reject(fn {_, r} -> r == :skip end) + |> Enum.take_while(fn {_, r} -> r.indent > parent end) + + case children do + [] -> + [] + + [{_, first} | _] -> + for {j, %{indent: ind, dash: nil, key: "uses"}} <- children, + ind == first.indent, + site <- sites(elem(texts, j)), + do: {j + 1, site} + end + + _ -> + [] + end + end + @doc """ The one-based numbers of the lines of `content` whose leading block `uses:` key the indentation above it proves is neither a step's nor a job's own: diff --git a/lib/rules/pr_automerge.ex b/lib/rules/pr_automerge.ex index 930dc6ef..cea2d944 100644 --- a/lib/rules/pr_automerge.ex +++ b/lib/rules/pr_automerge.ex @@ -643,8 +643,8 @@ defmodule Hypatia.Rules.PrAutomerge do hunk = Enum.reject(hunk, &Regex.match?(~r/^[-+]{3}/, &1)) old = side_view(hunk, "-") new = side_view(hunk, "+") - old_cont = side_continuations(old) - new_cont = side_continuations(new) + old_cont = Map.merge(side_continuations(old), side_aliases(old), fn _, a, b -> a ++ b end) + new_cont = Map.merge(side_continuations(new), side_aliases(new), fn _, a, b -> a ++ b end) hunk |> Enum.with_index() @@ -675,6 +675,28 @@ defmodule Hypatia.Rules.PrAutomerge do end end + # The anchored values one side of a hunk shows an alias running, as + # `%{hunk_index => [pin]}` on the anchor's line: `- uses: *bad` in context + # with `BAD: &bad @` changed (`PinSites.alias_sites/2`), or a + # step given whole as an alias whose anchored mapping's `uses:` changed + # (`PinSites.node_alias_sites/1`). The re-pinned anchor is the site. One + # that is already a site on its own line is counted there, not twice. + defp side_aliases(view) do + texts = for {_, text, state} <- view, do: if(state == :text, do: "", else: text) + side = Enum.join(texts, "\n") + index = view |> Enum.map(&elem(&1, 0)) |> List.to_tuple() + masked = PinSites.mask_lines(side) + + (PinSites.alias_sites(masked, PinSites.flow_prefixes(side)) ++ PinSites.node_alias_sites(side)) + |> Enum.reject(fn %{anchor_line: n, site: site} -> + texts |> Enum.at(n - 1) |> PinSites.sites() |> Enum.any?(&(&1.ref_at == site.ref_at)) + end) + |> Enum.group_by( + &elem(index, &1.anchor_line - 1), + &%{action: &1.site.action, base: action_base(&1.site.action), ref: &1.site.ref} + ) + end + # One side of a hunk (context lines plus the changed lines of `mark`), in # order, as `{hunk_index, text, state}` — `state` as # `PinSites.line_states/1` reads the side: `:yaml`, `:text` or `:open`; a @@ -691,7 +713,10 @@ defmodule Hypatia.Rules.PrAutomerge do side = Enum.map_join(entries, "\n", &elem(&1, 1)) states = PinSites.line_states(side) - data = PinSites.data_uses_lines(side) + # Unless a step or job given whole as an alias runs the mapping it sits + # in (`- *step`, the mapping anchored in a matrix entry). + run = for %{anchor_line: n} <- PinSites.node_alias_sites(side), into: MapSet.new(), do: n + data = MapSet.difference(PinSites.data_uses_lines(side), run) entries |> Enum.zip(states) diff --git a/lib/rules/workflow_audit.ex b/lib/rules/workflow_audit.ex index def1c637..79031982 100644 --- a/lib/rules/workflow_audit.ex +++ b/lib/rules/workflow_audit.ex @@ -1558,11 +1558,19 @@ defmodule Hypatia.Rules.WorkflowAudit do {step, wf026_step_key_indent(step), opener} end - # A list item is a step only when its parent key is `steps:`; an item - # inside `run: |`, or inside a fixture kept in `env:`, is text however much - # it looks like a step, and must never send the fleet to rewrite scalar data. + # A list item is a step only when its parent key is a job's `steps:`, or + # a composite action's under `runs:`. An item inside `run: |`, inside a + # fixture kept in `env:`, or of a `steps:` some other mapping holds (a + # matrix entry's `fixture: { steps: ... }`) is data however much it looks + # like a step, and must never send the fleet to rewrite it. A `steps:` + # with nothing above it (a fragment) is still read as a step list. defp wf026_under_steps?(lines, opener) do - match?(["steps" | _], wf_ancestor_keys(lines, opener)) + case wf_ancestor_keys(lines, opener) do + ["steps"] -> true + ["steps", "runs" | _] -> true + ["steps", job, "jobs" | _] when is_binary(job) -> true + _ -> false + end end # The mapping keys enclosing the line at `number`, nearest first — list @@ -1727,8 +1735,8 @@ defmodule Hypatia.Rules.WorkflowAudit do defp wf026_with_value(value, line, rest, key_indent, lines, number) do cond do # `with: *inputs` — the mapping the latest `&inputs` before it anchors. - match?([_ | _], Regex.run(~r/^\*([\w.-]+)$/, value)) -> - [_, name] = Regex.run(~r/^\*([\w.-]+)$/, value) + match?([_ | _], Regex.run(~r/^\*([^\s,\[\]{}]+)$/, value)) -> + [_, name] = Regex.run(~r/^\*([^\s,\[\]{}]+)$/, value) [{col, _}] = Regex.run(~r/\*/, line, return: :index) wf026_alias_verdict(name, lines, :alias, {number, col}) @@ -1770,8 +1778,8 @@ defmodule Hypatia.Rules.WorkflowAudit do value = String.trim(value) cond do - match?([_ | _], Regex.run(~r/^\*([\w.-]+)$/, value)) -> - [_, name] = Regex.run(~r/^\*([\w.-]+)$/, value) + match?([_ | _], Regex.run(~r/^\*([^\s,\[\]{}]+)$/, value)) -> + [_, name] = Regex.run(~r/^\*([^\s,\[\]{}]+)$/, value) wf026_alias_verdict(name, lines, :alias, pos) String.contains?(value, "{") -> @@ -1823,7 +1831,7 @@ defmodule Hypatia.Rules.WorkflowAudit do defp wf026_toolchain_in(entries, lines, seen, pos) do case Enum.find(entries, fn {k, _} -> k == "toolchain" end) do {_, v} when is_binary(v) -> - case Regex.run(~r/^\s*\*([\w.-]+)\s*$/, v) do + case Regex.run(~r/^\s*\*([^\s,\[\]{}]+)\s*$/, v) do [_, name] -> wf026_scalar_alias(name, lines, wf_entry_pos(pos, "toolchain")) nil -> if wf026_null_input?(v), do: :null, else: {:set, :own} end @@ -1866,10 +1874,10 @@ defmodule Hypatia.Rules.WorkflowAudit do v = String.trim(value) cond do - m = Regex.run(~r/^\*([\w.-]+)$/, v) -> + m = Regex.run(~r/^\*([^\s,\[\]{}]+)$/, v) -> {:ok, [Enum.at(m, 1)]} - m = Regex.run(~r/^\[\s*(\*[\w.-]+(?:\s*,\s*\*[\w.-]+)*)\s*,?\s*\]$/, v) -> + m = Regex.run(~r/^\[\s*(\*[^\s,\[\]{}]+(?:\s*,\s*\*[^\s,\[\]{}]+)*)\s*,?\s*\]$/, v) -> names = m |> Enum.at(1) @@ -2059,10 +2067,12 @@ defmodule Hypatia.Rules.WorkflowAudit do |> List.last() end + # Inside a flow collection a node also starts directly after the colon of + # a quoted key, JSON-style (`{"BAD":&bad "a/b@x"}`). defp wf_node_start?(before, depth) do Regex.match?(~r/(?:^\s*(?:-\s+)*|:\s+)(?:![^\s]*\s+)*$/, before) or ((depth > 0 or String.contains?(before, ["[", "{"])) and - Regex.match?(~r/[\[{,]\s*(?:![^\s]*\s+)*$/, before)) + Regex.match?(~r/(?:[\[{,]|["']:)\s*(?:![^\s]*\s+)*$/, before)) end # The line of each direct child of the block mapping whose key sits at diff --git a/scripts/sweeps/estate-pin-integrity.sh b/scripts/sweeps/estate-pin-integrity.sh index 144421e3..e3a1b804 100755 --- a/scripts/sweeps/estate-pin-integrity.sh +++ b/scripts/sweeps/estate-pin-integrity.sh @@ -315,11 +315,14 @@ replace_site() { # # spelling, or a blocked version written after ITS action # (`github/codeql-action/init@v4.38.1` in a `run: |` script) — phase 1 found # the candidate by that token, so phase 2 must see it too, never call the -# file clean. A blocked version is blocked for its own action only, as at a -# site (`denylisted_pair`): `some/action@v4.38.1` is no finding. +# file clean. A SHA is a token wherever it stands, bounded by anything but +# a letter or digit: after `@` in a reference, but also as a lock record's +# `ref:` or `commit: 'sha1-'` under a key that names a tag. A blocked +# version is blocked for its own action only, as at a site +# (`denylisted_pair`): `some/action@v4.38.1` is no finding. line_has_denied_token() { local tok t - if [ -n "$SHA_ALTERNATION" ] && printf '%s\n' "$1" | grep -qiE "@(${SHA_ALTERNATION})(${REF_END})"; then + if [ -n "$SHA_ALTERNATION" ] && printf '%s\n' "$1" | grep -qiE "(^|[^0-9A-Za-z])(${SHA_ALTERNATION})([^0-9A-Za-z]|\$)"; then return 0 fi [ ${#DENY_VERSIONS[@]} -gt 0 ] || return 1 diff --git a/test/rules/pin_integrity_test.exs b/test/rules/pin_integrity_test.exs index 17072f58..18da36d3 100644 --- a/test/rules/pin_integrity_test.exs +++ b/test/rules/pin_integrity_test.exs @@ -514,6 +514,74 @@ defmodule Hypatia.Rules.PinIntegrityTest do assert PI.pi001_denylisted_pin(String.replace(listed, "list:", "steps:"), @policy) == [] end + test "an anchor directly after a quoted key's colon in a flow mapping is the anchor" do + # JSON-style `{"BAD":&bad ...}`: the value starts right after the colon. + content = + "env: {\"BAD\":&bad \"github/codeql-action/init@#{@poison}\"}\n" <> + "jobs:\n a:\n steps:\n - uses: *bad\n" + + assert [%{line: 1}] = PI.pi001_denylisted_pin(content, @policy) + assert {:ok, fixed, [_]} = PI.substitute_denylisted_pins(content, @policy) + refute fixed =~ @poison + end + + test "an anchored scalar continued inside a flow mapping may start at column 0" do + content = + "env: {\n BAD: &bad\ngithub/codeql-action/init@#{@poison}\n}\n" <> + "jobs:\n a:\n steps:\n - uses: *bad\n" + + assert [%{line: 3}] = PI.pi001_denylisted_pin(content, @policy) + assert {:ok, fixed, [_]} = PI.substitute_denylisted_pins(content, @policy) + refute fixed =~ @poison + end + + test "a step given whole as an alias of a flow mapping runs that mapping's uses:" do + # The anchored mapping sits in a matrix entry, which is data: the + # alias in `steps:` makes it the step. + matrix = + "jobs:\n a:\n strategy:\n matrix:\n include:\n" <> + " - step: &step { uses: github/codeql-action/init@#{@poison} }\n" <> + " steps:\n - *step\n" + + assert [%{line: 6}] = PI.pi001_denylisted_pin(matrix, @policy) + assert {:ok, fixed, [_]} = PI.substitute_denylisted_pins(matrix, @policy) + refute fixed =~ @poison + + flow_seq = + "x: &step { uses: github/codeql-action/init@#{@poison} }\n" <> + "jobs:\n a:\n steps: [*step]\n" + + assert [%{line: 1}] = PI.pi001_denylisted_pin(flow_seq, @policy) + + # Control: with no alias in a step position, the matrix entry is data. + unused = String.replace(matrix, " - *step\n", " - run: echo\n") + assert PI.pi001_denylisted_pin(unused, @policy) == [] + end + + test "a step or job given whole as an alias of a block mapping runs its uses:" do + step = + "x:\n - &step\n uses: github/codeql-action/init@#{@poison}\n" <> + "jobs:\n a:\n steps:\n - *step\n" + + assert [%{line: 3}] = PI.pi001_denylisted_pin(step, @policy) + + # The job's mapping anchored in a matrix entry, data until aliased. + job = + "jobs:\n b:\n strategy:\n matrix:\n include:\n - job: &job\n" <> + " uses: github/codeql-action/init@#{@poison}\n a: *job\n" + + assert [%{line: 7}] = PI.pi001_denylisted_pin(job, @policy) + assert PI.pi001_denylisted_pin(String.replace(job, " a: *job\n", ""), @policy) == [] + end + + test "an anchor name runs to whitespace or a flow indicator" do + content = + "env:\n BAD: &bad/x github/codeql-action/init@#{@poison}\n" <> + "jobs:\n a:\n steps:\n - uses: *bad/x\n" + + assert [%{line: 2}] = PI.pi001_denylisted_pin(content, @policy) + end + test "a KYAML flow step on its own line is still a site" do content = "steps: [\n { name: \"x\", run: \"echo 'a\" },\n" <> diff --git a/test/rules/pr_automerge_test.exs b/test/rules/pr_automerge_test.exs index 7c496b91..1b473c66 100644 --- a/test/rules/pr_automerge_test.exs +++ b/test/rules/pr_automerge_test.exs @@ -662,6 +662,47 @@ defmodule Hypatia.Rules.PrAutomergeTest do :close_poison_only end + test "a re-pinned anchor an unchanged uses: *alias runs is a poison site" do + patch = """ + @@ -1,7 +1,7 @@ + env: + - BAD: &bad github/codeql-action/init@#{@good} + + BAD: &bad github/codeql-action/init@#{@poison} + jobs: + a: + steps: + - uses: *bad + """ + + f = file(".github/workflows/ci.yml", patch) + assert [%{certain: true}] = PA.poison_sites([f], @policy) + + decision = PA.classify(pr([f]), @policy) + assert decision.disposition == :flag + assert decision.blocked_by == "introduces_denylisted_pin_alongside_unmergeable_changes" + end + + test "a step given whole as an alias makes its anchored mapping's re-pin a poison site" do + # The mapping is anchored in a matrix entry (data); `- *step` in the + # job's steps runs it. + for {anchor, changed} <- [ + {" - &step", " uses: github/codeql-action/init@"}, + {nil, " - step: &step { uses: github/codeql-action/init@"} + ] do + close = if anchor, do: "", else: " }" + head = if anchor, do: " #{anchor}\n", else: "" + + patch = + "@@ -1,9 +1,9 @@\n jobs:\n a:\n strategy:\n matrix:\n include:\n" <> + head <> + "-#{changed}#{@good}#{close}\n+#{changed}#{@poison}#{close}\n" <> + " steps:\n - *step\n" + + f = file(".github/workflows/ci.yml", patch) + assert [%{certain: true}] = PA.poison_sites([f], @policy), patch + end + end + test "a pin inside a multi-line quoted run: is text; one the hunk never closes is uncertain" do # The quote opens on a context line and closes inside the hunk: every # line between is string text, never a site. diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs index 2a98a883..281a3bed 100644 --- a/test/rules/workflow_audit_pins_test.exs +++ b/test/rules/workflow_audit_pins_test.exs @@ -1096,6 +1096,42 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do assert [] == wf026(String.replace(wf, "x: &inputs {}", "x: &inputs {toolchain: stable}")) end + test "a steps: list some other mapping holds is data, not a job's steps" do + wf = """ + jobs: + check: + strategy: + matrix: + include: + - fixture: + steps: + - uses: dtolnay/rust-toolchain@#{@toolchain_sha} + steps: + - run: echo + """ + + assert [] == wf026(wf) + + # Control: the job's own steps. + assert [%{line: 4}] = wf026(@rust_step) + end + + test "an alias name runs to whitespace or a flow indicator" do + wf = + "x: &e/x \"\"\n" <> @rust_step <> " with:\n toolchain: *e/x\n" + + assert [%{line: 5, auto_fixable: true}] = wf026(wf) + end + + test "an anchor directly after a quoted key's colon in a flow mapping is the anchor" do + wf = + "x: {\"e\":&e \"\"}\n" <> @rust_step <> " with:\n toolchain: *e\n" + + # The anchor is read, so the empty value it holds routes to the fixer + # rather than to review as an alias of no anchor. + assert [%{line: 5, auto_fixable: true}] = wf026(wf) + end + test "a toolchain: key with no value is no input: null and ~ included" do wf = """ jobs: From c579fef27d64809effc792bb1f692a615f668ed0 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 09:14:40 +0000 Subject: [PATCH 45/47] fix(pins): claims stay with their own action and agree with any resolved endpoint; aliased flow mappings read across lines; WF026 judges alias-run steps; PI004 per workflow; repairs proved free of denied pins - PrAutomerge: a body claim belongs to the action it names or its root repository; a basename-only claim only when no other action of the PR shares that basename, and `other/setup` never borrows `acme/setup`'s claim. With one endpoint resolved, a claim that contradicts it is a conflict, not a verified delta. - PinSites: a mapping anchored as `&step {` is read through the lines below to its closing brace, so `- *step` runs a `uses:` on any of them. - WF026: a `uses:` line a whole-step alias runs is a live step wherever the mapping is anchored (a matrix entry), and routes to review. - PI004: with the workflow's path, only that workflow's own lock entry (ActionsLock.workflow_refs/2) vouches for its pin. - PinIntegrity: a version is matched past one optional `v` only (`vv4.38.1` is another tag); a substitution that leaves any denylisted pin is refused, not logged as an excision. - tests: 7 new, each failing on a6180b1. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- lib/rules/actions_lock.ex | 9 +++ lib/rules/pin_integrity.ex | 46 +++++++++-- lib/rules/pin_sites.ex | 103 +++++++++++++++++++----- lib/rules/pr_automerge.ex | 43 ++++++++-- lib/rules/workflow_audit.ex | 62 ++++++++++---- test/rules/pin_integrity_test.exs | 63 +++++++++++++++ test/rules/pr_automerge_test.exs | 38 +++++++++ test/rules/workflow_audit_pins_test.exs | 22 +++++ 8 files changed, 336 insertions(+), 50 deletions(-) diff --git a/lib/rules/actions_lock.ex b/lib/rules/actions_lock.ex index 9fcec245..1c3484a7 100644 --- a/lib/rules/actions_lock.ex +++ b/lib/rules/actions_lock.ex @@ -108,6 +108,15 @@ defmodule Hypatia.Rules.ActionsLock do def pinned?(_, _, _), do: false + @doc """ + The refs the lock associates with `workflow_path` (`owner/repo@ref` keys), + or nil when its `workflows:` section lists no such workflow. Basenames + resolve under `.github/workflows/`, as for `pinned?/3`. + """ + @spec workflow_refs(t(), String.t()) :: MapSet.t(String.t()) | nil + def workflow_refs(%__MODULE__{} = lock, workflow_path) when is_binary(workflow_path), + do: Map.get(lock.workflows, normalise_workflow_path(workflow_path)) + @doc """ Return dependency refs from a fully validated lock. diff --git a/lib/rules/pin_integrity.ex b/lib/rules/pin_integrity.ex index e262c39b..41b2ed42 100644 --- a/lib/rules/pin_integrity.ex +++ b/lib/rules/pin_integrity.ex @@ -206,9 +206,10 @@ defmodule Hypatia.Rules.PinIntegrity do defp normalise_all(list), do: Enum.map(list, &normalise_ref/1) defp normalise_ref(ref) do - trimmed = ref |> String.trim() |> String.trim_leading("v") + trimmed = ref |> String.trim() |> String.replace_prefix("v", "") - # `v4.38.1` and `4.38.1` are the same claim about the same artefact; a + # `v4.38.1` and `4.38.1` are the same claim about the same artefact — one + # `v` only: `vv4.38.1` is another tag, not the blocked release; a # commit ID in upper-case hex is the same object as the lower-case one # the table carries, so both sides compare in lower case. if Regex.match?(~r/^[0-9a-fA-F]{40}$/, trimmed), do: String.downcase(trimmed), else: trimmed @@ -466,6 +467,20 @@ defmodule Hypatia.Rules.PinIntegrity do |> Enum.reverse() end + # The refs a parsed lock associates with workflow `path`, in the shape + # `locked_refs/1` returns them, or nil when the lock does not parse or + # lists no such workflow. + defp scoped_locked_refs(lock_content, path) do + with {:ok, lock} <- Hypatia.Rules.ActionsLock.parse(lock_content), + %MapSet{} = refs <- Hypatia.Rules.ActionsLock.workflow_refs(lock, path) do + for key <- refs, [action, ref] <- [String.split(key, "@", parts: 2)] do + {ref, {action_base(action), 0}} + end + else + _ -> nil + end + end + # What a ref names, for comparing two spellings of it: a commit ID is one # object in either case (`1C5B…` is `1c5b…`); a symbolic ref (`main`, # `v4`) is a name and compares exactly — `v4` and `4` are two refs. @@ -489,7 +504,14 @@ defmodule Hypatia.Rules.PinIntegrity do def pi004_lock_divergence(workflow_content, lock_content, opts \\ []) do path = Keyword.get(opts, :path, "") repo = Keyword.get(opts, :repo, "") - locked = locked_refs(lock_content) + + # A ref locked for another workflow does not vouch for this one: with the + # workflow's path, the comparison is that workflow's own `workflows:` + # entry. Without a path, or a lock that does not parse or name the + # workflow, every ref the lock holds is the comparison, as before. + locked = + (Keyword.has_key?(opts, :path) && scoped_locked_refs(lock_content, path)) || + locked_refs(lock_content) workflow_content |> pin_sites() @@ -581,11 +603,21 @@ defmodule Hypatia.Rules.PinIntegrity do end) new_content = Enum.join(new_lines, "\n") + new_sites = pin_sites(new_content) + + # The rewrite is proved, not assumed: as many sites as before, and + # none of them denylisted — a `known_good_sha` that is itself + # blocked (by this entry or another for the same action) repairs + # nothing, and is refused rather than logged as an excision. + cond do + length(new_sites) != length(sites) -> + {:error, "substitution would remove a pin site — refusing"} + + Enum.any?(new_sites, &denylisted?(policy, &1.action, &1.ref)) -> + {:error, "substitution leaves a denylisted pin — refusing"} - if length(pin_sites(new_content)) == length(sites) do - {:ok, new_content, excisions} - else - {:error, "substitution would remove a pin site — refusing"} + true -> + {:ok, new_content, excisions} end end end diff --git a/lib/rules/pin_sites.ex b/lib/rules/pin_sites.ex index f25e1890..7aa88d2f 100644 --- a/lib/rules/pin_sites.ex +++ b/lib/rules/pin_sites.ex @@ -459,9 +459,9 @@ defmodule Hypatia.Rules.PinSites do defp alias_step_stack?(_), do: false # The `uses:` of the mapping the anchor `d` holds, as `[{line, site}]`: a - # flow mapping opening on the anchor's own line, read to its top-level - # `uses` entry, or a block mapping on the lines below, read to its `uses:` - # key. + # flow mapping opening on the anchor's line, read through the lines below + # to the brace that closes it, or a block mapping on the lines below, read + # to its `uses:` key. defp anchored_uses(d, texts, rows) do line = elem(texts, d.line - 1) rest_at = byte_size(line) - byte_size(d.rest) @@ -470,9 +470,9 @@ defmodule Hypatia.Rules.PinSites do m = Regex.run(~r/^\s*(?:![^\s]*\s+)*\{/, d.rest, return: :index) -> [{0, len}] = m - case top_uses(mask(line), rest_at + len - 1) do + case flow_mapping_uses(texts, d.line, rest_at + len - 1) do nil -> [] - site -> [{d.line, site}] + found -> [found] end Regex.match?(~r/^\s*(?:![^\s]*\s*)*(?:#.*)?$/, d.rest) -> @@ -483,25 +483,86 @@ defmodule Hypatia.Rules.PinSites do end end - # The site of the top-level `uses` entry of the flow mapping whose `{` is - # at byte `open` of masked line `m`, or nil when the line names none: an - # entry after that `{` or a `,` with no other bracket open between. - defp top_uses(m, open) do - ~r/[{,]\s*["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*["']?([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)/ - |> Regex.scan(m, return: :index, offset: open) - |> Enum.find_value(fn [{at, _}, {a_at, a_len}, {r_at, r_len}] -> - between = binary_part(m, open, at - open + 1) - depth = length(Regex.scan(~r/[\[{]/, between)) - length(Regex.scan(~r/[\]}]/, between)) + # The top-level `uses` entry of the flow mapping whose `{` is at byte + # `open` of line `n`, as `{line, site}`, or nil when the mapping closes + # without one. The mapping is read through the lines below to the brace + # that closes it: an entry starts after its `{` or one of its commas — on + # the same line, or as the last thing a line above held — with no other + # bracket open between. Lines are read masked (`mask/1`), so brackets and + # commas inside quoted text and comments count for nothing. + defp flow_mapping_uses(texts, n, open) do + n..tuple_size(texts)//1 + |> Enum.reduce_while({:open, 0, nil}, fn k, {:open, depth, last} -> + from = if k == n, do: open, else: 0 + + case mapping_line_uses(mask(elem(texts, k - 1)), from, depth, last) do + {:found, site} -> {:halt, {k, site}} + :closed -> {:halt, nil} + open_state -> {:cont, open_state} + end + end) + |> case do + {k, %{} = site} -> {k, site} + _ -> nil + end + end - if depth == 1 do - %{ - action: binary_part(m, a_at, a_len), - ref: binary_part(m, r_at, r_len), - action_at: a_at, - ref_at: r_at - } + defp mapping_line_uses(m, from, depth, last) do + m + |> :binary.bin_to_list() + |> Enum.with_index() + |> Enum.drop(from) + |> Enum.reduce_while({:open, depth, last}, fn {c, i}, {:open, d, l} -> + cond do + c in [?{, ?[] -> + {:cont, {:open, d + 1, c}} + + c in [?}, ?]] and d <= 1 -> + {:halt, :closed} + + c in [?}, ?]] -> + {:cont, {:open, d - 1, c}} + + c in [?\s, ?\t, ?,] -> + {:cont, {:open, d, if(c == ?,, do: c, else: l)}} + + d == 1 and l in [?{, ?,] -> + case entry_uses(m, i) do + nil -> {:cont, {:open, d, c}} + site -> {:halt, site} + end + + true -> + {:cont, {:open, d, c}} end end) + |> case do + {:open, _, _} = state -> state + %{} = site -> {:found, site} + other -> other + end + end + + # The `uses` entry starting at byte `i` of masked line `m`, as a site. + defp entry_uses(m, i) do + rest = binary_part(m, i, byte_size(m) - i) + + case Regex.run( + ~r/\A["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*["']?([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)/, + rest, + return: :index + ) do + [_, {a_at, a_len}, {r_at, r_len}] -> + %{ + action: binary_part(rest, a_at, a_len), + ref: binary_part(rest, r_at, r_len), + action_at: i + a_at, + ref_at: i + r_at + } + + _ -> + nil + end end # A block mapping anchored with nothing after the anchor (`- &step`, diff --git a/lib/rules/pr_automerge.ex b/lib/rules/pr_automerge.ex index cea2d944..2c95a444 100644 --- a/lib/rules/pr_automerge.ex +++ b/lib/rules/pr_automerge.ex @@ -275,6 +275,17 @@ defmodule Hypatia.Rules.PrAutomerge do """ @spec pin_deltas([map()], map(), [map()]) :: [map()] def pin_deltas(files, resolution, claims) do + # Basenames two different actions of the PR share (`acme/setup`, + # `other/setup`): a claim naming only that basename belongs to neither. + shared = + files + |> Enum.flat_map(&removed_pins/1) + |> Enum.map(&String.downcase(&1.action)) + |> Enum.uniq() + |> Enum.frequencies_by(&(&1 |> String.split("/") |> List.last())) + |> Enum.filter(fn {_, n} -> n > 1 end) + |> MapSet.new(&elem(&1, 0)) + files |> Enum.flat_map(fn file -> # Every structural site of every changed line: a flow sequence carries @@ -295,7 +306,7 @@ defmodule Hypatia.Rules.PrAutomerge do {[], pool} i -> - {Enum.at(pool, i) |> pin_delta(old, file, resolution, claims), + {Enum.at(pool, i) |> pin_delta(old, file, resolution, claims, shared), List.delete_at(pool, i)} end end) @@ -303,13 +314,13 @@ defmodule Hypatia.Rules.PrAutomerge do end) end - defp pin_delta(new, old, file, resolution, claims) do + defp pin_delta(new, old, file, resolution, claims, shared) do if new.ref == old.ref do [] else from = resolve(resolution, old.action, old.ref) to = resolve(resolution, new.action, new.ref) - claim = claim_for(claims, old.action) + claim = claim_for(claims, old.action, shared) cond do from != nil and to != nil and claim != nil and @@ -319,6 +330,12 @@ defmodule Hypatia.Rules.PrAutomerge do from != nil and to != nil -> delta(old, new, from, to, :ok, "tags") + # One endpoint resolved: the claim must agree with it, or the + # evidence there is contradicts it (a claimed `2.0.0 -> 2.1.0` from + # a ref that resolves to `1.0.0` may hide a major crossing). + claim != nil and ((from != nil and claim.from != from) or (to != nil and claim.to != to)) -> + delta(old, new, from || claim.from, to || claim.to, :conflict, "tags") + claim != nil -> delta(old, new, claim.from, claim.to, :ok, "body-claim") @@ -796,11 +813,21 @@ defmodule Hypatia.Rules.PrAutomerge do Map.get(resolution, ref) end - defp claim_for(claims, action) do - short = action |> String.split("/") |> List.last() - - Enum.find(claims, fn c -> c.action == action or c.action == short end) || - Enum.find(claims, fn c -> String.ends_with?(c.action, "/" <> short) end) + # The body claim for `action`: one naming the action itself, or its root + # repository (`github/codeql-action` for `github/codeql-action/init`), in + # either case. A claim naming only a basename (`[setup](...)`) is this + # action's only when no other action of the PR shares that basename + # (`shared`). A claim for another repository that ends the same way + # (`acme/setup` for `other/setup`) is never borrowed: the delta stays + # unresolved rather than verified by someone else's version numbers. + defp claim_for(claims, action, shared) do + full = String.downcase(action) + base = String.downcase(action_base(action)) + short = full |> String.split("/") |> List.last() + + Enum.find(claims, &(String.downcase(&1.action) == full)) || + Enum.find(claims, &(String.downcase(&1.action) == base)) || + if(short not in shared, do: Enum.find(claims, &(String.downcase(&1.action) == short))) end defp major_of(version) do diff --git a/lib/rules/workflow_audit.ex b/lib/rules/workflow_audit.ex index 79031982..1421313f 100644 --- a/lib/rules/workflow_audit.ex +++ b/lib/rules/workflow_audit.ex @@ -1201,6 +1201,14 @@ defmodule Hypatia.Rules.WorkflowAudit do # A step inside an open flow collection is a flow-style step even when # its `{` stands alone on a line above (`steps: [`, `{`, `uses: ...`). flow = wf_flow_lines(normalised) + # A step given whole as an alias (`- *step`, `steps: [*step]`) runs the + # mapping its anchor holds, wherever that mapping sits (a matrix entry): + # its `uses:` line is a live step's, judged as one, and routes to + # review — the fixer does not rewrite anchored steps. + aliased = + for %{anchor_line: n} <- Hypatia.Rules.PinSites.node_alias_sites(normalised), + into: MapSet.new(), + do: n lines |> Enum.with_index(1) @@ -1213,6 +1221,8 @@ defmodule Hypatia.Rules.WorkflowAudit do :error -> {line, nil} end + live? = number in aliased + case Regex.run(@toolchain_sha_pin, line) do [_full, sha] -> # The fleet fixer matches the identity in lower case only, and a @@ -1222,14 +1232,15 @@ defmodule Hypatia.Rules.WorkflowAudit do cond do kyaml? -> :kyaml resolved != nil -> resolved + live? -> :node_alias wf_node_properties?(line) -> :properties not String.contains?(line, "dtolnay/rust-toolchain@") -> :spelling true -> nil end if wf_in_flow?(flow, number), - do: wf026_flow_line_finding(filename, lines, flow, number, sha, review), - else: wf026_block_step_finding(filename, lines, line, number, sha, review) + do: wf026_flow_line_finding(filename, lines, flow, number, sha, review, 0, live?), + else: wf026_block_step_finding(filename, lines, line, number, sha, review, live?) _ -> # Several flow-style steps may share one physical line @@ -1238,14 +1249,29 @@ defmodule Hypatia.Rules.WorkflowAudit do |> Regex.scan(line, return: :index) |> Enum.flat_map(fn [{at, _}, {sha_at, 40}] -> sha = binary_part(line, sha_at, 40) - review = if kyaml?, do: :kyaml + + review = + cond do + kyaml? -> :kyaml + live? -> :node_alias + true -> nil + end # On a line inside a flow collection opened above, the step's # ancestry is in the brackets, not the line's indentation. if wf_in_flow?(flow, number) do - wf026_flow_line_finding(filename, lines, flow, number, sha, review, at + 1) + wf026_flow_line_finding(filename, lines, flow, number, sha, review, at + 1, live?) else - wf026_inline_step_finding(filename, lines, line, number, at + 1, sha, review) + wf026_inline_step_finding( + filename, + lines, + line, + number, + at + 1, + sha, + review, + live? + ) end end) end @@ -1256,7 +1282,7 @@ defmodule Hypatia.Rules.WorkflowAudit do # The block-style path: `uses:` is a key on its own line, outside any flow # collection (one inside is read by `wf026_flow_line_finding/7`). A step # whose opener is `- {` is a flow-style step: the fixer refuses it. - defp wf026_block_step_finding(filename, lines, line, number, sha, review) do + defp wf026_block_step_finding(filename, lines, line, number, sha, review, live?) do {step, key_indent, opener} = wf026_step(lines, number) flow_item? = Regex.match?(~r/^\s*-\s+\{/, Enum.at(lines, opener - 1)) @@ -1275,8 +1301,9 @@ defmodule Hypatia.Rules.WorkflowAudit do [] # A list item is a step only under `steps:`: one inside a block - # scalar (a fixture kept in `env:`, a `run: |` script) is text. - not wf026_under_steps?(lines, opener) -> + # scalar (a fixture kept in `env:`, a `run: |` script) is text. A + # mapping a step alias runs (`live?`) is a step wherever it sits. + not live? and not wf026_under_steps?(lines, opener) -> [] true -> @@ -1294,9 +1321,9 @@ defmodule Hypatia.Rules.WorkflowAudit do # The inline path: `uses:` sits inside a flow mapping on the line that # opens it. The mapping is read through its braces (quote-aware, across # lines) and its `with` entry decides. - defp wf026_inline_step_finding(filename, lines, line, number, uses_at, sha, review) do + defp wf026_inline_step_finding(filename, lines, line, number, uses_at, sha, review, live?) do with {:ok, open_col} <- wf_flow_opener_before(line, uses_at), - true <- wf026_inline_step_at?(lines, line, number, open_col), + true <- live? or wf026_inline_step_at?(lines, line, number, open_col), {:ok, interior, _} <- wf_flow_interior(lines, number, open_col) do # Where the step's `with` value stands, line and column: an alias # there names the latest anchor before it, not one later on its line. @@ -1322,12 +1349,15 @@ defmodule Hypatia.Rules.WorkflowAudit do # (`wf_flow_context/4`), not its indentation: it is a step's own key when # the innermost one is a mapping that is an item of the `[` a `steps:` in # a job opens, and the step is read through that mapping's braces. - defp wf026_flow_line_finding(filename, lines, flow, number, sha, review, at \\ 0) do + defp wf026_flow_line_finding(filename, lines, flow, number, sha, review, at, live?) do with {:ok, keys, kinds, {brace_line, brace_col}} <- wf_flow_context(lines, flow, number, at), - true <- match?([nil, "steps", job, "jobs" | _] when is_binary(job), keys), + true <- live? or match?([nil, "steps", job, "jobs" | _] when is_binary(job), keys), # The `{` is an item of the `[` that `steps:` opens, or of the - # block sequence under it (`- {` on a line of its own). - true <- match?([:map, :seq | _], kinds) or kinds == [:map], + # block sequence under it (`- {` on a line of its own); a mapping + # a step alias runs is the step wherever it sits. + true <- + match?([:map, :seq | _], kinds) or kinds == [:map] or + (live? and match?([:map | _], kinds)), {:ok, interior, _} <- wf_flow_interior(lines, brace_line, brace_col) do pos = case wf_flow_entry_value_at(interior, "with") do @@ -1460,6 +1490,10 @@ defmodule Hypatia.Rules.WorkflowAudit do " The `uses:` value is continued below its key; the repair script reads a " <> "reference on the key's own line only, so this one routes to review." + _ when review == :node_alias -> + " The step is given as an alias (`- *name`) of a mapping anchored elsewhere; " <> + "the repair script does not rewrite anchored steps, so this one routes to review." + _ when review == :alias -> " The `uses:` value is an alias of an anchored reference; the repair script " <> "does not rewrite anchored values, so this one routes to review." diff --git a/test/rules/pin_integrity_test.exs b/test/rules/pin_integrity_test.exs index 18da36d3..eb68c052 100644 --- a/test/rules/pin_integrity_test.exs +++ b/test/rules/pin_integrity_test.exs @@ -582,6 +582,34 @@ defmodule Hypatia.Rules.PinIntegrityTest do assert [%{line: 2}] = PI.pi001_denylisted_pin(content, @policy) end + test "a step alias runs a flow mapping anchored over several lines" do + content = + "jobs:\n a:\n strategy:\n matrix:\n include:\n" <> + " - step: &step {\n name: init,\n" <> + " uses: github/codeql-action/init@#{@poison}\n }\n" <> + " steps:\n - *step\n" + + assert [%{line: 8}] = PI.pi001_denylisted_pin(content, @policy) + assert {:ok, fixed, [_]} = PI.substitute_denylisted_pins(content, @policy) + refute fixed =~ @poison + end + + test "a version is matched past one optional v only" do + policy = put_in(@policy, ["pin_denylist", Access.at(0), "blocked_versions"], ["4.38.1"]) + + assert [_] = PI.pi001_denylisted_pin("- uses: github/codeql-action/init@v4.38.1\n", policy) + assert PI.pi001_denylisted_pin("- uses: github/codeql-action/init@vv4.38.1\n", policy) == [] + end + + test "a substitution that leaves a denylisted pin is refused" do + # A `known_good_sha` that is itself blocked repairs nothing. + policy = put_in(@policy, ["pin_denylist", Access.at(0), "known_good_sha"], @poison) + content = "- uses: github/codeql-action/init@#{@poison}\n" + + assert {:error, reason} = PI.substitute_denylisted_pins(content, policy) + assert reason =~ "leaves a denylisted pin" + end + test "a KYAML flow step on its own line is still a site" do content = "steps: [\n { name: \"x\", run: \"echo 'a\" },\n" <> @@ -1010,6 +1038,41 @@ defmodule Hypatia.Rules.PinIntegrityTest do " - 'actions/checkout@v4'\n" ) end + + test "with the workflow's path, only that workflow's own lock entry vouches for its pin" do + lock = """ + version: 'v0.0.2' + workflows: + '.github/workflows/a.yml': + - 'actions/checkout@v7.0.1' + '.github/workflows/b.yml': + - 'actions/checkout@v6.0.0' + dependencies: + 'actions/checkout@v6.0.0': + ref: 'v6.0.0' + commit: 'sha1-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + owner_id: 44036562 + repo_id: 197814629 + 'actions/checkout@v7.0.1': + ref: 'v7.0.1' + commit: 'sha1-3d3c42e5aac5ba805825da76410c181273ba90b1' + owner_id: 44036562 + repo_id: 197814629 + """ + + workflow = "- uses: actions/checkout@v7.0.1\n" + + # `b.yml` locks v6.0.0: v7.0.1, locked for `a.yml`, does not vouch for it. + assert [finding] = + PI.pi004_lock_divergence(workflow, lock, path: ".github/workflows/b.yml") + + assert finding.detail.lock_refs == ["v6.0.0"] + + # Controls: `a.yml` agrees with its own entry; with no path the lock's + # refs are pooled, as before. + assert PI.pi004_lock_divergence(workflow, lock, path: ".github/workflows/a.yml") == [] + assert PI.pi004_lock_divergence(workflow, lock) == [] + end end describe "pi005_stale_pin/2 — advisory, never an action" do diff --git a/test/rules/pr_automerge_test.exs b/test/rules/pr_automerge_test.exs index 1b473c66..07c409ed 100644 --- a/test/rules/pr_automerge_test.exs +++ b/test/rules/pr_automerge_test.exs @@ -112,6 +112,44 @@ defmodule Hypatia.Rules.PrAutomergeTest do assert [delta] = decision.deltas assert delta.source == "body-claim" end + + test "a claim for one action is never borrowed by another of the same basename" do + patch = """ + @@ -1,2 +1,2 @@ + - - uses: acme/setup@v1.0.0 + - - uses: other/setup@v3.0.0 + + - uses: acme/setup@v1.1.0 + + - uses: other/setup@v4.0.0 + """ + + body = "Bumps [acme/setup](https://github.com/acme/setup) from 1.0.0 to 1.1.0.\n" + f = file(".github/workflows/ci.yml", patch) + deltas = PA.pin_deltas([f], %{}, PA.body_claims(body)) + + assert %{status: :ok, source: "body-claim"} = + Enum.find(deltas, &(&1.action == "acme/setup")) + + assert %{status: :unresolved} = Enum.find(deltas, &(&1.action == "other/setup")) + + # A claim naming only the basename belongs to neither action. + bare = PA.body_claims("Bumps [setup](https://example.com) from 1.0.0 to 1.1.0.\n") + assert Enum.all?(PA.pin_deltas([f], %{}, bare), &(&1.status == :unresolved)) + end + + test "a claim is checked against whichever endpoint resolved" do + resolution = %{{"actions/checkout", "v4.1.7"} => "1.0.0"} + + body = + "Bumps [actions/checkout](https://github.com/actions/checkout) from 2.0.0 to 2.1.0.\n" + + f = file(".github/workflows/ci.yml", checkout_patch("v4.1.7", "v4.2.0")) + + assert [%{status: :conflict}] = PA.pin_deltas([f], resolution, PA.body_claims(body)) + + # Control: a claim that agrees with the resolved endpoint is taken. + agreeing = String.replace(body, "from 2.0.0 to 2.1.0", "from 1.0.0 to 1.1.0") + assert [%{status: :ok}] = PA.pin_deltas([f], resolution, PA.body_claims(agreeing)) + end end describe "classify/2 — PA002, majors are never armed" do diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs index 281a3bed..b39a59b3 100644 --- a/test/rules/workflow_audit_pins_test.exs +++ b/test/rules/workflow_audit_pins_test.exs @@ -1132,6 +1132,28 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do assert [%{line: 5, auto_fixable: true}] = wf026(wf) end + test "a step given whole as an alias is judged where its mapping is anchored" do + wf = """ + jobs: + check: + strategy: + matrix: + include: + - &step + uses: dtolnay/rust-toolchain@#{@toolchain_sha} + steps: + - *step + """ + + # The anchored mapping is matrix data until the alias runs it: then it + # is a live step with no input, routed to review (not the fixer). + assert [%{line: 7, auto_fixable: false} = f] = wf026(wf) + assert f.reason =~ "given as an alias" + + # Control: with no alias, the matrix entry is data. + assert [] == wf026(String.replace(wf, " - *step\n", " - run: echo\n")) + end + test "a toolchain: key with no value is no input: null and ~ included" do wf = """ jobs: From 215a8207909cc8c360259bd2ca28254eb030618e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 09:42:13 +0000 Subject: [PATCH 46/47] fix(pins): root data keys hold data; uses: block scalars resolve to their reference; node aliases resolve uses: aliases; WF027 judges alias-run jobs - PinSites.data_uses_lines: a `uses:` under a column-0 key that is neither `jobs:` nor `runs:` is data when the text shows the root (a column-0 `jobs:`/`runs:`), or the key is one a workflow or an action defines for data (`env:`, `on:`, `inputs:`, ...), so a hunk showing `env:` at column 0 reads the same. Fragments with neither are judged as before. - PinSites.line_states: the one content line of a block scalar a `uses:` key or an anchor holds, when it is a whole reference, is kept as written with the new `:value` state. continued_site/2 reads it as the key's value, continued_anchor reads it as the anchor's, so PI001 and its repair, the PR sides, WF026 and WF027 see `- uses: >-` values. A block scalar that folds more lines names no site. - PinSites.node_alias_sites: the anchored mapping's `uses:` value is resolved as YAML resolves it: on its line, continued below it, or an alias (`uses: *bad`, sited at `&bad`'s value). Results carry `key_line` (the `uses:` key) beside `anchor_line` (where the reference stands); WF026 and the PR side view key on `key_line`. - WF027 consumes node_alias_sites: a call made only through a job alias (mapping anchored outside `jobs:`) is reported, routed to review with a :node_alias note; a job anchored under `jobs:` stays the fixer's. Tests for each case fail on c579fef and pass here. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- lib/rules/pin_sites.ex | 341 +++++++++++++++++------- lib/rules/pr_automerge.ex | 13 +- lib/rules/workflow_audit.ex | 58 +++- test/rules/pin_integrity_test.exs | 65 +++++ test/rules/pr_automerge_test.exs | 68 +++++ test/rules/workflow_audit_pins_test.exs | 69 +++++ 6 files changed, 506 insertions(+), 108 deletions(-) diff --git a/lib/rules/pin_sites.ex b/lib/rules/pin_sites.ex index 7aa88d2f..d8d23983 100644 --- a/lib/rules/pin_sites.ex +++ b/lib/rules/pin_sites.ex @@ -33,6 +33,14 @@ defmodule Hypatia.Rules.PinSites do @uses_tail_re ~r/(?:^|[-{,\s])["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*$/ + # A block scalar's content line that is a whole reference: no quotes (they + # would be part of the value) and nothing after it. + @block_ref_re ~r/^\s*([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)\s*$/ + + # Root keys a workflow or an action defines whose values are data: no step + # or job sits under any of them (`block_rows/1`). + @data_root_keys ~w(name run-name on permissions env defaults concurrency description author inputs outputs branding) + @doc """ The structural sites of `line`, in order, as `%{action: a, ref: r, action_at: byte, ref_at: byte}`. @@ -177,7 +185,10 @@ defmodule Hypatia.Rules.PinSites do than the key and holds nothing but the reference, quoted or plain, and a comment; a flow entry's value is the reference up to the delimiter that ends it (`,`, `}` or `]`), wherever the line is indented — inside a flow - collection the scalar continues on any line. + collection the scalar continues on any line. A block key may also hold a + block scalar (`- uses: >-`): its one content line, a whole reference + (`line_states/1`, `:value`), is the value; in masked lines any other + content is blanked, so a scalar that folds more lines names no site. """ @spec continued_site(String.t(), String.t()) :: map() | nil def continued_site(key_line, next_line) when is_binary(key_line) and is_binary(next_line) do @@ -190,6 +201,13 @@ defmodule Hypatia.Rules.PinSites do do: ~r/^\s*(?:[&!]\S*\s+)*["']?([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)["']?\s*(?:#.*)?$/ + m = + Regex.run( + ~r/^(\s*(?:-\s+)?)["']?uses["']?\s*:(?:\s+[&!]\S*)*\s+[|>][-+0-9]*\s*$/, + key + ) -> + if indent_of(next_line) > byte_size(Enum.at(m, 1)), do: @block_ref_re + Regex.match?(~r/[{,]\s*["']?uses["']?\s*:(?:\s+[&!]\S*)*\s*$/, key) and structural?(String.trim_trailing(key) <> " ") -> ~r/^\s*(?:[&!]\S*\s+)*["']?([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)["']?\s*(?:[,}\]]|#|$)/ @@ -230,11 +248,7 @@ defmodule Hypatia.Rules.PinSites do prefixes = flow_prefixes ++ List.duplicate("", max(length(lines) - length(flow_prefixes), 0)) numbered = lines |> Enum.zip(prefixes) |> Enum.with_index(1) - defs = - for {{line, prefix}, n} <- numbered, d <- anchor_defs(line, prefix) do - below = for {{l, _}, m} <- Enum.drop(numbered, n), do: {l, m} - d |> Map.put(:line, n) |> continued_anchor(line, below) - end + defs = resolved_defs(numbered) for {{line, prefix}, n} <- numbered, {name, at, len} <- uses_aliases(line, prefix), @@ -243,22 +257,48 @@ defmodule Hypatia.Rules.PinSites do end end + # Every anchor the numbered lines define, each with the site its value is + # (`anchor_defs/2`, `continued_anchor/3`) and its `:line`. + defp resolved_defs(numbered) do + for {{line, prefix}, n} <- numbered, d <- anchor_defs(line, prefix) do + below = for {{l, _}, m} <- Enum.drop(numbered, n), do: {l, m} + d |> Map.put(:line, n) |> continued_anchor(line, below) + end + end + + # The site the alias `*name` at byte `at` of line `n` runs, as + # `[{line, site}]` (the line its reference stands on), or `[]`. + defp alias_value(defs, name, n, at) do + case latest_anchor(defs, name, n, at) do + %{site: %{} = site} = d -> [{Map.get(d, :site_line, d.line), site}] + _ -> [] + end + end + # An anchor with nothing after it on its line but a tag or a comment # (`BAD: &bad`) names the scalar YAML continues on the next line with # content (` github/codeql-action/init@`): its site is there, # `:site_line` that line. In block context the scalar must sit deeper than # the key, and is all the line holds; inside a flow collection it may # stand at any indentation, even column 0, up to the `,`, `}` or `]` that - # ends it. + # ends it. An anchored block scalar (`BAD: &bad >-`) names the reference + # its one content line holds (`line_states/1`, `:value`). defp continued_anchor(%{site: nil, rest: rest} = d, line, below) do + block? = not d.flow and Regex.match?(~r/^\s*(?:![^\s]*\s+)*[|>][-+0-9]*\s*(?:#.*)?$/, rest) + value_re = - if d.flow, - do: - ~r/^\s*(?:![^\s]*\s+)*["']?([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)["']?\s*(?:[,}\]]|#|$)/, - else: + cond do + block? -> + @block_ref_re + + d.flow -> + ~r/^\s*(?:![^\s]*\s+)*["']?([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)["']?\s*(?:[,}\]]|#|$)/ + + true -> ~r/^\s*(?:![^\s]*\s+)*["']?([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)["']?\s*(?:#.*)?$/ + end - with true <- Regex.match?(~r/^\s*(?:![^\s]*\s*)*(?:#.*)?$/, rest), + with true <- block? or Regex.match?(~r/^\s*(?:![^\s]*\s*)*(?:#.*)?$/, rest), {next, n} <- Enum.find(below, fn {l, _} -> String.trim(head(l)) != "" end), true <- d.flow or indent_of(next) > indent_of(line), [_, {a_at, a_len}, {r_at, r_len}] <- Regex.run(value_re, next, return: :index) do @@ -367,8 +407,12 @@ defmodule Hypatia.Rules.PinSites do Every step or job given whole as an alias — `- *step` in a job's `steps:`, `*step` as an entry of a `steps: [ ... ]`, `rust: *job` under `jobs:` — resolved to the `uses:` of the mapping its anchor holds: - `%{line, anchor_line, site}`, the alias' line, the line of that `uses:` - key, and its site as `sites/1` returns one (offsets into that line). + `%{line, key_line, anchor_line, site}`, the alias' line, the line of that + `uses:` key, the line its reference stands on, and its site as `sites/1` + returns one (offsets into that line). The reference is the key's value as + YAML resolves it: written on the key's line, continued below it + (`continued_site/2`, a block scalar included), or an alias (`uses: *bad`, + on the line of `&bad`'s value). YAML resolves the alias to that very node, so its `uses:` is the step's or the job's action even where the anchor sits in data (`include: - &step { uses: ... }` in a matrix). The anchored mapping is read as a flow @@ -383,22 +427,15 @@ defmodule Hypatia.Rules.PinSites do numbered = lines |> Enum.zip(flow_prefixes(content)) |> Enum.with_index(1) texts = List.to_tuple(lines) - rows = - content - |> line_states() - |> Enum.zip_with(line_depths(content), &block_row/2) - |> List.to_tuple() + rows = block_rows(content) - defs = - for {{line, prefix}, n} <- numbered, - d <- anchor_defs(line, prefix), - do: Map.put(d, :line, n) + defs = resolved_defs(numbered) for {{line, prefix}, n} <- numbered, {name, at} <- node_aliases(line, prefix, rows, n - 1), %{} = d <- [latest_anchor(defs, name, n, at)], - {uses_line, site} <- anchored_uses(d, texts, rows) do - %{line: n, anchor_line: uses_line, site: site} + {key_line, site_line, site} <- anchored_uses(d, texts, rows, defs) do + %{line: n, key_line: key_line, anchor_line: site_line, site: site} end end @@ -458,25 +495,21 @@ defmodule Hypatia.Rules.PinSites do defp alias_step_stack?(_), do: false - # The `uses:` of the mapping the anchor `d` holds, as `[{line, site}]`: a - # flow mapping opening on the anchor's line, read through the lines below - # to the brace that closes it, or a block mapping on the lines below, read - # to its `uses:` key. - defp anchored_uses(d, texts, rows) do + # The `uses:` of the mapping the anchor `d` holds, as `[{key_line, + # site_line, site}]`: a flow mapping opening on the anchor's line, read + # through the lines below to the brace that closes it, or a block mapping + # on the lines below, read to its `uses:` key. + defp anchored_uses(d, texts, rows, defs) do line = elem(texts, d.line - 1) rest_at = byte_size(line) - byte_size(d.rest) cond do m = Regex.run(~r/^\s*(?:![^\s]*\s+)*\{/, d.rest, return: :index) -> [{0, len}] = m - - case flow_mapping_uses(texts, d.line, rest_at + len - 1) do - nil -> [] - found -> [found] - end + flow_mapping_uses(texts, d.line, rest_at + len - 1, defs) Regex.match?(~r/^\s*(?:![^\s]*\s*)*(?:#.*)?$/, d.rest) -> - block_uses(d.line, texts, rows) + block_uses(d.line, texts, rows, defs) true -> [] @@ -484,26 +517,35 @@ defmodule Hypatia.Rules.PinSites do end # The top-level `uses` entry of the flow mapping whose `{` is at byte - # `open` of line `n`, as `{line, site}`, or nil when the mapping closes - # without one. The mapping is read through the lines below to the brace - # that closes it: an entry starts after its `{` or one of its commas — on - # the same line, or as the last thing a line above held — with no other - # bracket open between. Lines are read masked (`mask/1`), so brackets and + # `open` of line `n`, as `[{key_line, site_line, site}]` (an alias value + # resolved to its anchor's), or `[]` when the mapping closes without one. + # The mapping is read through the lines below to the brace that closes + # it: an entry starts after its `{` or one of its commas — on the same + # line, or as the last thing a line above held — with no other bracket + # open between. Lines are read masked (`mask/1`), so brackets and # commas inside quoted text and comments count for nothing. - defp flow_mapping_uses(texts, n, open) do + defp flow_mapping_uses(texts, n, open, defs) do n..tuple_size(texts)//1 |> Enum.reduce_while({:open, 0, nil}, fn k, {:open, depth, last} -> from = if k == n, do: open, else: 0 case mapping_line_uses(mask(elem(texts, k - 1)), from, depth, last) do - {:found, site} -> {:halt, {k, site}} - :closed -> {:halt, nil} - open_state -> {:cont, open_state} + {:found, {:site, site}} -> + {:halt, [{k, k, site}]} + + {:found, {:alias, name, at}} -> + {:halt, for({line, site} <- alias_value(defs, name, k, at), do: {k, line, site})} + + :closed -> + {:halt, []} + + open_state -> + {:cont, open_state} end end) |> case do - {k, %{} = site} -> {k, site} - _ -> nil + found when is_list(found) -> found + _ -> [] end end @@ -538,27 +580,33 @@ defmodule Hypatia.Rules.PinSites do end) |> case do {:open, _, _} = state -> state - %{} = site -> {:found, site} - other -> other + :closed -> :closed + found -> {:found, found} end end - # The `uses` entry starting at byte `i` of masked line `m`, as a site. + # The `uses` entry starting at byte `i` of masked line `m`: `{:site, + # site}` for a reference written there, `{:alias, name, offset of the + # "*"}` for an alias, or nil. defp entry_uses(m, i) do rest = binary_part(m, i, byte_size(m) - i) case Regex.run( - ~r/\A["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*["']?([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)/, + ~r/\A["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*(?:["']?([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)|\*([^\s,\[\]{}]+))/, rest, return: :index ) do - [_, {a_at, a_len}, {r_at, r_len}] -> - %{ - action: binary_part(rest, a_at, a_len), - ref: binary_part(rest, r_at, r_len), - action_at: i + a_at, - ref_at: i + r_at - } + [_, {a_at, a_len}, {r_at, r_len}] when a_at >= 0 -> + {:site, + %{ + action: binary_part(rest, a_at, a_len), + ref: binary_part(rest, r_at, r_len), + action_at: i + a_at, + ref_at: i + r_at + }} + + [_, _, _, {n_at, n_len}] -> + {:alias, binary_part(rest, n_at, n_len), i + n_at - 1} _ -> nil @@ -568,8 +616,8 @@ defmodule Hypatia.Rules.PinSites do # A block mapping anchored with nothing after the anchor (`- &step`, # `step: &step`) on row `n`: its keys are the rows below deeper than the # dash or the key, the first of them setting their column; the one named - # `uses` holds the site(s). - defp block_uses(n, texts, rows) do + # `uses` holds the site(s) (`key_value/3`). + defp block_uses(n, texts, rows, defs) do case elem(rows, n - 1) do %{} = row -> parent = if row.key == nil, do: row.dash || row.indent, else: row.key_col @@ -587,8 +635,8 @@ defmodule Hypatia.Rules.PinSites do [{_, first} | _] -> for {j, %{indent: ind, dash: nil, key: "uses"}} <- children, ind == first.indent, - site <- sites(elem(texts, j)), - do: {j + 1, site} + {line, site} <- key_value(texts, j + 1, defs), + do: {j + 1, line, site} end _ -> @@ -596,22 +644,51 @@ defmodule Hypatia.Rules.PinSites do end end + # The site(s) the block `uses:` key on line `k` holds, as `[{line, site}]`: + # written on its line, an alias (`uses: *bad`) resolved to the anchor's + # value, or a reference continued below it (`continued_site/2`: the next + # line with content, a block scalar's included). + defp key_value(texts, k, defs) do + text = elem(texts, k - 1) + masked = mask(text) + + alias_re = ~r/^\s*["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*\*([^\s,\[\]{}]+)\s*$/ + + case {sites(text), Regex.run(alias_re, masked, return: :index)} do + {[_ | _] = found, _} -> + for site <- found, do: {k, site} + + {[], [_, {n_at, n_len}]} -> + alias_value(defs, binary_part(masked, n_at, n_len), k, n_at - 1) + + {[], nil} -> + with v when is_integer(v) <- + Enum.find( + (k + 1)..tuple_size(texts)//1, + &(String.trim(head(elem(texts, &1 - 1))) != "") + ), + %{} = site <- continued_site(text, elem(texts, v - 1)) do + [{v, site}] + else + _ -> [] + end + end + end + @doc """ The one-based numbers of the lines of `content` whose leading block `uses:` key the indentation above it proves is neither a step's nor a job's own: an input of that name (`with:` then `uses: a/b@x`), an entry of a sequence - some key other than `steps:` holds, or a key of a `steps:` that is no - job's. Such a key holds data, not a pin. A line whose ancestry the lines - above do not show far enough — a diff hunk cut short, a fragment with no - `jobs:` — is not listed: it is judged on its own line, as before. + some key other than `steps:` holds, a key of a `steps:` that is no + job's, or one under a root key that holds no job (`env:` then `uses: + a/b@x` in a workflow). Such a key holds data, not a pin. A line whose + ancestry the lines above do not show far enough — a diff hunk cut short, + a fragment with no `jobs:` — is not listed: it is judged on its own line, + as before. """ @spec data_uses_lines(String.t()) :: MapSet.t(pos_integer()) def data_uses_lines(content) when is_binary(content) do - rows = - content - |> line_states() - |> Enum.zip_with(line_depths(content), &block_row/2) - |> List.to_tuple() + rows = block_rows(content) for i <- 0..(tuple_size(rows) - 1)//1, match?(%{key: "uses"}, elem(rows, i)), @@ -620,6 +697,30 @@ defmodule Hypatia.Rules.PinSites do do: i + 1 end + # The lines of `content` as block structure (`block_row/2`), a tuple. A key + # at column 0 is a root key; a root key that is neither `jobs:` nor + # `runs:` (`env:`, `on:`, `inputs:`) holds no step or job, so it is marked + # `data_root?` — when the text shows the root (a `jobs:` or `runs:` at + # column 0), or the key is one a workflow or an action defines for data, + # so a hunk that shows `env:` at column 0 reads it the same. A fragment + # with neither (`build:` then `uses: ...`) is judged as before. + defp block_rows(content) do + rows = content |> line_states() |> Enum.zip_with(line_depths(content), &block_row/2) + + shown? = + Enum.any?(rows, &match?(%{indent: 0, dash: nil, key: k} when k in ["jobs", "runs"], &1)) + + rows + |> Enum.map(fn + %{indent: 0, dash: nil, key: k} = row when is_binary(k) and k not in ["jobs", "runs"] -> + Map.put(row, :data_root?, shown? or k in @data_root_keys) + + row -> + row + end) + |> List.to_tuple() + end + # One line as block structure: its indentation, the column of its last # list dash (nil without one), the column and name of the key after the # dashes, whether that key's value is on the lines below (`key:` alone, @@ -669,10 +770,16 @@ defmodule Hypatia.Rules.PinSites do case mapping_of(rows, i) do {:item, j, d} -> sequence_owner(rows, seq_owner(rows, j, d)) {:value, j, _key} -> job_key(rows, j) + :root -> root_owner(rows, i) _ -> :unknown end end + # A key of the root mapping (`block_rows/1`): :data when it is a data root, + # else :unknown — the text shows no more of where it sits. + defp root_owner(rows, j), + do: if(match?(%{data_root?: true}, elem(rows, j)), do: :data, else: :unknown) + defp sequence_owner(rows, {:value, s, "steps"}), do: steps_owner(rows, s) defp sequence_owner(_rows, {:value, _, _}), do: :data defp sequence_owner(_rows, _), do: :unknown @@ -684,6 +791,7 @@ defmodule Hypatia.Rules.PinSites do {:value, _p, "runs"} -> :site {:value, p, _job} -> job_key(rows, p) {:item, _, _} -> :data + :root -> root_owner(rows, s) _ -> :unknown end end @@ -695,6 +803,7 @@ defmodule Hypatia.Rules.PinSites do {:value, _, "jobs"} -> :site {:value, _, _} -> :data {:item, _, _} -> :data + :root -> root_owner(rows, j) _ -> :unknown end end @@ -762,10 +871,11 @@ defmodule Hypatia.Rules.PinSites do """ @spec line_depths(String.t()) :: [non_neg_integer()] def line_depths(content) when is_binary(content) do - content - |> String.split("\n") + lines = String.split(content, "\n") + + lines |> Enum.with_index() - |> Enum.map_reduce(%{q: nil, opened: nil, block: nil, depth: 0, pending: false}, fn + |> Enum.map_reduce(initial_state(lines), fn {line, n}, st -> {_, next} = mask_file_line(line, n, st) {st.depth, next} @@ -801,8 +911,9 @@ defmodule Hypatia.Rules.PinSites do @doc """ The lines of a whole file, each with the bytes that belong to a scalar opened on an earlier line blanked to `x`, byte for byte: every content - line of a block scalar (`run: |`), and a multi-line quoted scalar's - continuation up to and including its closing quote. A line read through + line of a block scalar (`run: |`) — but the reference one a `uses:` key or + an anchor holds is whole (`:value`, `line_states/1`) — and a multi-line + quoted scalar's continuation up to and including its closing quote. A line read through `sites/1` one at a time cannot see either, so a `uses:` written in a `run: |` script reads as a key; read from these lines it does not. @@ -822,20 +933,23 @@ defmodule Hypatia.Rules.PinSites do The lines of `mask_lines/1`, each with what the reader made of it: `:yaml` (structure, as written), `:text` (inside a scalar an earlier line opened: a block scalar's content, or a quoted scalar's continuation — at - least up to its closing quote) or `:open` (after the opening quote of a - quoted scalar the text never closes, returned as written: what it is - cannot be said from these lines alone). A diff hunk, which shows a few - lines of a file, is read this way too: a hunk that cuts a quoted scalar - off leaves its lines `:open`. + least up to its closing quote), `:value` (the one content line of a block + scalar a `uses:` key or an anchor holds, when that line is a whole + reference — `- uses: >-` then ` a/b@`: the scalar's value, kept as + written so the key's site can be read there) or `:open` (after the + opening quote of a quoted scalar the text never closes, returned as + written: what it is cannot be said from these lines alone). A diff hunk, + which shows a few lines of a file, is read this way too: a hunk that cuts + a quoted scalar off leaves its lines `:open`. """ - @spec line_states(String.t()) :: [{String.t(), :yaml | :text | :open}] + @spec line_states(String.t()) :: [{String.t(), :yaml | :text | :value | :open}] def line_states(content) when is_binary(content) do lines = String.split(content, "\n") {masked, st} = lines |> Enum.with_index() - |> Enum.map_reduce(%{q: nil, opened: nil, block: nil, depth: 0, pending: false}, fn + |> Enum.map_reduce(initial_state(lines), fn {line, n}, st -> mask_file_line(line, n, st) end) @@ -847,12 +961,31 @@ defmodule Hypatia.Rules.PinSites do end end + defp initial_state(lines), + do: %{ + q: nil, + opened: nil, + block: nil, + depth: 0, + pending: false, + keep: nil, + lines: List.to_tuple(lines) + } + # Inside a block scalar: a blank line or one deeper than the key is its - # content; anything else ends it. + # content; anything else ends it. The one line that is a `uses:` key's or + # an anchor's whole reference (`keep`, `block_value/4`) is kept as written. defp mask_file_line(line, n, %{block: col} = st) when is_integer(col) do - if String.trim(line) == "" or indent_of(line) > col, - do: {{String.duplicate("x", byte_size(line)), :text}, st}, - else: mask_file_line(line, n, %{st | block: nil}) + cond do + n == st.keep -> + {{line, :value}, st} + + String.trim(line) == "" or indent_of(line) > col -> + {{String.duplicate("x", byte_size(line)), :text}, st} + + true -> + mask_file_line(line, n, %{st | block: nil, keep: nil}) + end end defp mask_file_line(line, n, st) do @@ -892,7 +1025,7 @@ defmodule Hypatia.Rules.PinSites do # One line outside any scalar, `seen` being what precedes `rest` on it # (quote bodies as `x`). Tracks flow depth, opens a quote that does not # close on the line, and at its end a block scalar or a pending value. - defp walk(<<>>, seen, _n, st) do + defp walk(<<>>, seen, n, st) do trimmed = String.trim_trailing(seen) cond do @@ -901,7 +1034,8 @@ defmodule Hypatia.Rules.PinSites do st.depth == 0 and Regex.match?(~r/(?:^\s*(?:-\s+)*|:\s+)(?:[&!]\S*\s+)*[|>][-+0-9]*$/, trimmed) -> - %{st | block: block_parent(seen), pending: false} + col = block_parent(seen) + %{st | block: col, pending: false, keep: block_value(st.lines, n, col, trimmed)} true -> # A key or dash followed only by node properties (`- run: &cmd`, @@ -967,6 +1101,35 @@ defmodule Hypatia.Rules.PinSites do else: byte_size(hd(Regex.run(~r/^\s*(?:-\s+)*/, seen))) end + # A block scalar a `uses:` key holds (`- uses: >-`), or one an anchor names + # (`BAD: &bad |`), whose content is a single line holding nothing but a + # reference (` github/codeql-action/init@`): YAML resolves the + # scalar to that reference, so that line is the value. Its index in + # `lines`, or nil — for any other block scalar, and for one whose content + # runs over more lines (`>-` folds them into one value, no reference). + # Inside a block scalar `#` is text, so a reference with a comment after + # it is no reference either. + defp block_value(lines, n, col, header) do + anchored? = + case Regex.run(~r/(?:^\s*(?:-\s+)*|:\s+)((?:[&!]\S*\s+)*)[|>][-+0-9]*$/, header) do + [_, props] -> String.contains?(props, "&") + _ -> false + end + + if anchored? or + Regex.match?(~r/^\s*(?:-\s+)*["']?uses["']?\s*:\s+(?:[&!]\S*\s+)*[|>][-+0-9]*$/, header) do + (n + 1)..(tuple_size(lines) - 1)//1 + |> Stream.map(&{&1, elem(lines, &1)}) + |> Stream.reject(fn {_, l} -> String.trim(l) == "" end) + |> Stream.take_while(fn {_, l} -> indent_of(l) > col end) + |> Enum.take(2) + |> case do + [{v, l}] -> if Regex.match?(@block_ref_re, l), do: v + _ -> nil + end + end + end + defp indent_of(line), do: byte_size(line) - byte_size(String.trim_leading(line, " ")) # `prefix` is the masked line up to the site's action. diff --git a/lib/rules/pr_automerge.ex b/lib/rules/pr_automerge.ex index 2c95a444..20cebd85 100644 --- a/lib/rules/pr_automerge.ex +++ b/lib/rules/pr_automerge.ex @@ -696,8 +696,9 @@ defmodule Hypatia.Rules.PrAutomerge do # `%{hunk_index => [pin]}` on the anchor's line: `- uses: *bad` in context # with `BAD: &bad @` changed (`PinSites.alias_sites/2`), or a # step given whole as an alias whose anchored mapping's `uses:` changed - # (`PinSites.node_alias_sites/1`). The re-pinned anchor is the site. One - # that is already a site on its own line is counted there, not twice. + # (`PinSites.node_alias_sites/1`). The re-pinned anchor is the site, once + # however many readers reach it; one that is already a site on its own + # line is counted there, not twice. defp side_aliases(view) do texts = for {_, text, state} <- view, do: if(state == :text, do: "", else: text) side = Enum.join(texts, "\n") @@ -705,6 +706,7 @@ defmodule Hypatia.Rules.PrAutomerge do masked = PinSites.mask_lines(side) (PinSites.alias_sites(masked, PinSites.flow_prefixes(side)) ++ PinSites.node_alias_sites(side)) + |> Enum.uniq_by(&{&1.anchor_line, &1.site.ref_at}) |> Enum.reject(fn %{anchor_line: n, site: site} -> texts |> Enum.at(n - 1) |> PinSites.sites() |> Enum.any?(&(&1.ref_at == site.ref_at)) end) @@ -716,7 +718,8 @@ defmodule Hypatia.Rules.PrAutomerge do # One side of a hunk (context lines plus the changed lines of `mark`), in # order, as `{hunk_index, text, state}` — `state` as - # `PinSites.line_states/1` reads the side: `:yaml`, `:text` or `:open`; a + # `PinSites.line_states/1` reads the side: `:yaml`, `:text`, `:value` (a + # `uses:` block scalar's one reference line) or `:open`; a # block `uses:` key the side shows under no step or job # (`PinSites.data_uses_lines/1`) is `:text` too. defp side_view(hunk, mark) do @@ -732,7 +735,7 @@ defmodule Hypatia.Rules.PrAutomerge do states = PinSites.line_states(side) # Unless a step or job given whole as an alias runs the mapping it sits # in (`- *step`, the mapping anchored in a matrix entry). - run = for %{anchor_line: n} <- PinSites.node_alias_sites(side), into: MapSet.new(), do: n + run = for %{key_line: n} <- PinSites.node_alias_sites(side), into: MapSet.new(), do: n data = MapSet.difference(PinSites.data_uses_lines(side), run) entries @@ -750,7 +753,7 @@ defmodule Hypatia.Rules.PrAutomerge do certain? = state != :open and (before ++ rest) - |> Enum.reject(fn {_, t, s} -> s == :text or String.trim(t) == "" end) + |> Enum.reject(fn {_, t, s} -> s in [:text, :value] or String.trim(t) == "" end) |> Enum.filter(fn {_, t, _} -> indent_of(t) <= indent end) |> Enum.all?(fn {_, t, _} -> yaml_like?(t) end) diff --git a/lib/rules/workflow_audit.ex b/lib/rules/workflow_audit.ex index 1421313f..0f96cb0b 100644 --- a/lib/rules/workflow_audit.ex +++ b/lib/rules/workflow_audit.ex @@ -1206,7 +1206,7 @@ defmodule Hypatia.Rules.WorkflowAudit do # its `uses:` line is a live step's, judged as one, and routes to # review — the fixer does not rewrite anchored steps. aliased = - for %{anchor_line: n} <- Hypatia.Rules.PinSites.node_alias_sites(normalised), + for %{key_line: n} <- Hypatia.Rules.PinSites.node_alias_sites(normalised), into: MapSet.new(), do: n @@ -2086,7 +2086,7 @@ defmodule Hypatia.Rules.WorkflowAudit do |> Enum.with_index(1) |> Enum.take(pos_line) |> Enum.flat_map(fn {{l, {{_, state}, depth}}, number} -> - if state != :text and wf026_plain?(l) do + if state not in [:text, :value] and wf026_plain?(l) do masked = wf_mask_quoted(l) re @@ -2456,7 +2456,8 @@ defmodule Hypatia.Rules.WorkflowAudit do # `%{key_line => {joined, how}}`: the key's line with the reference in # place, which the rules read as they read a one-line key. `how` is # :continued for a reference YAML continues below the key (`- uses:`, then - # — past any blank or comment-only lines — the reference on a deeper line) + # — past any blank or comment-only lines — the reference on a deeper line; + # or a block scalar, `- uses: >-`, whose one content line is the reference) # and :alias for `uses: *name`, the reference the latest `&name` before it # anchors (`PinSites.alias_sites/1`). Found on the masked lines # (`PinSites.mask_lines/1`), so a `uses:` in a `run: |` script or a quoted @@ -2468,11 +2469,15 @@ defmodule Hypatia.Rules.WorkflowAudit do continued = for {key, _value, site} <- Hypatia.Rules.PinSites.continuations(masked), into: %{} do - line = Enum.at(lines, key - 1) - - {key, - {String.trim_trailing(Hypatia.Rules.PinSites.head(line)) <> - " " <> site.action <> "@" <> site.ref, :continued}} + # A block scalar's indicator (`>-`, `|`) is no part of the value. + head = + lines + |> Enum.at(key - 1) + |> Hypatia.Rules.PinSites.head() + |> String.trim_trailing() + |> String.replace(~r/\s+[|>][-+0-9]*$/, "") + + {key, {head <> " " <> site.action <> "@" <> site.ref, :continued}} end for %{line: n, at: at, len: len, site: site} <- @@ -2764,12 +2769,16 @@ defmodule Hypatia.Rules.WorkflowAudit do lets a caller write. A job's `uses:` whose value YAML continues below its key (past any blank - or comment-only lines) is the call all the same. + or comment-only lines, or as a block scalar) is the call all the same. So + is the `uses:` of a mapping a job is given whole as an alias (`rust: + *job`), wherever that mapping is anchored (`PinSites.node_alias_sites/1`). Auto-fixable: gitbot-fleet `scripts/fix-stale-reusable-pin.sh` rewrites the pin to the replacement the table names, never to "newest". It refuses a value continued below its key (exit 2, file untouched), so that finding - carries `auto_fixable: false` and routes to review. + carries `auto_fixable: false` and routes to review, as does a call made + through a job alias whose mapping is no job's own: the fixer rewrites the + keys of `jobs:` only. Severity is low and the finding is `advisory: true`: like WF026 the rule reports and routes to the fleet; it is never a blocking gate (AGENTIC @@ -2780,6 +2789,12 @@ defmodule Hypatia.Rules.WorkflowAudit do lines = String.split(content, "\n") continued = wf_continued_uses(lines) flow = wf_flow_lines(content) + # A job given whole as an alias (`rust: *job`) runs the mapping its + # anchor holds: that mapping's `uses:` line is a call wherever it sits. + aliased = + for %{key_line: n} <- Hypatia.Rules.PinSites.node_alias_sites(content), + into: MapSet.new(), + do: n Enum.flat_map(@broken_reusable_revisions, fn rev -> # Anchored to a `uses:` key at the start of a line (quoted or not) that @@ -2809,8 +2824,12 @@ defmodule Hypatia.Rules.WorkflowAudit do :error -> {line, nil} end + live? = number in aliased + # Each call with the layout the fleet fixer would meet: it rewrites - # a bare reference on a block key's own line, nothing else. + # a bare reference on a block key's own line, nothing else. A call + # only a job alias makes (its mapping anchored outside `jobs:`) is + # no key the fixer places. shas = case Regex.run(ref_re, line) do [_full, sha] -> @@ -2824,16 +2843,21 @@ defmodule Hypatia.Rules.WorkflowAudit do case wf027_job_key(lines, flow, number) do :block -> [{sha, review}] :flow -> [{sha, review || :flow}] + nil when live? -> [{sha, review || :node_alias}] nil -> [] end nil -> inline_re |> Regex.scan(line, return: :index) - |> Enum.filter(fn [{at, _} | _] -> - wf027_inline_job?(lines, flow, line, number, at + 1) + |> Enum.flat_map(fn [{at, _}, {sha_at, 40}] -> + cond do + wf027_inline_job?(lines, flow, line, number, at + 1) -> [:flow] + live? -> [:node_alias] + true -> [] + end + |> Enum.map(&{binary_part(line, sha_at, 40), &1}) end) - |> Enum.map(fn [_, {sha_at, 40}] -> {binary_part(line, sha_at, 40), :flow} end) end Enum.flat_map(shas, fn {sha, review} -> @@ -2983,6 +3007,12 @@ defmodule Hypatia.Rules.WorkflowAudit do " The job is written as a flow mapping; the repair script rewrites block-style " <> "jobs only, so this one routes to review." + defp wf027_review_note(:node_alias), + do: + " The call is made by a job given as an alias of a mapping anchored outside " <> + "`jobs:`; the repair script rewrites the keys of `jobs:` only, so this one " <> + "routes to review." + # ─── WF017: Secret-consuming action without secret-presence gate ────── # Actions whose primary input is a secret and that fail outright when diff --git a/test/rules/pin_integrity_test.exs b/test/rules/pin_integrity_test.exs index eb68c052..18184722 100644 --- a/test/rules/pin_integrity_test.exs +++ b/test/rules/pin_integrity_test.exs @@ -610,6 +610,71 @@ defmodule Hypatia.Rules.PinIntegrityTest do assert reason =~ "leaves a denylisted pin" end + test "a uses: key under a root key that holds no job is data in a whole workflow" do + for root <- ["env", "x-template"] do + content = + "#{root}:\n uses: github/codeql-action/init@#{@poison}\n" <> + "jobs:\n a:\n steps:\n - run: echo\n" + + assert PI.pi001_denylisted_pin(content, @policy) == [], root + assert {:ok, ^content, []} = PI.substitute_denylisted_pins(content, @policy) + end + + # Control: a fragment that shows no root (no `jobs:`) is judged on its + # own line, as before: a job's `uses:` for all it shows. + assert [%{line: 2}] = + PI.pi001_denylisted_pin( + "build:\n uses: github/codeql-action/init@#{@poison}\n", + @policy + ) + end + + test "a step alias whose mapping names its action by alias runs the anchored reference" do + content = + "env:\n BAD: &bad github/codeql-action/init@#{@poison}\n" <> + "jobs:\n a:\n strategy:\n matrix:\n include:\n" <> + " - &step\n uses: *bad\n" <> + " steps:\n - *step\n" + + # The site is the anchor's value, where the repair rewrites it. + assert [%{line: 2}] = PI.pi001_denylisted_pin(content, @policy) + assert {:ok, fixed, [_]} = PI.substitute_denylisted_pins(content, @policy) + refute fixed =~ @poison + + # Control: with no step alias the matrix entry, alias and all, is data. + data = String.replace(content, " - *step\n", " - run: echo\n") + assert PI.pi001_denylisted_pin(data, @policy) == [] + end + + test "a uses: block scalar whose one line is a reference is a site on that line" do + content = + "jobs:\n a:\n steps:\n" <> + " - uses: >-\n github/codeql-action/init@#{@poison}\n" <> + " - uses: |\n github/codeql-action/init@#{@poison}\n" + + assert [%{line: 5}, %{line: 7}] = PI.pi001_denylisted_pin(content, @policy) + assert {:ok, fixed, [_, _]} = PI.substitute_denylisted_pins(content, @policy) + refute fixed =~ @poison + assert fixed =~ " - uses: >-\n github/codeql-action/init@#{@good}\n" + + # An anchored block scalar names the same reference to a `uses: *bad`. + anchored = + "env:\n BAD: &bad >-\n github/codeql-action/init@#{@poison}\n" <> + "jobs:\n a:\n steps:\n - uses: *bad\n" + + assert [%{line: 3}] = PI.pi001_denylisted_pin(anchored, @policy) + + # Controls: a block scalar that folds more lines into its value holds + # no reference; under `run:` or an input it is text or data. + for other <- [ + " - uses: >-\n github/codeql-action/init@#{@poison}\n extra\n", + " - run: |\n github/codeql-action/init@#{@poison}\n", + " - uses: a/b@v1\n with:\n uses: >-\n github/codeql-action/init@#{@poison}\n" + ] do + assert PI.pi001_denylisted_pin("jobs:\n a:\n steps:\n" <> other, @policy) == [], other + end + end + test "a KYAML flow step on its own line is still a site" do content = "steps: [\n { name: \"x\", run: \"echo 'a\" },\n" <> diff --git a/test/rules/pr_automerge_test.exs b/test/rules/pr_automerge_test.exs index 07c409ed..48f4312a 100644 --- a/test/rules/pr_automerge_test.exs +++ b/test/rules/pr_automerge_test.exs @@ -741,6 +741,74 @@ defmodule Hypatia.Rules.PrAutomergeTest do end end + test "a uses: key under a root data key at column 0 is data: no poison site, no pin-only change" do + patch = """ + @@ -1,3 +1,3 @@ + env: + - uses: github/codeql-action/init@#{@good} + + uses: github/codeql-action/init@#{@poison} + """ + + f = file(".github/workflows/ci.yml", patch) + assert PA.poison_sites([f], @policy) == [] + refute PA.pin_lines_only?(f) + + # Control: a job's own key is the pin it was. + job = """ + @@ -1,4 +1,4 @@ + jobs: + a: + - uses: github/codeql-action/init@#{@good} + + uses: github/codeql-action/init@#{@poison} + """ + + assert [%{denylist_id: "PIN-001"}] = + PA.poison_sites([file(".github/workflows/ci.yml", job)], @policy) + end + + test "a re-pinned block scalar a uses: key holds is the site it resolves to" do + patch = """ + @@ -1,3 +1,3 @@ + - uses: >- + - github/codeql-action/init@#{@good} + + github/codeql-action/init@#{@poison} + """ + + f = file(".github/workflows/codeql.yml", patch) + assert [%{denylist_id: "PIN-001"}] = PA.poison_sites([f], @policy) + assert PA.pin_lines_only?(f) + + # Control: the same block scalar under `run:` is script text. + run = String.replace(patch, "- uses: >-", "- run: >-") + assert PA.poison_sites([file(".github/workflows/codeql.yml", run)], @policy) == [] + refute PA.pin_lines_only?(file(".github/workflows/codeql.yml", run)) + end + + test "a step alias whose mapping names its action by alias makes the anchor's re-pin a poison site" do + patch = """ + @@ -1,11 +1,11 @@ + env: + - BAD: &bad github/codeql-action/init@#{@good} + + BAD: &bad github/codeql-action/init@#{@poison} + jobs: + a: + strategy: + matrix: + include: + - &step + uses: *bad + steps: + - *step + """ + + f = file(".github/workflows/ci.yml", patch) + assert [%{certain: true}] = PA.poison_sites([f], @policy) + + # Control: with no step alias the matrix entry, alias and all, is data. + data = String.replace(patch, " - *step", " - run: echo") + assert PA.poison_sites([file(".github/workflows/ci.yml", data)], @policy) == [] + end + test "a pin inside a multi-line quoted run: is text; one the hunk never closes is uncertain" do # The quote opens on a context line and closes inside the hunk: every # line between is string text, never a site. diff --git a/test/rules/workflow_audit_pins_test.exs b/test/rules/workflow_audit_pins_test.exs index b39a59b3..7cdfaabb 100644 --- a/test/rules/workflow_audit_pins_test.exs +++ b/test/rules/workflow_audit_pins_test.exs @@ -1154,6 +1154,41 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do assert [] == wf026(String.replace(wf, " - *step\n", " - run: echo\n")) end + test "a step alias whose mapping names the action by alias is judged at its uses: key" do + wf = """ + env: + TC: &tc dtolnay/rust-toolchain@#{@toolchain_sha} + jobs: + check: + strategy: + matrix: + include: + - &step + uses: *tc + steps: + - *step + """ + + assert [%{line: 9, auto_fixable: false}] = wf026(wf) + assert [] == wf026(String.replace(wf, " - *step\n", " - run: echo\n")) + end + + test "a uses: block scalar whose one line is the reference is the step's key; routed to review" do + wf = """ + jobs: + check: + steps: + - uses: >- + dtolnay/rust-toolchain@#{@toolchain_sha} + """ + + assert [%{line: 4, auto_fixable: false} = f] = wf026(wf) + assert f.reason =~ "continued below its key" + + # Control: with the input given, the block-scalar step is silent. + assert [] == wf026(wf <> " with:\n toolchain: stable\n") + end + test "a toolchain: key with no value is no input: null and ~ included" do wf = """ jobs: @@ -1362,6 +1397,40 @@ defmodule Hypatia.Rules.WorkflowAuditPinsTest do assert WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => good}) == [] end + test "a job given whole as an alias calls the reusable its anchored mapping names; routed to review" do + for template <- [ + "template: &job\n uses: #{@rust_ci}@#{@broken_standards}\n", + "template: &job { uses: #{@rust_ci}@#{@broken_standards} }\n" + ] do + wf = template <> "jobs:\n rust: *job\n" + + assert [%{auto_fixable: false, fix_script: nil} = f] = + WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => wf}) + + assert f.reason =~ "given as an alias" + + # Control: with no job alias the anchored mapping is data. + plain = template <> "jobs:\n rust:\n uses: #{@rust_ci}@#{@good_standards}\n" + assert WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => plain}) == [] + end + + # A job anchored under `jobs:` and aliased is still the fixer's: the + # anchored job's own key is the one it rewrites. + own = "jobs:\n base: &job\n uses: #{@rust_ci}@#{@broken_standards}\n rust: *job\n" + + assert [%{line: 3, auto_fixable: true}] = + WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => own}) + end + + test "a job's uses: block scalar whose one line is the call is the call; routed to review" do + wf = "jobs:\n rust-ci:\n uses: >-\n #{@rust_ci}@#{@broken_standards}\n" + + assert [%{line: 3, auto_fixable: false} = f] = + WorkflowAudit.check_known_broken_reusable_revision(%{"rust-ci.yml" => wf}) + + assert f.reason =~ "continued below its key" + end + test "a flow-style job written without indentation is still a job; routed to review" do # Every line in column 0: the brackets, not the indentation, say the # `uses:` is a job's key. The fleet fixer places a key by indentation From 407b1867c2fe83e7a30bfe8c84a65b803ddcae24 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 12:14:32 +0000 Subject: [PATCH 47/47] =?UTF-8?q?fix(pins):=20read=20KYAML=20as=20GitHub?= =?UTF-8?q?=20does=20=E2=80=94=20double-quoted=20values=20decoded,=20compo?= =?UTF-8?q?site-action=20steps=20in=20flow=20form?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit KYAML is the estate's authoring dialect (YAML-POLICY Y-3, D280): every value double-quoted, every collection in flow form. Checked against the estate's own rewriter (`yq -p yaml -o kyaml`, v4.53.3) over 156 workflows and composite actions from six estate repositories: PinIntegrity's sites agreed with the YAML originals on 155 and missed every pin of the one KYAML composite action. - PinSites: a composite action's root `runs:` steps are sites in flow form (`{ runs: { steps: [ { uses: ... } ] } }`), as in block form. `runs` must be the document's own key: the outermost `{`, or directly inside a root `{`; a `runs:` mapping inside an input stays data. - PinSites.decode_dq/1: a YAML double-quoted body decoded (YAML 1.2 escapes, including the JSON-style ones `yq -o kyaml` writes) with the source span of every decoded byte. sites/2 and continued_site/2 read an escaped `uses` value through it: `"github/codeql-action/init@\x31c5b..."` is the denylisted commit GitHub runs. The site carries `ref_len`, the ref's length as written, so the PI001 repair replaces exactly those bytes. An undefined escape decodes to nothing; a double-quoted value continued over lines is not read. - PinSites.pin_line?/1: a line whose `uses` value holds an escape is no plain re-pin, so PA never counts it pin-only; PA001 reads its decoded pin. On 215a820 an escaped poisoned re-pin read as a clean pin-only change. After: 156/156 agree. Tests for each case fail on 215a820 and pass here. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Xn9nqHVbUTRLgBdedoXAc8 --- lib/rules/pin_integrity.ex | 7 +- lib/rules/pin_sites.ex | 226 ++++++++++++++++++++++++++++-- test/rules/pin_integrity_test.exs | 58 ++++++++ test/rules/pr_automerge_test.exs | 19 +++ 4 files changed, 293 insertions(+), 17 deletions(-) diff --git a/lib/rules/pin_integrity.ex b/lib/rules/pin_integrity.ex index 41b2ed42..3124d43f 100644 --- a/lib/rules/pin_integrity.ex +++ b/lib/rules/pin_integrity.ex @@ -78,7 +78,9 @@ defmodule Hypatia.Rules.PinIntegrity do Returns a list of maps with `:line`, `:action` (the bare action path as written, e.g. `github/codeql-action/init`), `:action_base` (the first two path segments, which is what an allow/deny list keys on), `:ref`, - `:ref_at` (the ref's byte offset in its line) and `:comment` — the + `:ref_at` (the ref's byte offset in its line), `:ref_len` (its length as + written — a value written with escapes, `"a/b@\\x31…"`, is read decoded, + so the ref's text is longer than the ref) and `:comment` — the line's comment, trimmed, when the site is the line's only one, else `""`: `[ { uses: a/b@x }, { uses: c/d@y } ] # v4` labels neither pin. """ @@ -151,6 +153,7 @@ defmodule Hypatia.Rules.PinIntegrity do action_base: action_base(site.action), ref: site.ref, ref_at: site.ref_at, + ref_len: Map.get(site, :ref_len, byte_size(site.ref)), comment: comment } end @@ -633,7 +636,7 @@ defmodule Hypatia.Rules.PinIntegrity do |> Enum.sort_by(& &1.ref_at, :desc) |> Enum.reduce(original, fn site, acc -> %{sha: sha} = known_good(policy, site.action) - tail_at = site.ref_at + byte_size(site.ref) + tail_at = site.ref_at + site.ref_len binary_part(acc, 0, site.ref_at) <> sha <> binary_part(acc, tail_at, byte_size(acc) - tail_at) diff --git a/lib/rules/pin_sites.ex b/lib/rules/pin_sites.ex index d8d23983..7324b83b 100644 --- a/lib/rules/pin_sites.ex +++ b/lib/rules/pin_sites.ex @@ -21,6 +21,12 @@ defmodule Hypatia.Rules.PinSites do `#`, inside an input's mapping (`with: { uses: ... }`), or after free text (`echo { uses: ... }` in a script). + A double-quoted value is read as YAML decodes it — KYAML (YAML-POLICY + Y-3) writes every value so, and an escape (`"a/b@\\x31…"`) is the + reference GitHub runs, not its spelling (`decode_dq/1`). A composite + action's steps are its root `runs:` steps in flow form too (`{ runs: { + steps: [ { uses: ... } ] } }`), as they are in block form. + Every position this module returns is a byte offset into the line as given: the mask keeps the line's bytes where they were. """ @@ -55,9 +61,13 @@ defmodule Hypatia.Rules.PinSites do @spec sites(String.t(), String.t()) :: [map()] def sites(line, flow_prefix \\ "") when is_binary(line) and is_binary(flow_prefix) do masked = mask(line) + escaped = escaped_values(masked, flow_prefix) masked |> scan(flow_prefix) + |> Enum.reject(fn [_full, {a_at, _} | _] -> + Enum.any?(escaped, &(a_at >= &1.from and a_at <= &1.to)) + end) |> Enum.map(fn [_full, {a_at, a_len}, {r_at, r_len}] -> %{ action: binary_part(masked, a_at, a_len), @@ -66,8 +76,150 @@ defmodule Hypatia.Rules.PinSites do ref_at: r_at } end) + |> Kernel.++(for %{site: %{} = site} <- escaped, do: site) + |> Enum.sort_by(& &1.action_at) end + # The double-quoted `uses` values of a masked line that hold an escape + # (`"github/codeql-action/init@\x31c5b…"`), each as `%{from, to, site}`: + # the body's byte span and the site its value decodes to (`decoded_site/2`), + # or nil — a value that is no structural site, that the line does not + # close (a double-quoted scalar continued over lines is not read here), or + # that does not decode to a reference. Such a value is read only through + # its decoding: GitHub runs the decoded reference, never the text. + defp escaped_values(masked, flow_prefix) do + ~r/(?:^|[^A-Za-z0-9_.\/-])["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*"/ + |> Regex.scan(masked, return: :index) + |> Enum.flat_map(fn [{at, len}] -> + from = at + len + + {body, close, _tail} = + take_quoted(binary_part(masked, from, byte_size(masked) - from), ?", "") + + prefix = binary_part(masked, 0, from) + + structural? = + if flow_prefix == "", + do: structural?(prefix), + else: structural_in_flow?(flow_prefix <> prefix) + + cond do + not String.contains?(body, "\\") -> + [] + + structural? and close != "" -> + [%{from: from, to: from + byte_size(body), site: decoded_site(body, from)}] + + true -> + [%{from: from, to: from + byte_size(body), site: nil}] + end + end) + end + + # The site a double-quoted body (starting at byte `from` of its line) + # decodes to: the action and ref as YAML reads them, `action_at`/`ref_at` + # the source bytes they start at, `ref_len` the source length of the ref + # as written (a repair replaces exactly those bytes), `escaped: true`. + defp decoded_site(body, from) do + with {:ok, decoded, spans} <- decode_dq(body), + [_, {a_at, a_len}, {r_at, r_len}] <- + Regex.run(~r/\A([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)/, decoded, + return: :index + ) do + spans = List.to_tuple(spans) + {action_from, _} = elem(spans, a_at) + {ref_from, _} = elem(spans, r_at) + {_, ref_to} = elem(spans, r_at + r_len - 1) + + %{ + action: binary_part(decoded, a_at, a_len), + ref: binary_part(decoded, r_at, r_len), + action_at: from + action_from, + ref_at: from + ref_from, + ref_len: ref_to - ref_from, + escaped: true + } + else + _ -> nil + end + end + + @doc """ + The body of a YAML double-quoted scalar (the text between its quotes, on + one line) as YAML decodes it: `{:ok, decoded, spans}`, `spans` the source + byte span `{from, to}` of every decoded byte, or `:error` for an escape + YAML does not define. The escapes are YAML 1.2's: `\\0 \\a \\b \\t \\n \\v + \\f \\r \\e \\ \\" \\/ \\\\ \\N \\_ \\L \\P`, `\\xXX`, `\\uXXXX`, `\\UXXXXXXXX` — the + JSON-style subset is what `yq -o kyaml` writes. + """ + @spec decode_dq(String.t()) :: + {:ok, String.t(), [{non_neg_integer(), non_neg_integer()}]} | :error + def decode_dq(body) when is_binary(body), do: decode_dq(body, 0, [], []) + + defp decode_dq(<<>>, _at, out, spans), + do: {:ok, out |> Enum.reverse() |> IO.iodata_to_binary(), Enum.reverse(spans)} + + defp decode_dq(<>, at, out, spans) do + case escape(rest) do + {:ok, bytes, used} -> + span = {at, at + 1 + used} + spans = Enum.reduce(1..byte_size(bytes)//1, spans, fn _, acc -> [span | acc] end) + + decode_dq( + binary_part(rest, used, byte_size(rest) - used), + at + 1 + used, + [bytes | out], + spans + ) + + :error -> + :error + end + end + + defp decode_dq(<>, at, out, spans), + do: decode_dq(rest, at + 1, [<> | out], [{at, at + 1} | spans]) + + @simple_escapes %{ + ?0 => <<0>>, + ?a => <<7>>, + ?b => <<8>>, + ?t => <<9>>, + ?\t => <<9>>, + ?n => <<10>>, + ?v => <<11>>, + ?f => <<12>>, + ?r => <<13>>, + ?e => <<27>>, + ?\s => " ", + ?" => "\"", + ?/ => "/", + ?\\ => "\\", + ?N => <<0x85::utf8>>, + ?_ => <<0xA0::utf8>>, + ?L => <<0x2028::utf8>>, + ?P => <<0x2029::utf8>> + } + + # The escape after a `\` in `rest`: `{:ok, bytes, bytes of rest used}`. + defp escape(<>) when is_map_key(@simple_escapes, c), + do: {:ok, Map.fetch!(@simple_escapes, c), 1} + + defp escape(<>) when c in [?x, ?u, ?U] do + n = %{?x => 2, ?u => 4, ?U => 8}[c] + + with <> <- rest, + true <- Regex.match?(~r/\A[0-9A-Fa-f]+\z/, hex), + cp = String.to_integer(hex, 16), + true <- cp <= 0x10FFFF and cp not in 0xD800..0xDFFF do + {:ok, <>, 1 + n} + else + _ -> :error + end + end + + defp escape(_), do: :error + @doc """ The structural site matches of an already masked line, each as `[full, action, ref]` `{byte, length}` pairs (`Regex.scan/3` with @@ -132,12 +284,14 @@ defmodule Hypatia.Rules.PinSites do indentation, list dashes, flow brackets and commas — so `- uses: a/b@x`, `{ uses: "a/b@x" },` and `[ { uses: a/b@x }, { uses: c/d@y } ]` are pin lines, and `steps: [ { uses: a/b@x } ]`, `- { name: x, uses: a/b@x }` - and `uses: a/b@x extra` are not. The comment is never part of it. + and `uses: a/b@x extra` are not. The comment is never part of it. Nor + is a line whose `uses` value is written with escapes (`"a/b@\\x31…"`): its + reference is read decoded (`sites/2`), but it is no plain re-pin. """ @spec pin_line?(String.t()) :: boolean() def pin_line?(line) when is_binary(line) do masked = mask(line) - sites = scan(masked) + sites = if escaped_values(masked, "") == [], do: scan(masked), else: [] remaining = sites @@ -194,29 +348,33 @@ defmodule Hypatia.Rules.PinSites do def continued_site(key_line, next_line) when is_binary(key_line) and is_binary(next_line) do key = head(key_line) - value_re = + # Each form's value: the reference it may be, and what may follow it. + form = cond do m = Regex.run(~r/^(\s*(?:-\s+)?)["']?uses["']?\s*:(?:\s+[&!]\S*)*\s*$/, key) -> if indent_of(next_line) > byte_size(Enum.at(m, 1)), do: - ~r/^\s*(?:[&!]\S*\s+)*["']?([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)["']?\s*(?:#.*)?$/ + {~r/^\s*(?:[&!]\S*\s+)*["']?([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)["']?\s*(?:#.*)?$/, + ~r/\A\s*(?:#.*)?\z/} m = Regex.run( ~r/^(\s*(?:-\s+)?)["']?uses["']?\s*:(?:\s+[&!]\S*)*\s+[|>][-+0-9]*\s*$/, key ) -> - if indent_of(next_line) > byte_size(Enum.at(m, 1)), do: @block_ref_re + if indent_of(next_line) > byte_size(Enum.at(m, 1)), do: {@block_ref_re, nil} Regex.match?(~r/[{,]\s*["']?uses["']?\s*:(?:\s+[&!]\S*)*\s*$/, key) and structural?(String.trim_trailing(key) <> " ") -> - ~r/^\s*(?:[&!]\S*\s+)*["']?([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)["']?\s*(?:[,}\]]|#|$)/ + {~r/^\s*(?:[&!]\S*\s+)*["']?([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\/-]*?)@([^\s#"',}\]]+)["']?\s*(?:[,}\]]|#|$)/, + ~r/\A\s*(?:[,}\]]|#|\z)/} true -> nil end - with %Regex{} <- value_re, + with {value_re, tail_re} <- form, + :plain <- escaped_value(next_line, tail_re), [_, {a_at, a_len}, {r_at, r_len}] <- Regex.run(value_re, next_line, return: :index) do %{ action: binary_part(next_line, a_at, a_len), @@ -225,10 +383,28 @@ defmodule Hypatia.Rules.PinSites do ref_at: r_at } else + {:escaped, site} -> site _ -> nil end end + # A continued value written double-quoted with an escape (`"a/b@\x31…"`) + # is read decoded: `{:escaped, site}` — nil when it decodes to no + # reference or more than `tail_re` allows follows its closing quote — or + # :plain for any other value (a block scalar's, `tail_re` nil, included). + defp escaped_value(_line, nil), do: :plain + + defp escaped_value(line, tail_re) do + with [{0, from}] <- Regex.run(~r/\A\s*(?:[&!]\S*\s+)*"/, line, return: :index), + {body, close, tail} = + take_quoted(binary_part(line, from, byte_size(line) - from), ?", ""), + true <- String.contains?(body, "\\") do + {:escaped, if(close != "" and Regex.match?(tail_re, tail), do: decoded_site(body, from))} + else + _ -> :plain + end + end + @doc """ Every `uses:` whose value is an alias (`- uses: *checkout`, a block key or a step/job flow entry as `sites/1` reads one), resolved to the site the @@ -1139,7 +1315,7 @@ defmodule Hypatia.Rules.PinSites do true Regex.match?(~r/[{,]\s*["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*["']?$/, prefix) -> - prefix |> flow_stack() |> structural_flow_stack?() + prefix |> flow_stack() |> structural_flow_stack?(prefix) true -> false @@ -1152,7 +1328,7 @@ defmodule Hypatia.Rules.PinSites do # and the brackets still open must make it a step's or a job's own. defp structural_in_flow?(text) do Regex.match?(~r/(?:[{,]|\n)\s*(?:-\s+)?["']?uses["']?\s*:\s*(?:[&!]\S*\s+)*["']?$/, text) and - text |> flow_stack() |> structural_flow_stack?() + text |> flow_stack() |> structural_flow_stack?(text) end # The brackets still open in `text`, innermost first, each with what opens @@ -1187,14 +1363,34 @@ defmodule Hypatia.Rules.PinSites do # line, which this reader cannot see past), or one that goes on through a # job key into `jobs` (`jobs: { b: { steps: [ {`). The same `steps:` # inside any other mapping (`fixture: { steps: [ {` in a matrix) is data. - defp structural_flow_stack?(stack) do + # A composite action's steps are those of its root `runs:` (`runs: { + # steps: [ {`, KYAML or a flow line at column 0): `runs` must be the + # document's own key, its `{` the outermost bracket or one directly inside + # a root `{` the text starts with (`text`, from where the collection + # opened). + defp structural_flow_stack?(stack, text) do Enum.all?(stack, fn {_, key} -> key != :text end) and case stack do - [{"{", :none}] -> true - [{"{", :none}, {"[", key}] when key in ["steps", :none] -> true - [{"{", :none}, {"[", "steps"}, {"{", job}, {"{", "jobs"} | _] when is_binary(job) -> true - [{"{", job}, {"{", "jobs"} | _] when is_binary(job) -> true - _ -> false + [{"{", :none}] -> + true + + [{"{", :none}, {"[", key}] when key in ["steps", :none] -> + true + + [{"{", :none}, {"[", "steps"}, {"{", job}, {"{", "jobs"} | _] when is_binary(job) -> + true + + [{"{", job}, {"{", "jobs"} | _] when is_binary(job) -> + true + + [{"{", :none}, {"[", "steps"}, {"{", "runs"}] -> + Regex.match?(~r/\A["']?runs["']?\s*:/, text) + + [{"{", :none}, {"[", "steps"}, {"{", "runs"}, {"{", :none}] -> + String.starts_with?(text, "{") + + _ -> + false end end diff --git a/test/rules/pin_integrity_test.exs b/test/rules/pin_integrity_test.exs index 18184722..af7c5ef4 100644 --- a/test/rules/pin_integrity_test.exs +++ b/test/rules/pin_integrity_test.exs @@ -675,6 +675,64 @@ defmodule Hypatia.Rules.PinIntegrityTest do end end + test "a composite action written as KYAML: its runs: steps are sites" do + # `yq -p yaml -o kyaml` of a composite action (YAML-POLICY Y-3). + content = """ + { + name: "setup", + runs: { + using: "composite", + steps: [ + { + uses: "github/codeql-action/init@#{@poison}", # v4.38.1 + }, + ], + }, + } + """ + + assert [%{line: 7}] = PI.pi001_denylisted_pin(content, @policy) + assert {:ok, fixed, [_]} = PI.substitute_denylisted_pins(content, @policy) + assert fixed =~ ~s(uses: "github/codeql-action/init@#{@good}", # v4.38.0) + + # Control: a `runs:` mapping that is an input's value, not the action's + # own root key, holds data. + data = + "runs:\n using: composite\n steps:\n - uses: some/wrapper@v1\n with:\n" <> + " runs: { steps: [ { uses: github/codeql-action/init@#{@poison} } ] }\n" + + assert PI.pi001_denylisted_pin(data, @policy) == [] + end + + test "a double-quoted value is the reference YAML decodes it to" do + tail = String.slice(@poison, 1..-1//1) + escaped = "\\x31" <> tail + steps = "jobs:\n a:\n steps:\n" + + # An escape in the ref, an escaped `/` in the action, the `\u` form + # and a value continued below its key: each is the denylisted + # commit GitHub runs, and the repair rewrites exactly the ref's bytes. + for {value, repaired} <- [ + {~s( - uses: "github/codeql-action/init@#{escaped}"\n), + ~s( - uses: "github/codeql-action/init@#{@good}"\n)}, + {~s( - uses: "github\\/codeql-action/init@#{@poison}"\n), + ~s( - uses: "github\\/codeql-action/init@#{@good}"\n)}, + {~s( - { uses: "github/codeql-action/init@\\u0031#{tail}" }\n), + ~s( - { uses: "github/codeql-action/init@#{@good}" }\n)}, + {~s( - uses:\n "github/codeql-action/init@#{escaped}"\n), + ~s( - uses:\n "github/codeql-action/init@#{@good}"\n)} + ] do + content = steps <> value + assert [_] = PI.pi001_denylisted_pin(content, @policy), value + assert {:ok, fixed, [_]} = PI.substitute_denylisted_pins(content, @policy) + assert fixed == steps <> repaired + end + + # Control: an escape YAML does not define decodes to nothing. + bad = steps <> ~s( - uses: "github/codeql-action/init@\\q#{@poison}"\n) + assert PI.pi001_denylisted_pin(bad, @policy) == [] + end + test "a KYAML flow step on its own line is still a site" do content = "steps: [\n { name: \"x\", run: \"echo 'a\" },\n" <> diff --git a/test/rules/pr_automerge_test.exs b/test/rules/pr_automerge_test.exs index 48f4312a..26c65199 100644 --- a/test/rules/pr_automerge_test.exs +++ b/test/rules/pr_automerge_test.exs @@ -809,6 +809,25 @@ defmodule Hypatia.Rules.PrAutomergeTest do assert PA.poison_sites([file(".github/workflows/ci.yml", data)], @policy) == [] end + test "a pin written with escapes is the reference it decodes to, and no plain re-pin" do + escaped = "\\x31" <> String.slice(@poison, 1..-1//1) + + patch = """ + @@ -1,3 +1,3 @@ + steps: + - - uses: "github/codeql-action/init@#{@good}" + + - uses: "github/codeql-action/init@#{escaped}" + """ + + f = file(".github/workflows/ci.yml", patch) + assert [%{denylist_id: "PIN-001"}] = PA.poison_sites([f], @policy) + refute PA.pin_lines_only?(f) + + # Control: the same re-pin written plainly is the pin-only change it was. + plain = String.replace(patch, escaped, @poison) + assert PA.pin_lines_only?(file(".github/workflows/ci.yml", plain)) + end + test "a pin inside a multi-line quoted run: is text; one the hunk never closes is uncertain" do # The quote opens on a context line and closes inside the hunk: every # line between is string text, never a site.