diff --git a/scripts/apply-branch-gates.sh b/scripts/apply-branch-gates.sh index dae3ab712..815eb954a 100755 --- a/scripts/apply-branch-gates.sh +++ b/scripts/apply-branch-gates.sh @@ -66,7 +66,9 @@ # --gates-file F default config/rulesets/gates.json # --strip-retired also remove the 4 retired rule types. Off by default. # --require-green N drop any derived context that is not green across the -# last N default-branch runs of its workflow. Off (0) by +# last N default-branch runs of its workflow, and REFUSE +# the repo outright if any of those runs cannot be read. +# Off (0) by # default. Owner ruling on #956: "require the reliably- # green set" -- a required context that is currently red # blocks the branch the moment it is required, so gating @@ -210,14 +212,32 @@ while IFS= read -r R; do done < "$WORK/gatewf" # ---- 2. DERIVE contexts from real runs -------------------------------- - : > "$WORK/ctx"; NORUN='' + # FAIL CLOSED. A swallowed API error here raises nothing -- it silently + # SHORTENS the list, and the gate is written weaker than intended while + # every other line of output still reports success. (Measured 2026-09-22: + # this dropped 2 of 18 required contexts on standards/main.) So capture + # the exit status of every fetch and refuse to write if any one failed. + : > "$WORK/ctx"; : > "$WORK/gatewf3"; NORUN=''; DERIVEFAIL='' while IFS= read -r WFN; do [ -n "$WFN" ] || continue - RID=$(gh api "repos/$R/actions/workflows/$WFN/runs?branch=$DEF&per_page=1" \ - --jq '.workflow_runs[0].id // empty' 2>/dev/null) + if ! RID=$(gh api "repos/$R/actions/workflows/$WFN/runs?branch=$DEF&per_page=1" \ + --jq '.workflow_runs[0].id // empty'); then + DERIVEFAIL="${DERIVEFAIL:+$DERIVEFAIL,}$WFN(runs-query-failed)"; continue + fi if [ -z "$RID" ]; then NORUN="${NORUN:+$NORUN,}$WFN"; continue; fi - gh api "repos/$R/actions/runs/$RID/jobs?per_page=100" --paginate \ - --jq '.jobs[]?|.name' 2>/dev/null >> "$WORK/ctx" + if ! gh api "repos/$R/actions/runs/$RID/jobs?per_page=100" --paginate \ + --jq '.jobs[]?|.name' > "$WORK/jobs1"; then + DERIVEFAIL="${DERIVEFAIL:+$DERIVEFAIL,}$WFN(jobs-query-failed)"; continue + fi + # a run that EXISTS but reports zero jobs is a failed read, not an empty gate + if [ ! -s "$WORK/jobs1" ]; then + DERIVEFAIL="${DERIVEFAIL:+$DERIVEFAIL,}$WFN(run $RID returned zero jobs)"; continue + fi + cat "$WORK/jobs1" >> "$WORK/ctx" + # Record the workflows that actually CONTRIBUTED contexts. The greenness + # probe below must iterate these, not gatewf2: gatewf2 still holds every + # NORUN workflow, for which an empty runs query is the LEGITIMATE state. + printf '%s\n' "$WFN" >> "$WORK/gatewf3" done < "$WORK/gatewf2" sort -u "$WORK/ctx" -o "$WORK/ctx" @@ -229,22 +249,53 @@ while IFS= read -r R; do done < "$WORK/ctx" # ---- optional: keep only contexts that are RELIABLY green ------------ + # The same rule in the other direction: an UNREAD run cannot prove a + # context green, so a failed fetch here must refuse, never silently admit. NOTGREEN='' if [ "$REQUIRE_GREEN" -gt 0 ] 2>/dev/null; then : > "$WORK/bad" while IFS= read -r WFN; do [ -n "$WFN" ] || continue - gh api "repos/$R/actions/workflows/$WFN/runs?branch=$DEF&per_page=$REQUIRE_GREEN" \ - --jq '.workflow_runs[]?.id' 2>/dev/null > "$WORK/rids" + if ! gh api "repos/$R/actions/workflows/$WFN/runs?branch=$DEF&per_page=$REQUIRE_GREEN" \ + --jq '.workflow_runs[]?.id' > "$WORK/rids"; then + DERIVEFAIL="${DERIVEFAIL:+$DERIVEFAIL,}$WFN(green-runs-query-failed)"; continue + fi + # Third instance of the same class. A runs query that SUCCEEDS with an + # empty list leaves rids empty, the loop below never runs, nothing lands + # in "bad", and every context this workflow contributed is admitted as + # green. We are iterating gatewf3 -- workflows that DID contribute + # contexts, hence had a run -- so zero runs here is a failed read. + if [ ! -s "$WORK/rids" ]; then + DERIVEFAIL="${DERIVEFAIL:+$DERIVEFAIL,}$WFN(green-runs-query-returned-none)"; continue + fi while IFS= read -r RID2; do [ -n "$RID2" ] || continue + # Ask for NAME + CONCLUSION of every job and classify locally. + # Asking the API only for the NON-green jobs cannot tell "this run is + # all green" apart from "this run was not read": both return ZERO + # lines, and zero lines is read as greenness. That is the same + # fail-open as the derivation loop above, pointing the other way -- + # there it SHORTENS the gate, here it ADMITS a context that was never + # shown to be green. An unread run proves nothing in either direction. + if ! gh api "repos/$R/actions/runs/$RID2/jobs?per_page=100" --paginate \ + --jq '.jobs[]? | [.name, (.conclusion // "pending")] | @tsv' \ + > "$WORK/jobs2"; then + DERIVEFAIL="${DERIVEFAIL:+$DERIVEFAIL,}$WFN(run $RID2 green-check-failed)"; continue + fi + # a run that EXISTS but reports zero jobs is a failed read, not a green run + if [ ! -s "$WORK/jobs2" ]; then + DERIVEFAIL="${DERIVEFAIL:+$DERIVEFAIL,}$WFN(run $RID2 returned zero jobs)"; continue + fi # a job is acceptable when success/skipped/neutral, or still running - gh api "repos/$R/actions/runs/$RID2/jobs?per_page=100" --paginate \ - --jq '.jobs[]? | select((.conclusion // "pending") as $c - | ["success","skipped","neutral","pending"] | index($c) | not) | .name' \ - 2>/dev/null >> "$WORK/bad" + while IFS=$'\t' read -r JN JC; do + [ -n "$JN" ] || continue + case "$JC" in + success|skipped|neutral|pending) ;; + *) printf '%s\n' "$JN" >> "$WORK/bad" ;; + esac + done < "$WORK/jobs2" done < "$WORK/rids" - done < "$WORK/gatewf2" + done < "$WORK/gatewf3" sort -u "$WORK/bad" -o "$WORK/bad" : > "$WORK/ctx3" while IFS= read -r C; do @@ -261,6 +312,15 @@ while IFS= read -r R; do [ -n "$NORUN" ] && DETAIL="$DETAIL no_run=[$NORUN]" [ -n "$EXCLUDED" ] && DETAIL="$DETAIL excluded=[$EXCLUDED]" + # ---- THE OTHER REFUSAL: a gate derived from an incomplete read -------- + # An unread run is not an absent context. Writing here would produce a + # real, plausible, permanent ruleset that is simply WEAKER than intended, + # reported as success, with nothing anywhere to say so. + if [ -n "$DERIVEFAIL" ]; then + emit "$R" "REFUSED" "$DETAIL derive_failed=[$DERIVEFAIL] — refusing to write a gate derived from an incomplete read" + continue + fi + # ---- THE REFUSAL: a rule with an empty list is a vacuous gate --------- if [ "$NCTX" -eq 0 ]; then emit "$R" "UNGATED" "$DETAIL — refusing to write an empty required_status_checks rule" diff --git a/scripts/tests/branch-gates-apply-test.sh b/scripts/tests/branch-gates-apply-test.sh index eff2d61f3..9be5c81b4 100755 --- a/scripts/tests/branch-gates-apply-test.sh +++ b/scripts/tests/branch-gates-apply-test.sh @@ -234,6 +234,216 @@ else ok "empty target list: refused" fi + +# ================================================================ CASE 8 +# THE FAIL-OPEN REFUSAL: a derivation fetch that FAILS must never be read as +# "this workflow contributes no contexts". Measured 2026-09-22 on +# hyperpolymath/standards: a swallowed error dropped 2 of 18 required +# contexts and the run still reported GATED. A short gate is a weak gate, +# and nothing in the output said so. +# +# Fixture shape: governance.yml resolves and yields jobs; codeql.yml resolves +# to run 22 but its JOBS fixture is ABSENT, so the shim exits non-zero -- +# exactly a transient API failure. +reset_fix +R=acme/flaky +mkfix "repos/$R" '{"default_branch":"main"}' +mkfix "repos/$R/contents/.github/workflows" '[{"name":"governance.yml"},{"name":"codeql.yml"}]' +mkfix "repos/$R/contents" '[{"name":"README.md"}]' +mkfix "repos/$R/actions/workflows/governance.yml/runs?branch=main&per_page=1" '{"workflow_runs":[{"id":11}]}' +mkfix "repos/$R/actions/workflows/codeql.yml/runs?branch=main&per_page=1" '{"workflow_runs":[{"id":22}]}' +mkfix "repos/$R/actions/runs/11/jobs?per_page=100" '{"jobs":[{"name":"governance / Governance"}]}' +# NOTE: no fixture for run 22's jobs -- the shim will exit 1. +mkfix "repos/$R/rulesets" '[{"id":9,"target":"branch","enforcement":"active"}]' +mkfix "repos/$R/rulesets/9" '{"name":"Base","target":"branch","enforcement":"active","conditions":{},"bypass_actors":[],"rules":[{"type":"deletion"}]}' + +OUT=$(run_applier "$R" --apply) +S=$(state_of "$OUT"); D=$(detail_of "$OUT") +[ "$S" = "REFUSED" ] && ok "flaky derive: state is REFUSED" || bad "flaky derive: state=$S (want REFUSED)" +case "$D" in *"derive_failed=["*"codeql.yml"*) ok "flaky derive: the failed workflow is NAMED, not silently absent" ;; *) bad "flaky derive: failure not reported — $D" ;; esac +[ -s "$FIX/PUTS.log" ] && bad "flaky derive: PUT a gate built from an incomplete read" || ok "flaky derive: no PUT even with --apply" + +# ---- MUTANT C: delete the incomplete-read refusal. The suite MUST go red. ---- +# This is the mutant that matters: without it the applier does not error, it +# writes a SHORTER gate and calls it success. +MUTC="$WORK/mutant-c.sh" +sed 's|if \[ -n "\$DERIVEFAIL" \]; then|if false; then|' "$APPLIER" > "$MUTC" +reset_fix +mkfix "repos/$R" '{"default_branch":"main"}' +mkfix "repos/$R/contents/.github/workflows" '[{"name":"governance.yml"},{"name":"codeql.yml"}]' +mkfix "repos/$R/contents" '[{"name":"README.md"}]' +mkfix "repos/$R/actions/workflows/governance.yml/runs?branch=main&per_page=1" '{"workflow_runs":[{"id":11}]}' +mkfix "repos/$R/actions/workflows/codeql.yml/runs?branch=main&per_page=1" '{"workflow_runs":[{"id":22}]}' +mkfix "repos/$R/actions/runs/11/jobs?per_page=100" '{"jobs":[{"name":"governance / Governance"}]}' +mkfix "repos/$R/rulesets" '[{"id":9,"target":"branch","enforcement":"active"}]' +mkfix "repos/$R/rulesets/9" '{"name":"Base","target":"branch","enforcement":"active","conditions":{},"bypass_actors":[],"rules":[{"type":"deletion"}]}' +OUT=$(MUTANT="$MUTC" run_applier "$R" --apply) +if [ "$(state_of "$OUT")" = "REFUSED" ]; then + bad "MUTANT C SURVIVED: refusal removed yet still REFUSED — the control is decorative" +else + ok "mutant C killed: without the refusal it becomes $(state_of "$OUT") and PUTs $(wc -l < "$FIX/PUTS.log") time(s)" +fi +if [ -r "$FIX/LAST_PUT.json" ] && + [ "$(jq '[.rules[]|select(.type=="required_status_checks")|.parameters.required_status_checks|length]|add // 0' "$FIX/LAST_PUT.json")" = "1" ]; then + ok "mutant C wrote the SHORTENED gate (1 context, not 2) — the silent weakening being guarded" +else + bad "mutant C: expected a 1-context gate from the incomplete read" +fi + +# ================================================================ CASE 9 +# THE SAME FAIL-OPEN, POINTING THE OTHER WAY: --require-green asked the API +# only for the NON-green jobs, so "this run is entirely green" and "this run +# was not read" were the SAME observation -- zero lines -- and zero lines was +# read as greenness. There it does not shorten the gate; it ADMITS a context +# that was never shown to be green, which is how a red check ends up required +# and every PR blocks on it. +# +# Fixture shape: derive (per_page=1) resolves run 11, which has jobs. The +# greenness probe (per_page=3) additionally resolves run 12, whose jobs +# fixture EXISTS and is EMPTY -- a successful read of nothing, which the +# missing-fixture trick cannot simulate. +reset_fix +R=acme/zerojobs +mkfix "repos/$R" '{"default_branch":"main"}' +mkfix "repos/$R/contents/.github/workflows" '[{"name":"governance.yml"}]' +mkfix "repos/$R/contents" '[{"name":"README.md"}]' +mkfix "repos/$R/actions/workflows/governance.yml/runs?branch=main&per_page=1" '{"workflow_runs":[{"id":11}]}' +mkfix "repos/$R/actions/workflows/governance.yml/runs?branch=main&per_page=3" '{"workflow_runs":[{"id":11},{"id":12}]}' +mkfix "repos/$R/actions/runs/11/jobs?per_page=100" '{"jobs":[{"name":"governance / Governance","conclusion":"success"}]}' +mkfix "repos/$R/actions/runs/12/jobs?per_page=100" '{"jobs":[]}' +mkfix "repos/$R/rulesets" '[{"id":9,"target":"branch","enforcement":"active"}]' +mkfix "repos/$R/rulesets/9" '{"name":"Base","target":"branch","enforcement":"active","conditions":{},"bypass_actors":[],"rules":[{"type":"deletion"}]}' + +OUT=$(run_applier "$R" --apply --require-green 3) +S=$(state_of "$OUT"); D=$(detail_of "$OUT") +[ "$S" = "REFUSED" ] && ok "zero-job green probe: state is REFUSED" || bad "zero-job green probe: state=$S (want REFUSED)" +case "$D" in *"returned zero jobs"*) ok "zero-job green probe: the unread run is NAMED, not counted as green" ;; *) bad "zero-job green probe: not reported — $D" ;; esac +[ -s "$FIX/PUTS.log" ] && bad "zero-job green probe: PUT a gate whose greenness was never read" || ok "zero-job green probe: no PUT even with --apply" + +# ---- MUTANT D: delete the zero-jobs guard from the GREENNESS probe only. ---- +# Keyed on jobs2 so it cannot touch the derivation loop's jobs1 guard. +MUTD="$WORK/mutant-d.sh" +sed 's|if \[ ! -s "\$WORK/jobs2" \]; then|if false; then|' "$APPLIER" > "$MUTD" +if ! cmp -s "$MUTD" "$APPLIER"; then + reset_fix + mkfix "repos/$R" '{"default_branch":"main"}' + mkfix "repos/$R/contents/.github/workflows" '[{"name":"governance.yml"}]' + mkfix "repos/$R/contents" '[{"name":"README.md"}]' + mkfix "repos/$R/actions/workflows/governance.yml/runs?branch=main&per_page=1" '{"workflow_runs":[{"id":11}]}' + mkfix "repos/$R/actions/workflows/governance.yml/runs?branch=main&per_page=3" '{"workflow_runs":[{"id":11},{"id":12}]}' + mkfix "repos/$R/actions/runs/11/jobs?per_page=100" '{"jobs":[{"name":"governance / Governance","conclusion":"success"}]}' + mkfix "repos/$R/actions/runs/12/jobs?per_page=100" '{"jobs":[]}' + mkfix "repos/$R/rulesets" '[{"id":9,"target":"branch","enforcement":"active"}]' + mkfix "repos/$R/rulesets/9" '{"name":"Base","target":"branch","enforcement":"active","conditions":{},"bypass_actors":[],"rules":[{"type":"deletion"}]}' + OUT=$(MUTANT="$MUTD" run_applier "$R" --apply --require-green 3) + if [ "$(state_of "$OUT")" = "REFUSED" ]; then + bad "MUTANT D SURVIVED: zero-jobs guard removed yet still REFUSED — the control is decorative" + else + ok "mutant D killed: without the guard it becomes $(state_of "$OUT") and PUTs $(wc -l < "$FIX/PUTS.log") time(s)" + fi + if [ -r "$FIX/LAST_PUT.json" ] && + [ "$(jq -r '[.rules[]|select(.type=="required_status_checks")|.parameters.required_status_checks[].context]|join(",")' "$FIX/LAST_PUT.json")" = "governance / Governance" ]; then + ok "mutant D REQUIRED a context whose greenness was never read — the silent admission being guarded" + else + bad "mutant D: expected the unverified context to be required anyway" + fi +else + bad "mutant D was not applied — the sed pattern no longer matches the applier" +fi + +# =============================================================== CASE 10 +# THE THIRD INSTANCE OF THE SAME CLASS, three lines above CASE 9's. The +# greenness probe's RUNS query has the identical shape: a query that succeeds +# with {"workflow_runs":[]} leaves rids empty, so the per-run loop never +# executes, nothing is ever appended to "bad", and every context the workflow +# contributed is admitted as green. Zero runs and zero BAD runs were the same +# observation. Since the probe now iterates gatewf3 -- the workflows that DID +# contribute contexts, and therefore DID have a run -- an empty runs list here +# can only be a failed read. +reset_fix +R=acme/zeroruns +mkfix "repos/$R" '{"default_branch":"main"}' +mkfix "repos/$R/contents/.github/workflows" '[{"name":"governance.yml"}]' +mkfix "repos/$R/contents" '[{"name":"README.md"}]' +mkfix "repos/$R/actions/workflows/governance.yml/runs?branch=main&per_page=1" '{"workflow_runs":[{"id":11}]}' +mkfix "repos/$R/actions/workflows/governance.yml/runs?branch=main&per_page=3" '{"workflow_runs":[]}' +mkfix "repos/$R/actions/runs/11/jobs?per_page=100" '{"jobs":[{"name":"governance / Governance","conclusion":"success"}]}' +mkfix "repos/$R/rulesets" '[{"id":9,"target":"branch","enforcement":"active"}]' +mkfix "repos/$R/rulesets/9" '{"name":"Base","target":"branch","enforcement":"active","conditions":{},"bypass_actors":[],"rules":[{"type":"deletion"}]}' + +OUT=$(run_applier "$R" --apply --require-green 3) +S=$(state_of "$OUT"); D=$(detail_of "$OUT") +[ "$S" = "REFUSED" ] && ok "zero-run green probe: state is REFUSED" \ + || bad "zero-run green probe: expected REFUSED, got '$S' ($D)" +case "$D" in + *"green-runs-query-returned-none"*) ok "zero-run green probe: the unread workflow is NAMED, not counted as green" ;; + *) bad "zero-run green probe: detail does not name the unread workflow: $D" ;; +esac +[ -s "$FIX/PUTS.log" ] && bad "zero-run green probe: PUT a gate whose greenness was never read" \ + || ok "zero-run green probe: no PUT even with --apply" + +# =============================================================== CASE 11 +# THE REGRESSION THE NAIVE FIX WOULD CAUSE, and the reason the probe iterates +# gatewf3 rather than gatewf2. gatewf2 still holds every NORUN workflow, for +# which an empty runs list is the LEGITIMATE state, not a failed read. A bare +# `[ -s rids ]` check over gatewf2 would turn every repo owning one run-less +# gate workflow into REFUSED under --require-green. Note that NO per_page=3 +# fixture exists for the NORUN workflow: the shim exits 1 on a missing fixture, +# so if the probe ever asks about it this case goes REFUSED and fails. +reset_fix +R=acme/norun-beside-good +mkfix "repos/$R" '{"default_branch":"main"}' +mkfix "repos/$R/contents/.github/workflows" '[{"name":"governance.yml"},{"name":"codeql.yml"}]' +mkfix "repos/$R/contents" '[{"name":"README.md"}]' +mkfix "repos/$R/actions/workflows/governance.yml/runs?branch=main&per_page=1" '{"workflow_runs":[{"id":11}]}' +mkfix "repos/$R/actions/workflows/governance.yml/runs?branch=main&per_page=3" '{"workflow_runs":[{"id":11},{"id":12}]}' +mkfix "repos/$R/actions/runs/11/jobs?per_page=100" '{"jobs":[{"name":"governance / Governance","conclusion":"success"}]}' +mkfix "repos/$R/actions/runs/12/jobs?per_page=100" '{"jobs":[{"name":"governance / Governance","conclusion":"success"}]}' +mkfix "repos/$R/actions/workflows/codeql.yml/runs?branch=main&per_page=1" '{"workflow_runs":[]}' +mkfix "repos/$R/rulesets" '[{"id":9,"target":"branch","enforcement":"active"}]' +mkfix "repos/$R/rulesets/9" '{"name":"Base","target":"branch","enforcement":"active","conditions":{},"bypass_actors":[],"rules":[{"type":"deletion"}]}' + +OUT=$(run_applier "$R" --require-green 3) +S=$(state_of "$OUT"); D=$(detail_of "$OUT") +[ "$S" != "REFUSED" ] && ok "NORUN beside a good workflow: not REFUSED (state=$S) — the run-less workflow is not a failed read" \ + || bad "NORUN beside a good workflow: REFUSED ($D) — the rids guard is scoped to gatewf2, not gatewf3" +case "$D" in + *codeql.yml*) ok "NORUN beside a good workflow: the run-less workflow is reported by name" ;; + *) bad "NORUN beside a good workflow: detail does not name codeql.yml: $D" ;; +esac + +# ---- MUTANT E: delete the empty-runs guard from the greenness probe only. ---- +# Keyed on rids so it cannot touch either jobs guard. +MUTE="$WORK/mutant-e.sh" +sed 's|if \[ ! -s "\$WORK/rids" \]; then|if false; then|' "$APPLIER" > "$MUTE" +chmod +x "$MUTE" +if ! cmp -s "$MUTE" "$APPLIER"; then + reset_fix + R=acme/zeroruns + mkfix "repos/$R" '{"default_branch":"main"}' + mkfix "repos/$R/contents/.github/workflows" '[{"name":"governance.yml"}]' + mkfix "repos/$R/contents" '[{"name":"README.md"}]' + mkfix "repos/$R/actions/workflows/governance.yml/runs?branch=main&per_page=1" '{"workflow_runs":[{"id":11}]}' + mkfix "repos/$R/actions/workflows/governance.yml/runs?branch=main&per_page=3" '{"workflow_runs":[]}' + mkfix "repos/$R/actions/runs/11/jobs?per_page=100" '{"jobs":[{"name":"governance / Governance","conclusion":"success"}]}' + mkfix "repos/$R/rulesets" '[{"id":9,"target":"branch","enforcement":"active"}]' + mkfix "repos/$R/rulesets/9" '{"name":"Base","target":"branch","enforcement":"active","conditions":{},"bypass_actors":[],"rules":[{"type":"deletion"}]}' + OUT=$(MUTANT="$MUTE" run_applier "$R" --apply --require-green 3) + if [ "$(state_of "$OUT")" = "REFUSED" ]; then + bad "MUTANT E SURVIVED: empty-runs guard removed yet still REFUSED — the control is decorative" + else + ok "mutant E killed: without the guard it becomes $(state_of "$OUT") and PUTs $(wc -l < "$FIX/PUTS.log") time(s)" + fi + if [ -r "$FIX/LAST_PUT.json" ] && + [ "$(jq -r '[.rules[]|select(.type=="required_status_checks")|.parameters.required_status_checks[].context]|join(",")' "$FIX/LAST_PUT.json")" = "governance / Governance" ]; then + ok "mutant E REQUIRED a context across ZERO examined runs — the silent admission being guarded" + else + bad "mutant E: expected the unverified context to be required anyway" + fi +else + bad "mutant E was not applied — the sed pattern no longer matches the applier" +fi + echo echo "passed=$pass failed=$fail" [ "$fail" -eq 0 ]