From 1567edfb1e984539d8c3bde8da841ec0990165b3 Mon Sep 17 00:00:00 2001 From: ShihChi Huang Date: Wed, 2 Sep 2026 11:17:45 -0700 Subject: [PATCH] feat(pr-risk): allow human risk overrides --- .github/workflows/pr-risk.yml | 34 +++++++++--------- docs/callers/pr-risk.md | 36 ++++++++++--------- scripts/pr-risk/README.md | 19 +++++----- scripts/pr-risk/apply-risk-label.sh | 22 +++++++++--- .../pr-risk/tests/test_apply_risk_label.sh | 21 +++++++++++ 5 files changed, 85 insertions(+), 47 deletions(-) diff --git a/.github/workflows/pr-risk.yml b/.github/workflows/pr-risk.yml index d62bb4d7..12e3cd67 100644 --- a/.github/workflows/pr-risk.yml +++ b/.github/workflows/pr-risk.yml @@ -1,11 +1,10 @@ name: PR Risk Grade (reusable) # Reusable ADVISORY PR risk grader — the shadow-check rung of the PR risk-grading ladder. -# Grades every PR event into a tier R0 (safest) .. R3 (riskiest) and syncs ONE label -# (`risk:R0` .. `risk:R3`, or `risk:ungraded` when an input was unreadable). That label is -# the entire product: nothing is gated, nothing is blocked, nothing merges, no comment is -# posted. Humans look at the label and agree or disagree; disagreement is recorded by adding -# the `risk-dispute` label (which this workflow never touches) plus a comment saying why. +# Grades every PR event into a tier R0 (safest) .. R3 (riskiest) and syncs ONE visible label +# (`risk:R0` .. `risk:R3`, or `risk:ungraded` when an input was unreadable). A human can override +# that visible label with `risk-dispute:low` .. `risk-dispute:xhigh`; the computed tier remains in +# the grade record. Nothing is gated, blocked, routed, commented, or merged. # # grade = worst(path_floor, provenance, reversibility) — three deterministic axes; the worst # tier wins, so no axis can move a PR into a safer lane than another axis put it. No LLM, no @@ -67,8 +66,8 @@ name: PR Risk Grade (reusable) # # ON-DEMAND GRADING (`pr_number` / `pr_numbers`): supply a PR number and that PR is graded with # no `pull_request` event involved — which is how a repo enrolling mid-stream grades the open -# queue it already has, and how a PR is re-graded after a `.github/risk.json` change or a -# `risk-dispute`. Absent, the workflow reads the event exactly as before. Three things differ on +# queue it already has, and how a PR is re-graded after a `.github/risk.json` change. Absent, the +# workflow reads the event exactly as before. Three things differ on # the by-number path, all of them deliberate: # * FORK PRs ARE GRADED. The `head.repo.full_name == github.repository` clause callers put in # their `if:` is a TOKEN guard, not a policy one: a fork's `pull_request` run gets a @@ -117,7 +116,7 @@ name: PR Risk Grade (reusable) # labels on a PR under one `label_map`; remapping `label_map` orphans the old names, which is a # one-time repo-side cleanup. What the shape does cost is a narrower residual: the PUT is built from # a snapshot read, so a NON-owned label added by someone else in the read→PUT window is dropped -# (`risk-dispute` included) and one removed in it is resurrected. That window opens only on a run +# (`risk-dispute:*` included) and one removed in it is resurrected. That window opens only on a run # that actually changes the grade and is roughly one API round-trip — about three on the first # grade in a repo, where the label pre-create sits inside it. The drop is recorded on the PR # timeline as an `unlabeled` event, so re-add a dispute that happens to land in that instant. @@ -147,7 +146,7 @@ name: PR Risk Grade (reusable) # name: CI - PR Risk Grade # on: # pull_request: -# types: [opened, synchronize, reopened, ready_for_review] +# types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled] # workflow_dispatch: # inputs: # pr_number: @@ -213,7 +212,9 @@ name: PR Risk Grade (reusable) # # can hand a Dependabot-triggered run an Actions secret. # if: >- # github.event_name != 'pull_request' || -# github.event.pull_request.head.repo.full_name == github.repository +# (((github.event.action != 'labeled' && github.event.action != 'unlabeled') || +# startsWith(github.event.label.name, 'risk-dispute:')) && +# github.event.pull_request.head.repo.full_name == github.repository) # permissions: # contents: read # issues: write # create the risk:* labels repo-side on first use @@ -284,8 +285,8 @@ on: `pull_request` run: with no number supplied the target, the base ref and every emitted label are exactly what they were before this input existed. Supplying it is what makes grading possible without a `pull_request` event — the enrollment backfill of an - already-open queue, and the manual re-grade after a risk-map change or a - `risk-dispute`. Bot-authored and fork PRs ARE graded on this path (see the header). + already-open queue, and the manual re-grade after a risk-map change. + Bot-authored and fork PRs ARE graded on this path (see the header). Typed `string` rather than `number` because `workflow_dispatch` inputs arrive as strings, and because an empty string is what lets the fall-through to the event's own number stay a single expression. @@ -378,10 +379,9 @@ on: Ticking the checkbox applies `risk-grade-disputed` on the next grade, and a re-grade preserves the tick rather than resetting it. That label is DISTINCT from the - human-owned `risk-dispute` convention, which the grader still never touches: one is a - machine-maintained mirror of a checkbox, rewritten on every grade; the other is a - human's own label. Needs no permission beyond the `pull-requests: write` the grade job - already holds for the label. + human-owned `risk-dispute:*` override labels: one is a machine-maintained mirror of a + checkbox, while the others select the visible risk tier. Needs no permission beyond the + `pull-requests: write` the grade job already holds for the label. type: boolean required: false default: false @@ -1104,7 +1104,7 @@ jobs: jq -r '"| #\(.pr) | \(.tier // "—") | \(.label // "—") | `\(.base_ref // "—")` | \(.waited)s | \(.note // "") |"' "$RESULTS" fi echo - echo "This label routes nothing and gates nothing. Disagree with the grade? Add the \`risk-dispute\` label and say why in a comment — the grader never touches that label." + echo "This label routes nothing and gates nothing. Override it with \`risk-dispute:low\`…\`risk-dispute:xhigh\`; remove the override to restore the computed label." } >> "$GITHUB_STEP_SUMMARY" # THE ONE JOB THAT HOLDS `checks: write`, and it reads NOTHING from a pull request. Its whole diff --git a/docs/callers/pr-risk.md b/docs/callers/pr-risk.md index 4c3c5fdf..358c2655 100644 --- a/docs/callers/pr-risk.md +++ b/docs/callers/pr-risk.md @@ -5,10 +5,9 @@ Read [the shared caller contract](README.md) first. ## What it does Grades every PR into a tier `R0` (safest) .. `R3` (riskiest) and syncs **one** -label (`risk:R0`..`risk:R3`, or `risk:ungraded` when an input was unreadable). -The label is the entire product: nothing is gated, routed, commented, or -merged — a human looks at the label and agrees or disagrees (recorded with a -`risk-dispute` label this workflow never touches). +visible label (`risk:R0`..`risk:R3`, or `risk:ungraded` when an input was +unreadable). A human can override it with `risk-dispute:low` through +`risk-dispute:xhigh`; the computed tier remains in the grade record. Deterministic, no LLM: `grade = worst(path_floor, provenance, reversibility)` — a path-glob map, what process produced the diff (registered runbooks, forks @@ -37,7 +36,7 @@ on: # a fork run under plain `pull_request` cannot write the label. See the # fork gotcha below before you swap it. pull_request: - types: [opened, synchronize, reopened, ready_for_review] + types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled] concurrency: group: pr-risk-${{ github.event.pull_request.number }} @@ -48,6 +47,10 @@ permissions: jobs: pr-risk: + if: >- + github.event_name != 'pull_request' || + ((github.event.action != 'labeled' && github.event.action != 'unlabeled') || + startsWith(github.event.label.name, 'risk-dispute:')) permissions: contents: read issues: write # create the risk:* labels repo-side on first use @@ -133,17 +136,18 @@ your own steps (or jobs) that check out or run PR head code under it reopens the classic pwn-request hole. Same-repo-only callers (private org repos that take no fork PRs) should stay on plain `pull_request`. -That is why the caller block above carries **no `if:` at all**, and why the two -paragraphs below talk about a clause it does not show. Neither shape this page -recommends needs one: under `pull_request_target` the token is writable for fork -PRs too, and a same-repo-only repo has no fork PRs to guard against. The third -shape — a public repo staying on plain `pull_request` and accepting that its fork -PRs go **ungraded** rather than red — is the one that needs -`github.event.pull_request.head.repo.full_name == github.repository`, scoped -behind an event test so `workflow_dispatch` still works. Copy that variant from -the caller pattern in -[`pr-risk.yml`'s header](../../.github/workflows/pr-risk.yml) rather than writing -it yourself; an unscoped clause makes the dispatch button a silent no-op. +The caller's `if:` limits label events to `risk-dispute:*`; other labels do not +need a re-grade. A public repo staying on plain `pull_request` also needs the +fork guard from [`pr-risk.yml`'s header](../../.github/workflows/pr-risk.yml). +Keep it behind the event test so `workflow_dispatch` still works. + +## Risk overrides + +`risk-dispute:low`, `risk-dispute:medium`, `risk-dispute:high`, and +`risk-dispute:xhigh` override the visible risk label. For example, +`risk-dispute:medium` changes a computed `risk:high` label to `risk:medium`. +Removing the dispute restores the computed label. If multiple override labels +are present, the highest risk wins. **Bot PRs are graded — do not add a `github.actor != 'dependabot[bot]'` skip.** Dependabot's `pull_request` runs do start from a read-only `GITHUB_TOKEN`, but diff --git a/scripts/pr-risk/README.md b/scripts/pr-risk/README.md index 2a4cbd76..c6aaf1d5 100644 --- a/scripts/pr-risk/README.md +++ b/scripts/pr-risk/README.md @@ -12,9 +12,10 @@ event is graded into a tier and gets ONE label: | — | `risk:ungraded` | an input could not be read; deliberately NOT a tier | human review | **The label is the entire product.** Nothing is gated, blocked, routed, commented -on, or merged. Humans glance at the label and either agree or disagree. -Disagree by adding the `risk-dispute` label (never touched by the grader) plus a -comment saying why — disputes are the pilot's calibration data. +on, or merged. `risk-dispute:low` through `risk-dispute:xhigh` override the +visible risk label while leaving the computed tier in the grade record. Removing +the dispute restores the computed label; if multiple overrides exist, the highest +risk wins. ## How a grade is computed @@ -106,9 +107,9 @@ Two CI-specific mechanics worth knowing: Every grade above is triggered by a `pull_request` event. Two things need a grade with no event: a repo that **enrolls mid-stream** and wants the open queue it -already has labeled, and a **manual re-grade** after a `.github/risk.json` change -or on a PR carrying `risk-dispute`. Both are a `workflow_dispatch` on the -consumer's caller, forwarding a number: +already has labeled, and a **manual re-grade** after a `.github/risk.json` +change. Both are a `workflow_dispatch` on the consumer's caller, forwarding a +number: ```yaml on: @@ -191,9 +192,9 @@ Operational caveats for a backfill: re-dispatch on `pr_number` if a final grade looks wrong. The residual it costs instead is narrower, but it cuts both ways: the PUT is built from a snapshot read, so a **non-owned** label added in the read→PUT window is dropped - (`risk-dispute` included — re-add a dispute that lands in that instant) and one - **removed** in that window is resurrected. The window opens only on a run that - actually changes the grade, and is about one API round-trip — three on the + (`risk-dispute:*` included — re-add an override that lands in that instant) + and one **removed** in that window is resurrected. The window opens only on a + run that actually changes the grade, and is about one API round-trip — three on the first grade in a repo, where the label pre-create sits inside it. A drop is not invisible: GitHub records it on the PR timeline as an `unlabeled` event by the grader token. Dispatch when the queue is quiet, and use `pr_number` when you diff --git a/scripts/pr-risk/apply-risk-label.sh b/scripts/pr-risk/apply-risk-label.sh index b413a6d7..1978b5c0 100755 --- a/scripts/pr-risk/apply-risk-label.sh +++ b/scripts/pr-risk/apply-risk-label.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # apply-risk-label.sh — sync a PR's risk label to the computed tier. The one write the -# reusable pr-risk.yml workflow performs. +# reusable pr-risk.yml workflow performs. A `risk-dispute:low` .. `risk-dispute:xhigh` +# label overrides the visible risk label while preserving the computed tier in the grade record. # # OWNERSHIP CONTRACT: this script owns EXACTLY the label names in LABEL_MAP's values, and a PR it # has written to carries EXACTLY ONE of them. That holds under concurrency BY CONSTRUCTION, not by @@ -15,14 +16,14 @@ # labels is impossible: every PUT this script builds contains the target, and LABEL_MAP is rejected # up front if any tier maps to an empty name, so the array can never degrade to the empty-labels # request that strips a PR. Labels it does NOT own are carried through the PUT as the snapshot read -# saw them (see RESIDUAL), so a human who disagrees with a grade records that with their -# OWN label (the pilot convention is `risk-dispute`) and the grader will never fight it. Editing the -# grader-owned label by hand is futile by design: the next push re-syncs it. +# saw them (see RESIDUAL). A human overrides the visible grade with one of the four named +# `risk-dispute:*` labels; a plain `risk-dispute` remains an untyped marker. Editing the grader-owned +# label by hand is futile by design: the next push re-syncs it. # # RESIDUAL — the price of atomicity, worth knowing for the pilot: the PUT is built from a SNAPSHOT # read, so it is an unguarded read-modify-write (the labels endpoint offers no version precondition # that could make it otherwise), and it clobbers BOTH directions inside that window. A NON-owned -# label ADDED there is silently dropped — `risk-dispute` INCLUDED — and a non-owned label REMOVED +# label ADDED there is silently dropped — `risk-dispute:*` INCLUDED — and a non-owned label REMOVED # there is RESURRECTED, so a `do-not-merge` or review label a human or a sibling labeler cleared in # that instant comes back. The window opens ONLY on a run that actually changes the grade (an # in-sync PR writes nothing at all) and is normally ~one API round-trip; on the FIRST grade in a @@ -177,6 +178,17 @@ has() { esac } +if has risk-dispute:xhigh; then + TIER=R3 +elif has risk-dispute:high; then + TIER=R2 +elif has risk-dispute:medium; then + TIER=R1 +elif has risk-dispute:low; then + TIER=R0 +fi +TARGET="$(label_for "$TIER")" + # ASCII case folding for shell-side comparisons. Literal `A-Z`/`a-z` rather than the suggested # [:upper:]/[:lower:] classes ON PURPOSE: this must fold EXACTLY what jq's ascii_downcase folds # above, and a locale-dependent class would make the two disagree on non-ASCII names. diff --git a/scripts/pr-risk/tests/test_apply_risk_label.sh b/scripts/pr-risk/tests/test_apply_risk_label.sh index 78ab30fe..d03b6bb7 100755 --- a/scripts/pr-risk/tests/test_apply_risk_label.sh +++ b/scripts/pr-risk/tests/test_apply_risk_label.sh @@ -143,6 +143,27 @@ eq "the PUT carries exactly the unowned labels plus the new target" \ "api -X PUT repos/test/repo/issues/7/labels -f labels[]=risk-dispute -f labels[]=bug -f labels[]=risk:R3" \ "$put3" +echo "— a named dispute overrides the visible risk label until removed —" +NAMED_MAP='R0=risk:low,R1=risk:medium,R2=risk:high,R3=risk:xhigh,unknown=risk:unknown' +: > "$GH_LOG"; printf 'risk:high\nrisk-dispute:medium\nbug\n' > "$CURRENT_LABELS" +outdispute="$(PATH="$SANDBOX/bin:$PATH" REPO=test/repo PR_NUMBER=7 TIER=R2 \ + LABEL_MAP="$NAMED_MAP" bash "$SCRIPT" 2>/dev/null)" +eq "risk-dispute:medium overrides computed R2" "risk:medium" "$outdispute" +putdispute="$(grep -- '-X PUT repos/test/repo/issues/7/labels ' "$GH_LOG")" +eq "the PUT preserves the dispute and replaces the visible risk label" \ + "api -X PUT repos/test/repo/issues/7/labels -f labels[]=risk-dispute:medium -f labels[]=bug -f labels[]=risk:medium" \ + "$putdispute" + +: > "$GH_LOG"; printf 'risk:medium\nbug\n' > "$CURRENT_LABELS" +outrestored="$(PATH="$SANDBOX/bin:$PATH" REPO=test/repo PR_NUMBER=7 TIER=R2 \ + LABEL_MAP="$NAMED_MAP" bash "$SCRIPT" 2>/dev/null)" +eq "removing the dispute restores computed R2" "risk:high" "$outrestored" + +: > "$GH_LOG"; printf 'risk:low\nrisk-dispute:low\nrisk-dispute:xhigh\n' > "$CURRENT_LABELS" +outmultiple="$(PATH="$SANDBOX/bin:$PATH" REPO=test/repo PR_NUMBER=7 TIER=R0 \ + LABEL_MAP="$NAMED_MAP" bash "$SCRIPT" 2>/dev/null)" +eq "the highest of multiple disputes wins" "risk:xhigh" "$outmultiple" + echo "— first use: the label is pre-created before the sync —" # A PUT creates the ASSOCIATION but never the label's color or description, so enrollment still # needs the explicit repo-side create on first use — and it must land BEFORE the sync.