From e1ce4b56f23eaacbc2258d7b95ced701bce7122d Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Wed, 28 Oct 2026 16:57:00 +0800 Subject: [PATCH] feat(pr-risk): retain grades without publishing --- .github/workflows/pr-risk.yml | 28 ++++++++++++++++++++- docs/callers/pr-risk.md | 1 + scripts/pr-risk/README.md | 13 ++++++++-- scripts/pr-risk/grade-targets.sh | 23 +++++++++++++++-- scripts/pr-risk/tests/test_grade_targets.sh | 24 ++++++++++++++++++ scripts/pr-risk/tests/test_job_gates.sh | 7 +++++- 6 files changed, 90 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pr-risk.yml b/.github/workflows/pr-risk.yml index 6319c149..ceb173a8 100644 --- a/.github/workflows/pr-risk.yml +++ b/.github/workflows/pr-risk.yml @@ -363,6 +363,14 @@ on: type: string required: false default: .github/risk-runbooks.json + report_only: + description: >- + Retain grade records as an Actions artifact without changing labels, posting comments, + or publishing the optional Check Run. The record is advisory and does not establish + that separate reads used one immutable snapshot. + type: boolean + required: false + default: false sticky_comment: description: >- Publish ONE sticky PR comment, and keep it SMALL: the visible body is a single line — @@ -1004,6 +1012,7 @@ jobs: WAIT_MINUTES: ${{ inputs.wait_for_checks_minutes }} # The sticky comment rides the `pull-requests: write` this job already holds for the # label; the Check Run does not, so it is only RENDERED here (see the `surfaces` output). + REPORT_ONLY: ${{ inputs.report_only && '1' || '0' }} PUBLISH_COMMENT: ${{ inputs.sticky_comment && '1' || '0' }} PUBLISH_CHECK: ${{ inputs.check_run && '1' || '0' }} JOB_TIMEOUT_MINUTES: 30 # keep in sync with timeout-minutes on the job @@ -1012,12 +1021,24 @@ jobs: set -uo pipefail bash _pr_risk_tool/scripts/pr-risk/grade-targets.sh + - name: Retain risk records + if: ${{ always() && inputs.report_only }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: pr-risk-report-${{ github.run_id }}-${{ github.run_attempt }} + path: | + pr-risk-results.jsonl + record-*.json + if-no-files-found: error + retention-days: 30 + - name: Step summary if: always() env: TIER: ${{ steps.grade.outputs.tier }} WAITED: ${{ steps.grade.outputs.waited }} TARGETS: ${{ steps.grade.outputs.targets }} + REPORT_ONLY: ${{ inputs.report_only && '1' || '0' }} run: | set -uo pipefail RESULTS=pr-risk-results.jsonl @@ -1044,6 +1065,10 @@ jobs: "$RESULTS" 2>/dev/null || printf '0\t0\t0\t0')" IFS=$'\t' read -r N_GRADED N_UNGRADED N_FAILED N_SKIPPED <<<"$COUNTS" { + if [ "$REPORT_ONLY" = 1 ]; then + echo "> Report only: no risk label, comment or optional Check Run was published." + echo + fi if [ "$ROWS" -eq 0 ]; then # No target was even attempted — a usage error (an empty or malformed target list), # which is a different failure from an unreadable PR and must not borrow its @@ -1088,7 +1113,7 @@ jobs: else echo "## PR risk grade: ${head_tier:-unknown} (advisory)" echo - echo "The PR could not be read via the API; nothing was graded (labeled ungraded — this is NOT a low-risk verdict)." + echo "${note:-The PR could not be read via the API; this is NOT a low-risk verdict.}" fi else # A BATCH has no single tier and no room for N axis tables, so it reports one row @@ -1146,6 +1171,7 @@ jobs: if: >- needs.grade.result != 'skipped' && inputs.check_run && + !inputs.report_only && !cancelled() name: Publish the risk Check Run runs-on: ubuntu-latest diff --git a/docs/callers/pr-risk.md b/docs/callers/pr-risk.md index 0bd08263..23f870ff 100644 --- a/docs/callers/pr-risk.md +++ b/docs/callers/pr-risk.md @@ -127,6 +127,7 @@ fail the caller's next run at startup. | `wait_for_checks_minutes` | `10` | How long to wait for the rest of the check rollup to settle before labeling (clamped to 25 — what a 30-minute job can spend waiting). `0` labels immediately, expect R2 floors from still-pending checks. | | `repo_map_path` | `.github/risk.json` | Consumer risk-map override, read from the PR **base ref**. Repo-relative — a leading `/` or a `..` segment is refused, not resolved. | | `repo_runbooks_path` | `.github/risk-runbooks.json` | Consumer runbook-registry override, read from the PR **base ref**. | +| `report_only` | `false` | Retain per-target records and results as a 30-day Actions artifact. Suppresses label, sticky comment and optional risk Check Run publication even when other inputs request them. GitHub Actions still creates run/check records, and the workflow retains its declared permissions. The record is advisory: separate reads may race with a base or head update. | | `sticky_comment` | `false` | Publish ONE sticky PR comment — a single visible line (the tier, the axis that decided it and that axis's reason), with the formula, the per-axis reasons, the map versions and the caveats folded into a collapsed `
` and a "this grade is wrong" checkbox below it. Created once and updated in place, so N pushes leave one comment — with two bounds worth stating. The marker lookup scans only the newest 10 pages of 100 comments (`STICKY_MAX_PAGES`), so on a PR chatty enough to bury the sticky past ~1,000 comments it reads as absent: the next grade posts a DUPLICATE with `disputed=0`, which can also drop an already-applied `risk-grade-disputed` label while the reviewer's box is still ticked. And the upsert is list-then-create / read-then-PATCH with no atomicity — a `pr_numbers` batch and an event run cannot share a per-PR concurrency group, so two first grades can both see no marker and both post, and a tick landing between the read and the PATCH is overwritten. Ordinary single-run pushes behave exactly as described; the guarantee is weaker than it sounds at the edges. **Defaults to false on purpose:** a bot comment on every PR is a visible behaviour change, so it is a per-consumer decision, and a caller that does not opt in behaves byte-identically to before this input existed. Ticking the checkbox applies `risk-grade-disputed` on the next grade — a machine-maintained mirror, distinct from the human-owned `risk-dispute:*` overrides — and needs no permission beyond the `pull-requests: write` already granted. | | `check_run` | `false` | Publish a Check Run on the PR's head commit carrying the tier and the reason: the immutable, timestamped, commit-attached record a mutable label cannot be. Its conclusion is hardcoded `neutral`, so it can never fail a PR even if a repo later marks it required. **Leaving it false does not excuse the `checks: write` grant** — see "Grant the whole union" above; the grant is validated at startup, the input only decides whether the `publish-check` job runs. **A Check Run also requires `enabled: true`.** `publish-check` is gated on `enabled` as well as on this input, while `grade` deliberately admits a `workflow_dispatch` regardless of it — so on exactly the staged-rollout path the `enabled` row advertises, a manual dispatch against a repo still switched off grades, labels and (with `sticky_comment`) comments, but publishes no Check Run. Evaluating the Check Run before switching the repo on therefore means making `enabled` true for that run — `vars.RISK_CONFIG` = `{"enabled": true}` (no PR) or `enabled: true` in the caller's `with:` — because grading alone will not publish one. | | `check_name` | `PR risk (advisory)` | Name of the published Check Run. Only used when `check_run` is true. **Do not reuse the name of a required check.** Nothing validates this value, and the conclusion is hardcoded `neutral` — which branch protection treats as satisfied — so pointing it at a required check's name lets this advisory run stand in for the real merge gate. Keep it distinct and obviously advisory. | diff --git a/scripts/pr-risk/README.md b/scripts/pr-risk/README.md index ddbd17fd..f05e0996 100644 --- a/scripts/pr-risk/README.md +++ b/scripts/pr-risk/README.md @@ -11,12 +11,21 @@ event is graded into a tier and gets ONE label: | R3 | `risk:R3` | elevated — auth, billing, migrations, IaC, CI, deps, secrets | owner + e2e | | — | `risk:ungraded` | an input could not be read; deliberately NOT a tier | human review | -**The label is the entire product.** Nothing is gated, blocked, routed, commented -on, or merged. `risk-dispute:low` through `risk-dispute:xhigh` override the +**The label is the default product.** Nothing is gated, blocked, routed, commented +on, or merged. Report-only mode retains records without changing the label. `risk-dispute:low` through `risk-dispute:xhigh` override the visible risk label while leaving the computed tier in the grade record. Removing the dispute restores the computed label; if multiple overrides exist, the highest risk wins. +## Report-only mode + +Set `report_only: true` on the reusable workflow to retain grade records and per-target +results as a 30-day Actions artifact. It skips risk labels, sticky comments and the +optional risk Check Run, even if their options are enabled. The workflow itself still +creates GitHub Actions run/check records and retains its declared token permissions. +An ungraded result is unknown, never a low-risk verdict. Separate GitHub reads can +race with changes to the base or head; the artifact is not readiness evidence. + ## How a grade is computed `grade = worst(path_floor, provenance, reversibility)` — three deterministic diff --git a/scripts/pr-risk/grade-targets.sh b/scripts/pr-risk/grade-targets.sh index 851cf97d..72e48929 100755 --- a/scripts/pr-risk/grade-targets.sh +++ b/scripts/pr-risk/grade-targets.sh @@ -38,7 +38,8 @@ # LABEL_MAP passed through to apply-risk-label.sh # MAX_TARGETS refuse a list longer than this (default 50) # RESULTS per-target JSONL outcome file (default pr-risk-results.jsonl) -# DRY_RUN 1 = grade and report, write no label +# DRY_RUN 1 = simulate label/comment writes +# REPORT_ONLY 1 = retain records only; ignore publication options # GH_TOKEN token for gh # GITHUB_OUTPUT when set, tier / waited / targets / graded / ungraded / failed / # skipped are appended @@ -101,6 +102,7 @@ SURFACES_BUDGET_BYTES="${SURFACES_BUDGET_BYTES:-100000}" SURFACES_BYTES=0 SURFACES_DROPPED=0 DRY_RUN="${DRY_RUN:-0}" +REPORT_ONLY="${REPORT_ONLY:-0}" # The retry/backoff constants are env-overridable ONLY so the suite can exercise the # unreadable-PR and settle-repeat branches without sleeping through the production backoff. CI # passes none of them, so the values below are what runs in production. (READ_RETRY_TRIES and @@ -196,7 +198,11 @@ settle_grade() { # "$record" break ;; *) log "grader failed for ${REPO}#${num} (rc=$rc)"; G_TIER=""; return 1 ;; # setup/usage bug @@ -295,6 +301,18 @@ process_target() { # -> rc 0 = label in sync, rc 1 = failed } log "${REPO}#${num}: graded tier ${G_TIER} (waited ${G_WAITED}s for the rollup, base ${base})" + # Report-only runs must not call any publisher, even when conflicting options are set. + # A null label says no label was applied; unknown remains distinct from a low-risk grade. + if [ "$REPORT_ONLY" = 1 ]; then + case "$G_TIER" in + ""|unknown|null) + record_result "$num" ungraded unknown "" "$record" "$G_WAITED" "$base" "risk evidence unavailable; no label was written" ;; + *) + record_result "$num" graded "$G_TIER" "" "$record" "$G_WAITED" "$base" "report only; no label was written" ;; + esac + return 0 + fi + # The labeler prints the applied label on stdout and its own log on stderr; the rc has to be # read from the command itself, not from a pipeline, so its output goes to a file first. label_rc=0 @@ -389,6 +407,7 @@ main() { # it first costs nothing and closes that window. trap 'rm -f "$ERRF" "$LABELF" "$OUTF"' EXIT init_scratch + case "$REPORT_ONLY" in 0|1) ;; *) die "REPORT_ONLY must be 0 or 1" ;; esac [ -n "$REPO" ] || die "REPO is required" [[ "$REPO" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || die "bad REPO '$REPO' (want owner/name)" [ -f "$GRADER" ] || die "grader not found at $GRADER (set TOOL_DIR)" diff --git a/scripts/pr-risk/tests/test_grade_targets.sh b/scripts/pr-risk/tests/test_grade_targets.sh index 669f1db0..6a7c4805 100755 --- a/scripts/pr-risk/tests/test_grade_targets.sh +++ b/scripts/pr-risk/tests/test_grade_targets.sh @@ -478,6 +478,30 @@ run_targets PR_NUMBERS=42 BASE_REF=main MAP_PATH='.github/risk.json' eq "a normal override path is unaffected" 0 "$RC" eq "…and the target still grades" graded "$(res '.status')" +echo "— report-only mode keeps evidence without publishing —" +run_targets PR_NUMBERS=42 BASE_REF=main REPORT_ONLY=1 DRY_RUN=0 PUBLISH_CHECK=1 PUBLISH_COMMENT=1 +eq "report-only grades successfully" 0 "$RC" +eq "report-only records the grade" graded "$(res '.status')" +eq "report-only does not claim a label was applied" null "$(res '.label')" +eq "report-only emits no check payload" 0 "$(jq -s length "$WORK/pr-risk-surfaces.jsonl")" +no_text "report-only makes no label or comment API call" "issues/42/labels" "$(calls)" +no_text "report-only makes no comment API call" "issues/42/comments" "$(calls)" +if [ -f "$WORK/record-42.json" ]; then + ok "report-only retains the grade record" +else + bad "grade record missing" +fi +touch "$STUB_DIR/graphql_fail" +run_targets PR_NUMBERS=42 BASE_REF=main REPORT_ONLY=1 DRY_RUN=0 PUBLISH_CHECK=1 PUBLISH_COMMENT=1 +eq "report-only preserves unavailable evidence" ungraded "$(res '.status')" +eq "unavailable evidence is not a low-risk verdict" unknown "$(res '.tier')" +eq "unavailable evidence writes no label" null "$(res '.label')" +no_text "unavailable evidence makes no label write" "issues/42/labels" "$(calls)" +rm -f "$STUB_DIR/graphql_fail" +run_targets PR_NUMBERS=42 BASE_REF=main REPORT_ONLY=bad DRY_RUN=0 +eq "invalid report-only mode fails closed" 2 "$RC" +no_text "invalid mode makes no API call" "api " "$(calls)" + echo "— phase 26: a mid-way mktemp failure leaves no scratch file behind —" # init_scratch takes THREE mktemps and `gt_die`s on any of them. With the EXIT trap armed after # init_scratch returned, a failure on the second or third exited 2 with the file(s) already created diff --git a/scripts/pr-risk/tests/test_job_gates.sh b/scripts/pr-risk/tests/test_job_gates.sh index 363c9558..966d54c2 100755 --- a/scripts/pr-risk/tests/test_job_gates.sh +++ b/scripts/pr-risk/tests/test_job_gates.sh @@ -164,7 +164,7 @@ eq "the grade gate is exactly what it should be" \ "needs.gate.outputs.enabled == 'true' || github.event_name == 'workflow_dispatch'" \ "$GRADE_IF" eq "the publish-check gate is exactly what it should be" \ - "needs.grade.result != 'skipped' && inputs.check_run && !cancelled()" \ + "needs.grade.result != 'skipped' && inputs.check_run && !inputs.report_only && !cancelled()" \ "$PUBLISH_IF" # --- grade: `enabled` OR a manual dispatch ---------------------------------------------------- @@ -182,6 +182,7 @@ needs_has "grade waits on gate" "$(job_needs grade)" "gate" needs_has "publish-check waits on grade" "$(job_needs 'publish-check')" "grade" has "publish-check is gated on grade having run" "$PUBLISH_IF" "needs.grade.result != 'skipped'" has "publish-check still honours the opt-in input" "$PUBLISH_IF" "inputs.check_run" +has "report-only forbids the optional Check Run" "$PUBLISH_IF" "!inputs.report_only" no "publish-check does not INVERT the opt-in" "$PUBLISH_IF" "!inputs.check_run" has "publish-check still runs on a failed grade" "$PUBLISH_IF" '!cancelled()' has "publish-check joins its terms with AND" "$PUBLISH_IF" "&&" @@ -206,5 +207,9 @@ else ok "publish-check never pairs a status function with no needs.grade.result test" fi +has "grade passes report-only mode to the script" "$(job_body grade)" "REPORT_ONLY: \${{ inputs.report_only && '1' || '0' }}" +has "report-only retains the results" "$(job_body grade)" "pr-risk-results.jsonl" +has "report-only retains the grade records" "$(job_body grade)" "record-*.json" + printf '\n%s passed, %s failed\n' "$PASS" "$FAIL" [ "$FAIL" -eq 0 ]