From 9b45e4c6d752d3de7e1724984885d45f55cf18c6 Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Wed, 28 Oct 2026 17:25:28 +0800 Subject: [PATCH 1/2] feat: retain integrated review shadow reports --- .github/workflows/review-readiness-shadow.yml | 56 ++++++++++++ .github/workflows/test-review-readiness.yml | 51 +++++++++++ README.md | 1 + docs/callers/README.md | 1 + docs/callers/review-readiness-shadow.md | 27 ++++++ scripts/review-readiness/README.md | 15 ++++ scripts/review-readiness/shadow_report.py | 86 +++++++++++++++++++ .../tests/test_shadow_report.py | 86 +++++++++++++++++++ 8 files changed, 323 insertions(+) create mode 100644 .github/workflows/review-readiness-shadow.yml create mode 100644 docs/callers/review-readiness-shadow.md create mode 100644 scripts/review-readiness/shadow_report.py create mode 100644 scripts/review-readiness/tests/test_shadow_report.py diff --git a/.github/workflows/review-readiness-shadow.yml b/.github/workflows/review-readiness-shadow.yml new file mode 100644 index 00000000..58950e45 --- /dev/null +++ b/.github/workflows/review-readiness-shadow.yml @@ -0,0 +1,56 @@ +name: Review readiness shadow (reusable) + +on: + workflow_call: + inputs: + snapshot_and_policy_json: + description: Snapshot and advisory policy JSON, supplied by the caller. + required: true + type: string + +permissions: + contents: read + pull-requests: read + checks: read + actions: read + statuses: read + +jobs: + report: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + # The runner exposes workflow_sha on the job context. Serialize that + # context because actionlint's static schema does not yet include it; + # the SHA guard fails closed on older runners or an absent value. + - name: Validate provider revision + env: + PROVIDER_SHA: ${{ fromJSON(toJSON(job)).workflow_sha }} + run: | + if [[ ! "$PROVIDER_SHA" =~ ^[0-9a-f]{40}$ ]]; then + echo '::error::Cannot resolve the reusable workflow revision.' + exit 1 + fi + - name: Load provider tools at the workflow revision + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: Comfy-Org/github-workflows + ref: ${{ fromJSON(toJSON(job)).workflow_sha }} + path: _review_tool + persist-credentials: false + - name: Build advisory report + env: + GH_TOKEN: ${{ github.token }} + GH_HOST: github.com + SHADOW_EXPECTED_REPO: ${{ github.repository }} + SHADOW_INPUT: ${{ inputs.snapshot_and_policy_json }} + run: | + set -euo pipefail + printf '%s' "$SHADOW_INPUT" | python3 _review_tool/scripts/review-readiness/shadow_report.py > readiness-report.json + - name: Retain report + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: review-readiness-shadow-${{ github.run_id }}-${{ github.run_attempt }} + path: readiness-report.json + if-no-files-found: error + retention-days: 30 diff --git a/.github/workflows/test-review-readiness.yml b/.github/workflows/test-review-readiness.yml index 5cf7d28a..fac0b1ee 100644 --- a/.github/workflows/test-review-readiness.yml +++ b/.github/workflows/test-review-readiness.yml @@ -5,14 +5,20 @@ on: paths: - 'scripts/review-readiness/**' - '.github/workflows/test-review-readiness.yml' + - '.github/workflows/review-readiness-shadow.yml' push: branches: [main] paths: - 'scripts/review-readiness/**' - '.github/workflows/test-review-readiness.yml' + - '.github/workflows/review-readiness-shadow.yml' permissions: contents: read + pull-requests: read + checks: read + actions: read + statuses: read jobs: test: @@ -25,3 +31,48 @@ jobs: with: python-version: '3.12' - run: python3 -m unittest discover -s scripts/review-readiness/tests -v + + prepare-shadow: + if: ${{ github.event_name == 'pull_request' }} + runs-on: ubuntu-latest + outputs: + input: ${{ steps.input.outputs.json }} + steps: + - id: input + env: + TARGET_REPO: ${{ github.repository }} + TARGET_PR: ${{ github.event.pull_request.number }} + AUTHOR: ${{ github.event.pull_request.user.login }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + set -euo pipefail + payload=$(jq -cn --arg repo "$TARGET_REPO" --argjson pr "$TARGET_PR" \ + --arg author "$AUTHOR" --arg base "$BASE_SHA" --arg head "$HEAD_SHA" \ + '{snapshot:{repo:$repo,pr:$pr,author:$author,base_sha:$base,head_sha:$head,tested_sha:$head,policy_version:"synthetic-ci"},policy:{version:"synthetic-ci",required_checks:[{name:"unit",revision:"tested"}],required_human_approvals:1}}') + printf 'json=%s\n' "$payload" >> "$GITHUB_OUTPUT" + + shadow: + if: ${{ github.event_name == 'pull_request' }} + needs: prepare-shadow + permissions: + contents: read + pull-requests: read + checks: read + actions: read + statuses: read + uses: ./.github/workflows/review-readiness-shadow.yml + with: + snapshot_and_policy_json: ${{ needs.prepare-shadow.outputs.input }} + + verify-shadow: + if: ${{ github.event_name == 'pull_request' }} + needs: shadow + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: review-readiness-shadow-${{ github.run_id }}-${{ github.run_attempt }} + - name: Verify report + run: | + jq -e '(.state == "WAITING" or .state == "BLOCKED") and .collection.snapshot == "matched" and .risk_collection == "observed"' readiness-report.json diff --git a/README.md b/README.md index 5fca45bb..81a3cb77 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ complete, copy-pasteable caller. | [`assign-reviewers.yml`](.github/workflows/assign-reviewers.yml) | Assigns PR owners using configured paths and recent human approval history. Chooses complementary file coverage before workload, with an explained one-owner default fallback and no random substitutions. Optionally generates a shared history artifact on schedule/manual dispatch and reuses fresh snapshots across PRs. Reads configuration from the base SHA, respects manual assignments, and preserves opt-in `REVIEWER_AUTHOR_ALLOWLIST` scoping. Requires `vars.APP_ID` + `CLOUD_CODE_BOT_PRIVATE_KEY`; callers skip forks and Dependabot triggers. | [assign-reviewers.md](docs/callers/assign-reviewers.md) | | [`assign-prs-to-author.yml`](.github/workflows/assign-prs-to-author.yml) | Housekeeping — assigns every open PR with no assignees to its author (bot-authored PRs skipped by default). Run on a schedule from a thin caller; useful when a team tracks PR ownership via assignees. The calling job needs `pull-requests: write` and `issues: write`. | [assign-prs-to-author.md](docs/callers/assign-prs-to-author.md) | | [`pr-size.yml`](.github/workflows/pr-size.yml) | PR-size cap — fails (or, in `mode: warn`, only reports) when a PR's net diff exceeds `max_lines` non-generated changed lines, keeping diffs reviewable. Excludes dependency lockfiles, `linguist-generated` files (read from the base ref, so a PR can't exempt itself), Go generated-code markers, and per-repo `extra_lockfiles` / `extra_generated_globs`. Opt in to `exclude_tests` to cap production code rather than test coverage (excluded test lines are always reported, never silently dropped). A `bypass_label` (default `oversized-ok`) waves through a legitimately large change; a sticky bot comment explains overages when `bot_app_id` + `BOT_APP_PRIVATE_KEY` are supplied (degrades to status + step summary without them). Counting logic + tests live in [`scripts/check-pr-size/`](scripts/check-pr-size). | [pr-size.md](docs/callers/pr-size.md) | +| [`review-readiness-shadow.yml`](.github/workflows/review-readiness-shadow.yml) | Read-only advisory risk and review evidence report. Retains one artifact; it cannot report READY or gate a merge. | [review-readiness-shadow.md](docs/callers/review-readiness-shadow.md) | | [`pr-risk.yml`](.github/workflows/pr-risk.yml) | **Advisory PR risk grading (shadow check)** — **automatic grading off by default** (`enabled: false`; a manual `workflow_dispatch` grades regardless, so a repo can trial it before switching on); switch it on with `enabled: true` or by setting the caller repo's `RISK_CONFIG` variable to `{"enabled": true}`, which outranks the input in both directions so `{"enabled": false}` is a no-PR kill switch. Grades every PR into a tier `R0` (safest) .. `R3` (riskiest) and syncs one label (`risk:R0`..`risk:R3`, or `risk:ungraded` when an input was unreadable). The label is the entire product: nothing is gated, routed, commented, or merged. Deterministic (`gh` + `jq`, no LLM): `grade = worst(path_floor, provenance, reversibility)` — path-glob map, what-process-produced-the-diff (registered runbooks with identity + diff-shape assertions; forks are R3 with no exceptions), and revertability (persistent-state mutation, deletions under sensitive classes, did green checks cover the lines). Grader + generic defaults live in [`scripts/pr-risk/`](scripts/pr-risk); a consumer sharpens them with `.github/risk.json` / `.github/risk-runbooks.json`, read from the PR's **base ref** so a PR can't edit the rules that judge it. The job excludes its own run from the check rollup and waits (`wait_for_checks_minutes`) for the rest to settle before labeling. Labels ride the plain `GITHUB_TOKEN` (cannot fire `labeled` triggers — no cascade risk); disagreement is recorded with a human-owned `risk-dispute` label. **Two further publish surfaces are available and are OFF by default**, so an enrolled caller behaves byte-identically until it opts in: `sticky_comment: true` posts ONE comment (created once, updated in place — N pushes leave one comment) carrying the per-file path-axis breakdown, the risk CONCENTRATION sentence ("94% of this diff is R0/R1; the 6% that puts the path floor at R3 is these two files, 40 lines") and a "this grade is wrong" checkbox whose state round-trips into a `risk-grade-disputed` label (distinct from the human-owned `risk-dispute`, which the grader still never touches); `check_run: true` publishes the tier and reason as a Check Run on the head commit — the immutable, timestamped, commit-attached record a mutable label cannot be — from a SEPARATE job, the only one holding `checks: write` — a grant every caller makes whether or not it switches the surface on, because GitHub validates every nested job's declared `permissions:` at startup and a job `if:` is a runtime condition (the input decides whether a check is PUBLISHED, not whether the grant is CHECKED). Both surfaces are advisory in the same sense as the label: the Check Run's conclusion is hardcoded `neutral`, and every publish failure is an annotation, never a red check. Label text is remappable via `label_map`. `workflows_ref` is **required**, and both its **shape and its ancestry are enforced** — every job that checks it out fails the run *before* the tool checkout unless the value is a full 40-hex lowercase commit SHA **and** that commit is an ancestor of `main` of this repo (fetched from a literal upstream URL; no `github` context can name this repo from inside a reusable workflow, so a variable there would be an alias a fork could point at itself). Shape rejects everything mutable — a branch, a tag, a `refs/pull/N/head`. Ancestry rejects everything unmerged: a fork of this public repo shares its object store, and GitHub serves a fork PR's head objects from this repo's own URL unauthenticated, so a fork-authored SHA is just as well-shaped and would otherwise be checked out into a job holding the caller's write token. There is deliberately **no opt-out input** — an opt-out would be set by the very pin-bump PR the check distrusts. Two consequences by design: **pinning a not-yet-merged SHA is rejected** (merge first, then bump), and the axis **fails closed** if this repo ever goes private or `main` is force-rewritten past a consumer's pin (re-pin to a commit on `main` to recover). **What is still not machine-checked is that the pin is the *current* one**: a stale pin left behind when `uses:` moved is a merged ancestor too. The test that would close that is "equal to the commit `uses:` resolved to". That value IS readable — not as `github.workflow_sha` (the *caller's* top-level file) and not as the `job_workflow_sha` OIDC claim (not a `github` context property; reading it would need `id-token: write` from every caller), but as `job.workflow_sha`, the `job`-context accessor added in runner v2.334.0 that `groom.yml` reads (BE-8077). It is deliberately not wired in here: asserting equality would fail red on every caller whose `uses:` and `workflows_ref` have already drifted apart, which is a caller-contract change tracked separately. The other residual is irreducible: a fork commit that edits `pr-risk.yml` itself is rejected by ancestry, but a caller pinned at one runs the fork's own copy of the guard, and no in-file check can bound a revision of that file an attacker chose. **Reviewing the caller is what bounds both, and it is the only thing that can: require `uses:` at a full commit SHA of this repo and `with: workflows_ref:` set to that same SHA written out literally, character-for-character — never an expression, never a tag.** The guard also runs *before* enablement is resolved (the resolver is itself loaded from `workflows_ref`), so a floating pin fails red even with the `RISK_CONFIG` kill switch set — the switch stops the grading, not a broken enrollment. Call the workflow directly: a nested `workflow_call` chain through an org wrapper is unsupported. Enroll it as its own workflow rather than a job inside an existing CI workflow (the rollup exclusion is per-run). The calling job needs `contents: read` + `issues: write` + `pull-requests: write` + `checks: write` + `actions: read` + `statuses: read`; GitHub rejects a shorter grant at startup (a reusable workflow can only narrow the caller's token, never elevate it), so a caller enrolled from an older copy of this row fails before any step runs. The two label writes are the ONE label: repo-side label creation on first use maps to `issues`, and labeling a PR maps to `pull-requests` (the labels endpoint is dual-mapped by what the "issue" is, so `issues: write` alone 403s on a PR). `checks: write` is the `publish-check` job's declaration, required unconditionally per above — grading itself uses only `checks: read`. `actions: read` is for the rollup's `CheckRun -> checkSuite -> workflowRun` self-exclusion hop. No secrets. | [pr-risk.md](docs/callers/pr-risk.md) | | [`pr-derisk.yml`](.github/workflows/pr-derisk.yml) | **On-demand de-risk split plan (`/derisk`, beta)** — **off by default** (`enabled: false`; `vars.DERISK_CONFIG` on the caller repo outranks it in both directions, so `{"enabled": false}` is a no-PR kill switch). Someone with write access comments `/derisk` on a pull request; the workflow re-grades that PR with the pr-risk grader, makes **one** model call for a semantic partition of the diff into a chain of 2–5 **sequential** PRs, and posts **one sticky advisory comment**: a verdict line and a chain table above the fold, the plan, the chain-landing rules and the floor math inside collapsed `
`. **Every floor shown is computed by `grade-pr-risk.sh --stdin`** over a synthetic record built from each step's files — the model proposes *which files go together* and nothing else, and any tier it writes is discarded before rendering. The partition must cover the changed-file set **exactly**; a missing or duplicated path is rejected, re-prompted **once**, then falls back. **Nothing is gated, routed, merged, labelled or filed** — filing tickets from a plan is a later rung behind its own command. Honest by construction: when no step lands below the PR's **path floor** the verdict reads "N smaller single-concern R3s, same lane", never a fake lane win, because the verdict is arithmetic over grader output rather than the model's prose — and the comparison is against the path floor rather than the headline tier precisely because a split only moves the path axis, so a fork PR (provenance R3, path floor R0) is told the truth instead of a reduction no partition can deliver. An over-budget diff, an unvalidatable partition and an API failure each post an explaining comment — a `/derisk` that quietly does nothing is never an outcome. Safe as a comment command because an `issue_comment` workflow runs from the caller's **default branch** (a PR cannot edit the workflow serving it), the commenter is gated on `author_association` (`allowed_associations`, default `OWNER,MEMBER,COLLABORATOR` — narrow it, never widen), and **no PR code is ever checked out**: the diff is read over the API and handed to a model as text. `workflows_ref` is **required** and its shape + ancestry are enforced by the guard byte-identical to `pr-risk.yml`'s (see that row for the full rationale and its two residuals); a test fails the build if the copies drift. The calling job needs `contents: read` + `pull-requests: write` + `checks: read` + `actions: read` + `statuses: read`, and the `anthropic_api_key` secret. | [pr-derisk.md](docs/callers/pr-derisk.md) | | [`pr-area-label.yml`](.github/workflows/pr-area-label.yml) | **Agentic PR area labeling** — classifies each PR into exactly one `area:*` label, and (on push to the consumer's default branch) syncs the repo's `area:*` labels to the taxonomy. The taxonomy is the consumer's own `.github/area-labels.yml` (`repo_context` + `labels[]` with `name`/`color`/`description`/optional `guidance`); the shared workflow carries nothing repo-specific. An optional `sub_labels[]` block adds **deterministic path sub-labels** — `area:*` names with `paths:` globs, matched against the PR's changed files with no model involved, applied *alongside* the classified area and exempt from its cleanup (retired automatically once a PR stops matching). That is how a subsystem living inside another area's service stays filterable without splitting the parent area; names must be disjoint from `labels[]`. An LLM makes the domain-vs-path judgement a static `paths:` map can't, but with **no tools and no token**: PR title/body/paths/labels go to the Anthropic Messages API as data inside `` tags (diff excluded), the reply is enum-constrained by a JSON schema to the taxonomy's own names, and a deterministic step applies it with targeted `area:*` add/remove ops (never a full-set PUT, so concurrent non-area edits survive). The taxonomy is read from the PR's **base ref** — a PR can't rewrite the rules that classify it — and validated (unique `area:[a-z0-9-]+` names, every label resolves to a non-blank routing guide) before it drives a write; everything fails soft rather than failing the check. Classifier logic lives in [`scripts/area-label/`](scripts/area-label), loaded from the pinned `workflows_ref` (validated to a full 40-hex SHA before checkout). Fork/Dependabot PRs are skipped by construction (no writable token, no secret). Labels ride the plain `GITHUB_TOKEN` (no `labeled`-trigger cascade). The calling job needs `contents: read` + `issues: write` + `pull-requests: write`; the `ANTHROPIC_API_KEY` secret is provided org-wide (unset ⇒ `label-pr` fails soft, sync still works). | [pr-area-label.md](docs/callers/pr-area-label.md) | diff --git a/docs/callers/README.md b/docs/callers/README.md index 315d5cbc..2d4cd8c3 100644 --- a/docs/callers/README.md +++ b/docs/callers/README.md @@ -10,6 +10,7 @@ per-workflow pages give you a complete, copy-pasteable caller: | `cursor-review.yml` | [cursor-review.md](cursor-review.md) | `CURSOR_API_KEY` | | `cursor-review-auto-label.yml` | [cursor-review-auto-label.md](cursor-review-auto-label.md) | `vars.APP_ID` + App key | | `pr-size.yml` | [pr-size.md](pr-size.md) | nothing (bot comment optional) | +| `review-readiness-shadow.yml` | [review-readiness-shadow.md](review-readiness-shadow.md) | nothing | | `pr-risk.yml` | [pr-risk.md](pr-risk.md) | nothing | | `pr-derisk.yml` | [pr-derisk.md](pr-derisk.md) | `ANTHROPIC_API_KEY` | | `pr-area-label.yml` | [pr-area-label.md](pr-area-label.md) | `.github/area-labels.yml` (+ `ANTHROPIC_API_KEY`, org-wide) | diff --git a/docs/callers/review-readiness-shadow.md b/docs/callers/review-readiness-shadow.md new file mode 100644 index 00000000..29860f8f --- /dev/null +++ b/docs/callers/review-readiness-shadow.md @@ -0,0 +1,27 @@ +# Review readiness shadow report + +This reusable workflow evaluates one caller-supplied PR snapshot and policy. It +reads risk inputs, current reviews and review threads, then retains one JSON +artifact for 30 days. It does not write labels, comments, reviews, check runs, +or repository rules. The Actions run and job checks still exist. The token has +read access to contents, pull requests, checks, actions and commit statuses. + +The input `snapshot_and_policy_json` is a JSON object containing `snapshot` and +`policy` as documented in [the evaluator](../../scripts/review-readiness/README.md). +The target repo must equal the caller repo. The caller is responsible for obtaining +and verifying the snapshot and for choosing an authorized policy; input JSON is +not itself proof. Use only a reviewed, merged provider commit pinned by full SHA +in the caller's `uses:`. The provider tools load at the workflow's own revision, +never from the PR. This workflow runs only when explicitly called. + +The report records the supplied `tested_sha` but cannot prove that tests actually +ran on it. It does not verify reviewer permissions or cover every finding source. +Missing checks and incomplete findings prevent `READY`; known current failures +can still produce `BLOCKED`. Data reads are not atomic, so a review change or an +ABA revision change can race collection. It is not an authorization to merge or +a substitute for GitHub branch protection. No pilot or enforcement follows from +adding the workflow; obtain owner approval and verify the integrated run first. + +To recover, remove the caller or revert its pin. Existing artifacts retain their +normal expiry. An unavailable risk grade is reported as unknown, and an invalid +input or failed workflow produces no report artifact rather than a ready result. diff --git a/scripts/review-readiness/README.md b/scripts/review-readiness/README.md index 874f3e00..995a4388 100644 --- a/scripts/review-readiness/README.md +++ b/scripts/review-readiness/README.md @@ -89,3 +89,18 @@ escape the recheck. It is advisory evidence, not a merge gate. ```sh python3 scripts/review-readiness/collect_reviews.py < snapshot-and-policy.json ``` + +## Integrated shadow report + +`shadow_report.py` runs the real risk grader with `REPORT_ONLY=1` and no +publication options, then collects GitHub reviews for the same supplied snapshot. +It runs the grader in a temporary directory and emits one JSON report to stdout. +A changed or unavailable final snapshot discards the risk grade too. The report +remains advisory: missing checks and incomplete findings prevent `READY`. + +The reusable workflow `review-readiness-shadow.yml` retains that report for +30 days. Its caller provides the snapshot and policy; the report records the +provided tested SHA but does not verify it. The target repository must match +the caller repository. The workflow checks out its tools at its own immutable +workflow revision and does not execute PR code. Only use a reviewed, merged +provider revision in a caller. See the caller guide for trust and rollout limits. diff --git a/scripts/review-readiness/shadow_report.py b/scripts/review-readiness/shadow_report.py new file mode 100644 index 00000000..d7b16d1e --- /dev/null +++ b/scripts/review-readiness/shadow_report.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Create an advisory report using the report-only grader and review collector.""" + +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile + +import collect_reviews +import from_risk +import readiness + +RISK_DIR = Path(__file__).resolve().parents[1] / 'pr-risk' + + +def grade(snapshot): + with tempfile.TemporaryDirectory(prefix='readiness-risk-') as directory: + env = os.environ.copy() + env.pop('GITHUB_OUTPUT', None) + env.update({ + 'REPO': snapshot['repo'], 'PR_NUMBERS': str(snapshot['pr']), + 'BASE_REF': '', 'TOOL_DIR': str(RISK_DIR), 'RESULTS': 'pr-risk-results.jsonl', + 'REPORT_ONLY': '1', 'DRY_RUN': '1', 'PUBLISH_CHECK': '0', 'PUBLISH_COMMENT': '0', + 'WAIT_MINUTES': '0', 'JOB_TIMEOUT_MINUTES': '10', 'MAX_TARGETS': '1', + 'MAP_PATH': '.github/risk.json', 'RB_PATH': '.github/risk-runbooks.json', + 'FLEET_LOGINS': '', 'BOT_LOGINS': '', 'LABEL_MAP': '', + 'SELF_CONTEXT': os.environ.get('GITHUB_WORKFLOW', ''), + 'SELF_RUN_ID': os.environ.get('GITHUB_RUN_ID', ''), + }) + try: + proc = subprocess.run(['bash', str(RISK_DIR / 'grade-targets.sh')], cwd=directory, + env=env, capture_output=True, text=True, timeout=600) + if proc.returncode: + return None + lines = (Path(directory) / 'pr-risk-results.jsonl').read_text().splitlines() + if len(lines) != 1: + return None + summary = json.loads(lines[0]) + if not isinstance(summary, dict): + return None + # The target number is validated by readiness before it reaches this path. + record_path = Path(directory) / f"record-{snapshot['pr']}.json" + record = json.loads(record_path.read_text()) if record_path.exists() else None + return {'summary': summary, 'record': record} + except (OSError, ValueError, subprocess.SubprocessError): + return None + + +def build(data, client, run_risk=grade, expected_repo=None): + if not isinstance(data, dict): + raise ValueError('input must be an object') + # Validate the snapshot and policy before sending anything to GitHub. + readiness.evaluate({'snapshot': data.get('snapshot'), 'policy': data.get('policy'), + 'checks_complete': False, 'checks': []}) + snapshot = data['snapshot'] + if expected_repo is not None and snapshot['repo'].casefold() != expected_repo.casefold(): + raise ValueError('target repository does not match caller') + report = run_risk(snapshot) + risk = from_risk.evaluate_report({**data, 'report': report})['risk'] + result = collect_reviews.collect(data, client) + if result['collection']['snapshot'] != 'matched': + risk = {'status': 'unknown', 'tier': None} + result['risk'] = risk + result['risk_collection'] = 'observed' if report is not None else 'unavailable' + return result + + +def main(): + try: + data = json.load(sys.stdin) + readiness.evaluate({'snapshot': data.get('snapshot'), 'policy': data.get('policy'), + 'checks_complete': False, 'checks': []}) + snapshot = data['snapshot'] + result = build(data, collect_reviews.GitHub(snapshot['repo'], snapshot['pr']), + expected_repo=os.environ.get('SHADOW_EXPECTED_REPO')) + except (ValueError, TypeError, AttributeError, json.JSONDecodeError) as exc: + print(f'invalid shadow input: {exc}', file=sys.stderr) + return 2 + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/scripts/review-readiness/tests/test_shadow_report.py b/scripts/review-readiness/tests/test_shadow_report.py new file mode 100644 index 00000000..855acc2a --- /dev/null +++ b/scripts/review-readiness/tests/test_shadow_report.py @@ -0,0 +1,86 @@ +import importlib.util +import json +from pathlib import Path +import sys +import unittest +from unittest.mock import patch + +MODULE_DIR = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(MODULE_DIR)) +spec = importlib.util.spec_from_file_location('shadow_report', MODULE_DIR / 'shadow_report.py') +shadow = importlib.util.module_from_spec(spec) +spec.loader.exec_module(shadow) +from test_collect_reviews import FakeClient, SNAPSHOT, POLICY +from test_from_risk import risk_record + + +def risk(): + record = risk_record() + record.update(repo=SNAPSHOT['repo'], pr=SNAPSHOT['pr'], head_sha=SNAPSHOT['head_sha'], author=SNAPSHOT['author']) + return {'summary': {'pr': SNAPSHOT['pr'], 'base_sha': SNAPSHOT['base_sha'], + 'tier': record['risk']['tier'], 'status': 'graded'}, 'record': record} + + +def input_data(): + return {'snapshot': SNAPSHOT, 'policy': POLICY} + + +class ShadowTests(unittest.TestCase): + def test_connects_real_risk_grade_to_review_evaluator(self): + result = shadow.build(input_data(), FakeClient(), lambda snapshot: risk()) + self.assertEqual(result['state'], 'WAITING') + self.assertEqual(result['risk']['status'], 'observed') + self.assertEqual(result['collection']['reviews'], 'observed') + self.assertIn('finding evidence incomplete', result['reasons']) + + def test_known_open_review_thread_still_blocks(self): + client = FakeClient() + client.pages[0]['nodes'] = [{'isResolved': False, 'isOutdated': False}] + result = shadow.build(input_data(), client, lambda snapshot: risk()) + self.assertEqual(result['state'], 'BLOCKED') + self.assertEqual(result['risk']['status'], 'observed') + + def test_changed_snapshot_discards_risk_and_reviews(self): + client = FakeClient() + client.last = {**client.pr, 'head': {'sha': 'd' * 40}} + result = shadow.build(input_data(), client, lambda snapshot: risk()) + self.assertEqual(result['state'], 'WAITING') + self.assertEqual(result['risk']['status'], 'unknown') + self.assertEqual(result['collection']['snapshot'], 'changed') + + def test_missing_grade_is_explicit_and_other_evidence_still_works(self): + client = FakeClient() + client.pages[0]['nodes'] = [{'isResolved': False, 'isOutdated': False}] + result = shadow.build(input_data(), client, lambda snapshot: None) + self.assertEqual(result['state'], 'BLOCKED') + self.assertEqual(result['risk']['status'], 'unknown') + self.assertEqual(result['risk_collection'], 'unavailable') + + def test_stale_grade_never_displays_as_current(self): + report = risk() + report['summary']['base_sha'] = 'd' * 40 + result = shadow.build(input_data(), FakeClient(), lambda snapshot: report) + self.assertEqual(result['risk']['status'], 'stale') + self.assertIsNone(result['risk']['tier']) + + def test_grader_is_forced_to_report_only_in_an_isolated_directory(self): + r = risk() + def invoke(args, *, cwd, env, **kwargs): + self.assertEqual(env['REPORT_ONLY'], '1') + self.assertEqual(env['DRY_RUN'], '1') + self.assertEqual(env['PUBLISH_CHECK'], '0') + self.assertEqual(env['PUBLISH_COMMENT'], '0') + self.assertNotIn('GITHUB_OUTPUT', env) + Path(cwd, 'pr-risk-results.jsonl').write_text(json.dumps(r['summary']) + '\n') + Path(cwd, f"record-{SNAPSHOT['pr']}.json").write_text(json.dumps(r['record'])) + return type('Proc', (), {'returncode': 0})() + with patch.object(shadow.subprocess, 'run', side_effect=invoke): + self.assertEqual(shadow.grade(SNAPSHOT)['summary']['status'], 'graded') + + def test_wrong_expected_repo_rejected(self): + with self.assertRaises(ValueError): + shadow.build(input_data(), FakeClient(), lambda snapshot: risk(), expected_repo='other/repo') + + +if __name__ == '__main__': + unittest.main() From 908be9f23608a680c1559ff73ed9f97ca3610f21 Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Wed, 28 Oct 2026 17:27:58 +0800 Subject: [PATCH 2/2] docs: register shadow workflow input contract --- .../tests/test_inputs_docs_drift.py | 1 + docs/callers/review-readiness-shadow.md | 33 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/.github/workflow-pins/tests/test_inputs_docs_drift.py b/.github/workflow-pins/tests/test_inputs_docs_drift.py index 1e507a44..9e1f735f 100644 --- a/.github/workflow-pins/tests/test_inputs_docs_drift.py +++ b/.github/workflow-pins/tests/test_inputs_docs_drift.py @@ -245,6 +245,7 @@ sentinel="workflows_ref", inputs_heading="## Inputs worth setting", ), + Row(workflow="review-readiness-shadow", sentinel="snapshot_and_policy_json", expect_header_with=False), Row(workflow="pr-risk", sentinel="workflows_ref"), Row(workflow="pr-size", sentinel="workflows_ref"), Row(workflow="public-repo-hygiene", sentinel="workflows_ref"), diff --git a/docs/callers/review-readiness-shadow.md b/docs/callers/review-readiness-shadow.md index 29860f8f..3374d564 100644 --- a/docs/callers/review-readiness-shadow.md +++ b/docs/callers/review-readiness-shadow.md @@ -25,3 +25,36 @@ adding the workflow; obtain owner approval and verify the integrated run first. To recover, remove the caller or revert its pin. Existing artifacts retain their normal expiry. An unavailable risk grade is reported as unknown, and an invalid input or failed workflow produces no report artifact rather than a ready result. + +## Inputs + +| Input | Description | +| --- | --- | +| `snapshot_and_policy_json` | Required JSON object containing the target snapshot and advisory policy. | + +## Example caller + +Replace the provider ref with a reviewed, merged 40-character commit. Supply the +input explicitly when dispatching; do not treat PR-controlled data as policy. + +```yaml +name: Manual review shadow +on: + workflow_dispatch: + inputs: + snapshot_and_policy_json: + description: Snapshot and advisory policy JSON + required: true + type: string +permissions: + contents: read + pull-requests: read + checks: read + actions: read + statuses: read +jobs: + shadow: + uses: Comfy-Org/github-workflows/.github/workflows/review-readiness-shadow.yml@ + with: + snapshot_and_policy_json: ${{ inputs.snapshot_and_policy_json }} +```