diff --git a/.github/workflow-pins/tests/test_inputs_docs_drift.py b/.github/workflow-pins/tests/test_inputs_docs_drift.py index 1e507a44..9e1f735f 100644 --- a/.github/workflow-pins/tests/test_inputs_docs_drift.py +++ b/.github/workflow-pins/tests/test_inputs_docs_drift.py @@ -245,6 +245,7 @@ sentinel="workflows_ref", inputs_heading="## Inputs worth setting", ), + Row(workflow="review-readiness-shadow", sentinel="snapshot_and_policy_json", expect_header_with=False), Row(workflow="pr-risk", sentinel="workflows_ref"), Row(workflow="pr-size", sentinel="workflows_ref"), Row(workflow="public-repo-hygiene", sentinel="workflows_ref"), diff --git a/.github/workflows/review-readiness-shadow.yml b/.github/workflows/review-readiness-shadow.yml new file mode 100644 index 00000000..58950e45 --- /dev/null +++ b/.github/workflows/review-readiness-shadow.yml @@ -0,0 +1,56 @@ +name: Review readiness shadow (reusable) + +on: + workflow_call: + inputs: + snapshot_and_policy_json: + description: Snapshot and advisory policy JSON, supplied by the caller. + required: true + type: string + +permissions: + contents: read + pull-requests: read + checks: read + actions: read + statuses: read + +jobs: + report: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + # The runner exposes workflow_sha on the job context. Serialize that + # context because actionlint's static schema does not yet include it; + # the SHA guard fails closed on older runners or an absent value. + - name: Validate provider revision + env: + PROVIDER_SHA: ${{ fromJSON(toJSON(job)).workflow_sha }} + run: | + if [[ ! "$PROVIDER_SHA" =~ ^[0-9a-f]{40}$ ]]; then + echo '::error::Cannot resolve the reusable workflow revision.' + exit 1 + fi + - name: Load provider tools at the workflow revision + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: Comfy-Org/github-workflows + ref: ${{ fromJSON(toJSON(job)).workflow_sha }} + path: _review_tool + persist-credentials: false + - name: Build advisory report + env: + GH_TOKEN: ${{ github.token }} + GH_HOST: github.com + SHADOW_EXPECTED_REPO: ${{ github.repository }} + SHADOW_INPUT: ${{ inputs.snapshot_and_policy_json }} + run: | + set -euo pipefail + printf '%s' "$SHADOW_INPUT" | python3 _review_tool/scripts/review-readiness/shadow_report.py > readiness-report.json + - name: Retain report + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: review-readiness-shadow-${{ github.run_id }}-${{ github.run_attempt }} + path: readiness-report.json + if-no-files-found: error + retention-days: 30 diff --git a/.github/workflows/test-review-readiness.yml b/.github/workflows/test-review-readiness.yml index 5cf7d28a..fac0b1ee 100644 --- a/.github/workflows/test-review-readiness.yml +++ b/.github/workflows/test-review-readiness.yml @@ -5,14 +5,20 @@ on: paths: - 'scripts/review-readiness/**' - '.github/workflows/test-review-readiness.yml' + - '.github/workflows/review-readiness-shadow.yml' push: branches: [main] paths: - 'scripts/review-readiness/**' - '.github/workflows/test-review-readiness.yml' + - '.github/workflows/review-readiness-shadow.yml' permissions: contents: read + pull-requests: read + checks: read + actions: read + statuses: read jobs: test: @@ -25,3 +31,48 @@ jobs: with: python-version: '3.12' - run: python3 -m unittest discover -s scripts/review-readiness/tests -v + + prepare-shadow: + if: ${{ github.event_name == 'pull_request' }} + runs-on: ubuntu-latest + outputs: + input: ${{ steps.input.outputs.json }} + steps: + - id: input + env: + TARGET_REPO: ${{ github.repository }} + TARGET_PR: ${{ github.event.pull_request.number }} + AUTHOR: ${{ github.event.pull_request.user.login }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + set -euo pipefail + payload=$(jq -cn --arg repo "$TARGET_REPO" --argjson pr "$TARGET_PR" \ + --arg author "$AUTHOR" --arg base "$BASE_SHA" --arg head "$HEAD_SHA" \ + '{snapshot:{repo:$repo,pr:$pr,author:$author,base_sha:$base,head_sha:$head,tested_sha:$head,policy_version:"synthetic-ci"},policy:{version:"synthetic-ci",required_checks:[{name:"unit",revision:"tested"}],required_human_approvals:1}}') + printf 'json=%s\n' "$payload" >> "$GITHUB_OUTPUT" + + shadow: + if: ${{ github.event_name == 'pull_request' }} + needs: prepare-shadow + permissions: + contents: read + pull-requests: read + checks: read + actions: read + statuses: read + uses: ./.github/workflows/review-readiness-shadow.yml + with: + snapshot_and_policy_json: ${{ needs.prepare-shadow.outputs.input }} + + verify-shadow: + if: ${{ github.event_name == 'pull_request' }} + needs: shadow + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: review-readiness-shadow-${{ github.run_id }}-${{ github.run_attempt }} + - name: Verify report + run: | + jq -e '(.state == "WAITING" or .state == "BLOCKED") and .collection.snapshot == "matched" and .risk_collection == "observed"' readiness-report.json diff --git a/README.md b/README.md index 5fca45bb..81a3cb77 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ complete, copy-pasteable caller. | [`assign-reviewers.yml`](.github/workflows/assign-reviewers.yml) | Assigns PR owners using configured paths and recent human approval history. Chooses complementary file coverage before workload, with an explained one-owner default fallback and no random substitutions. Optionally generates a shared history artifact on schedule/manual dispatch and reuses fresh snapshots across PRs. Reads configuration from the base SHA, respects manual assignments, and preserves opt-in `REVIEWER_AUTHOR_ALLOWLIST` scoping. Requires `vars.APP_ID` + `CLOUD_CODE_BOT_PRIVATE_KEY`; callers skip forks and Dependabot triggers. | [assign-reviewers.md](docs/callers/assign-reviewers.md) | | [`assign-prs-to-author.yml`](.github/workflows/assign-prs-to-author.yml) | Housekeeping — assigns every open PR with no assignees to its author (bot-authored PRs skipped by default). Run on a schedule from a thin caller; useful when a team tracks PR ownership via assignees. The calling job needs `pull-requests: write` and `issues: write`. | [assign-prs-to-author.md](docs/callers/assign-prs-to-author.md) | | [`pr-size.yml`](.github/workflows/pr-size.yml) | PR-size cap — fails (or, in `mode: warn`, only reports) when a PR's net diff exceeds `max_lines` non-generated changed lines, keeping diffs reviewable. Excludes dependency lockfiles, `linguist-generated` files (read from the base ref, so a PR can't exempt itself), Go generated-code markers, and per-repo `extra_lockfiles` / `extra_generated_globs`. Opt in to `exclude_tests` to cap production code rather than test coverage (excluded test lines are always reported, never silently dropped). A `bypass_label` (default `oversized-ok`) waves through a legitimately large change; a sticky bot comment explains overages when `bot_app_id` + `BOT_APP_PRIVATE_KEY` are supplied (degrades to status + step summary without them). Counting logic + tests live in [`scripts/check-pr-size/`](scripts/check-pr-size). | [pr-size.md](docs/callers/pr-size.md) | +| [`review-readiness-shadow.yml`](.github/workflows/review-readiness-shadow.yml) | Read-only advisory risk and review evidence report. Retains one artifact; it cannot report READY or gate a merge. | [review-readiness-shadow.md](docs/callers/review-readiness-shadow.md) | | [`pr-risk.yml`](.github/workflows/pr-risk.yml) | **Advisory PR risk grading (shadow check)** — **automatic grading off by default** (`enabled: false`; a manual `workflow_dispatch` grades regardless, so a repo can trial it before switching on); switch it on with `enabled: true` or by setting the caller repo's `RISK_CONFIG` variable to `{"enabled": true}`, which outranks the input in both directions so `{"enabled": false}` is a no-PR kill switch. Grades every PR into a tier `R0` (safest) .. `R3` (riskiest) and syncs one label (`risk:R0`..`risk:R3`, or `risk:ungraded` when an input was unreadable). The label is the entire product: nothing is gated, routed, commented, or merged. Deterministic (`gh` + `jq`, no LLM): `grade = worst(path_floor, provenance, reversibility)` — path-glob map, what-process-produced-the-diff (registered runbooks with identity + diff-shape assertions; forks are R3 with no exceptions), and revertability (persistent-state mutation, deletions under sensitive classes, did green checks cover the lines). Grader + generic defaults live in [`scripts/pr-risk/`](scripts/pr-risk); a consumer sharpens them with `.github/risk.json` / `.github/risk-runbooks.json`, read from the PR's **base ref** so a PR can't edit the rules that judge it. The job excludes its own run from the check rollup and waits (`wait_for_checks_minutes`) for the rest to settle before labeling. Labels ride the plain `GITHUB_TOKEN` (cannot fire `labeled` triggers — no cascade risk); disagreement is recorded with a human-owned `risk-dispute` label. **Two further publish surfaces are available and are OFF by default**, so an enrolled caller behaves byte-identically until it opts in: `sticky_comment: true` posts ONE comment (created once, updated in place — N pushes leave one comment) carrying the per-file path-axis breakdown, the risk CONCENTRATION sentence ("94% of this diff is R0/R1; the 6% that puts the path floor at R3 is these two files, 40 lines") and a "this grade is wrong" checkbox whose state round-trips into a `risk-grade-disputed` label (distinct from the human-owned `risk-dispute`, which the grader still never touches); `check_run: true` publishes the tier and reason as a Check Run on the head commit — the immutable, timestamped, commit-attached record a mutable label cannot be — from a SEPARATE job, the only one holding `checks: write` — a grant every caller makes whether or not it switches the surface on, because GitHub validates every nested job's declared `permissions:` at startup and a job `if:` is a runtime condition (the input decides whether a check is PUBLISHED, not whether the grant is CHECKED). Both surfaces are advisory in the same sense as the label: the Check Run's conclusion is hardcoded `neutral`, and every publish failure is an annotation, never a red check. Label text is remappable via `label_map`. `workflows_ref` is **required**, and both its **shape and its ancestry are enforced** — every job that checks it out fails the run *before* the tool checkout unless the value is a full 40-hex lowercase commit SHA **and** that commit is an ancestor of `main` of this repo (fetched from a literal upstream URL; no `github` context can name this repo from inside a reusable workflow, so a variable there would be an alias a fork could point at itself). Shape rejects everything mutable — a branch, a tag, a `refs/pull/N/head`. Ancestry rejects everything unmerged: a fork of this public repo shares its object store, and GitHub serves a fork PR's head objects from this repo's own URL unauthenticated, so a fork-authored SHA is just as well-shaped and would otherwise be checked out into a job holding the caller's write token. There is deliberately **no opt-out input** — an opt-out would be set by the very pin-bump PR the check distrusts. Two consequences by design: **pinning a not-yet-merged SHA is rejected** (merge first, then bump), and the axis **fails closed** if this repo ever goes private or `main` is force-rewritten past a consumer's pin (re-pin to a commit on `main` to recover). **What is still not machine-checked is that the pin is the *current* one**: a stale pin left behind when `uses:` moved is a merged ancestor too. The test that would close that is "equal to the commit `uses:` resolved to". That value IS readable — not as `github.workflow_sha` (the *caller's* top-level file) and not as the `job_workflow_sha` OIDC claim (not a `github` context property; reading it would need `id-token: write` from every caller), but as `job.workflow_sha`, the `job`-context accessor added in runner v2.334.0 that `groom.yml` reads (BE-8077). It is deliberately not wired in here: asserting equality would fail red on every caller whose `uses:` and `workflows_ref` have already drifted apart, which is a caller-contract change tracked separately. The other residual is irreducible: a fork commit that edits `pr-risk.yml` itself is rejected by ancestry, but a caller pinned at one runs the fork's own copy of the guard, and no in-file check can bound a revision of that file an attacker chose. **Reviewing the caller is what bounds both, and it is the only thing that can: require `uses:` at a full commit SHA of this repo and `with: workflows_ref:` set to that same SHA written out literally, character-for-character — never an expression, never a tag.** The guard also runs *before* enablement is resolved (the resolver is itself loaded from `workflows_ref`), so a floating pin fails red even with the `RISK_CONFIG` kill switch set — the switch stops the grading, not a broken enrollment. Call the workflow directly: a nested `workflow_call` chain through an org wrapper is unsupported. Enroll it as its own workflow rather than a job inside an existing CI workflow (the rollup exclusion is per-run). The calling job needs `contents: read` + `issues: write` + `pull-requests: write` + `checks: write` + `actions: read` + `statuses: read`; GitHub rejects a shorter grant at startup (a reusable workflow can only narrow the caller's token, never elevate it), so a caller enrolled from an older copy of this row fails before any step runs. The two label writes are the ONE label: repo-side label creation on first use maps to `issues`, and labeling a PR maps to `pull-requests` (the labels endpoint is dual-mapped by what the "issue" is, so `issues: write` alone 403s on a PR). `checks: write` is the `publish-check` job's declaration, required unconditionally per above — grading itself uses only `checks: read`. `actions: read` is for the rollup's `CheckRun -> checkSuite -> workflowRun` self-exclusion hop. No secrets. | [pr-risk.md](docs/callers/pr-risk.md) | | [`pr-derisk.yml`](.github/workflows/pr-derisk.yml) | **On-demand de-risk split plan (`/derisk`, beta)** — **off by default** (`enabled: false`; `vars.DERISK_CONFIG` on the caller repo outranks it in both directions, so `{"enabled": false}` is a no-PR kill switch). Someone with write access comments `/derisk` on a pull request; the workflow re-grades that PR with the pr-risk grader, makes **one** model call for a semantic partition of the diff into a chain of 2–5 **sequential** PRs, and posts **one sticky advisory comment**: a verdict line and a chain table above the fold, the plan, the chain-landing rules and the floor math inside collapsed `
`. **Every floor shown is computed by `grade-pr-risk.sh --stdin`** over a synthetic record built from each step's files — the model proposes *which files go together* and nothing else, and any tier it writes is discarded before rendering. The partition must cover the changed-file set **exactly**; a missing or duplicated path is rejected, re-prompted **once**, then falls back. **Nothing is gated, routed, merged, labelled or filed** — filing tickets from a plan is a later rung behind its own command. Honest by construction: when no step lands below the PR's **path floor** the verdict reads "N smaller single-concern R3s, same lane", never a fake lane win, because the verdict is arithmetic over grader output rather than the model's prose — and the comparison is against the path floor rather than the headline tier precisely because a split only moves the path axis, so a fork PR (provenance R3, path floor R0) is told the truth instead of a reduction no partition can deliver. An over-budget diff, an unvalidatable partition and an API failure each post an explaining comment — a `/derisk` that quietly does nothing is never an outcome. Safe as a comment command because an `issue_comment` workflow runs from the caller's **default branch** (a PR cannot edit the workflow serving it), the commenter is gated on `author_association` (`allowed_associations`, default `OWNER,MEMBER,COLLABORATOR` — narrow it, never widen), and **no PR code is ever checked out**: the diff is read over the API and handed to a model as text. `workflows_ref` is **required** and its shape + ancestry are enforced by the guard byte-identical to `pr-risk.yml`'s (see that row for the full rationale and its two residuals); a test fails the build if the copies drift. The calling job needs `contents: read` + `pull-requests: write` + `checks: read` + `actions: read` + `statuses: read`, and the `anthropic_api_key` secret. | [pr-derisk.md](docs/callers/pr-derisk.md) | | [`pr-area-label.yml`](.github/workflows/pr-area-label.yml) | **Agentic PR area labeling** — classifies each PR into exactly one `area:*` label, and (on push to the consumer's default branch) syncs the repo's `area:*` labels to the taxonomy. The taxonomy is the consumer's own `.github/area-labels.yml` (`repo_context` + `labels[]` with `name`/`color`/`description`/optional `guidance`); the shared workflow carries nothing repo-specific. An optional `sub_labels[]` block adds **deterministic path sub-labels** — `area:*` names with `paths:` globs, matched against the PR's changed files with no model involved, applied *alongside* the classified area and exempt from its cleanup (retired automatically once a PR stops matching). That is how a subsystem living inside another area's service stays filterable without splitting the parent area; names must be disjoint from `labels[]`. An LLM makes the domain-vs-path judgement a static `paths:` map can't, but with **no tools and no token**: PR title/body/paths/labels go to the Anthropic Messages API as data inside `` tags (diff excluded), the reply is enum-constrained by a JSON schema to the taxonomy's own names, and a deterministic step applies it with targeted `area:*` add/remove ops (never a full-set PUT, so concurrent non-area edits survive). The taxonomy is read from the PR's **base ref** — a PR can't rewrite the rules that classify it — and validated (unique `area:[a-z0-9-]+` names, every label resolves to a non-blank routing guide) before it drives a write; everything fails soft rather than failing the check. Classifier logic lives in [`scripts/area-label/`](scripts/area-label), loaded from the pinned `workflows_ref` (validated to a full 40-hex SHA before checkout). Fork/Dependabot PRs are skipped by construction (no writable token, no secret). Labels ride the plain `GITHUB_TOKEN` (no `labeled`-trigger cascade). The calling job needs `contents: read` + `issues: write` + `pull-requests: write`; the `ANTHROPIC_API_KEY` secret is provided org-wide (unset ⇒ `label-pr` fails soft, sync still works). | [pr-area-label.md](docs/callers/pr-area-label.md) | diff --git a/docs/callers/README.md b/docs/callers/README.md index 315d5cbc..2d4cd8c3 100644 --- a/docs/callers/README.md +++ b/docs/callers/README.md @@ -10,6 +10,7 @@ per-workflow pages give you a complete, copy-pasteable caller: | `cursor-review.yml` | [cursor-review.md](cursor-review.md) | `CURSOR_API_KEY` | | `cursor-review-auto-label.yml` | [cursor-review-auto-label.md](cursor-review-auto-label.md) | `vars.APP_ID` + App key | | `pr-size.yml` | [pr-size.md](pr-size.md) | nothing (bot comment optional) | +| `review-readiness-shadow.yml` | [review-readiness-shadow.md](review-readiness-shadow.md) | nothing | | `pr-risk.yml` | [pr-risk.md](pr-risk.md) | nothing | | `pr-derisk.yml` | [pr-derisk.md](pr-derisk.md) | `ANTHROPIC_API_KEY` | | `pr-area-label.yml` | [pr-area-label.md](pr-area-label.md) | `.github/area-labels.yml` (+ `ANTHROPIC_API_KEY`, org-wide) | diff --git a/docs/callers/review-readiness-shadow.md b/docs/callers/review-readiness-shadow.md new file mode 100644 index 00000000..3374d564 --- /dev/null +++ b/docs/callers/review-readiness-shadow.md @@ -0,0 +1,60 @@ +# Review readiness shadow report + +This reusable workflow evaluates one caller-supplied PR snapshot and policy. It +reads risk inputs, current reviews and review threads, then retains one JSON +artifact for 30 days. It does not write labels, comments, reviews, check runs, +or repository rules. The Actions run and job checks still exist. The token has +read access to contents, pull requests, checks, actions and commit statuses. + +The input `snapshot_and_policy_json` is a JSON object containing `snapshot` and +`policy` as documented in [the evaluator](../../scripts/review-readiness/README.md). +The target repo must equal the caller repo. The caller is responsible for obtaining +and verifying the snapshot and for choosing an authorized policy; input JSON is +not itself proof. Use only a reviewed, merged provider commit pinned by full SHA +in the caller's `uses:`. The provider tools load at the workflow's own revision, +never from the PR. This workflow runs only when explicitly called. + +The report records the supplied `tested_sha` but cannot prove that tests actually +ran on it. It does not verify reviewer permissions or cover every finding source. +Missing checks and incomplete findings prevent `READY`; known current failures +can still produce `BLOCKED`. Data reads are not atomic, so a review change or an +ABA revision change can race collection. It is not an authorization to merge or +a substitute for GitHub branch protection. No pilot or enforcement follows from +adding the workflow; obtain owner approval and verify the integrated run first. + +To recover, remove the caller or revert its pin. Existing artifacts retain their +normal expiry. An unavailable risk grade is reported as unknown, and an invalid +input or failed workflow produces no report artifact rather than a ready result. + +## Inputs + +| Input | Description | +| --- | --- | +| `snapshot_and_policy_json` | Required JSON object containing the target snapshot and advisory policy. | + +## Example caller + +Replace the provider ref with a reviewed, merged 40-character commit. Supply the +input explicitly when dispatching; do not treat PR-controlled data as policy. + +```yaml +name: Manual review shadow +on: + workflow_dispatch: + inputs: + snapshot_and_policy_json: + description: Snapshot and advisory policy JSON + required: true + type: string +permissions: + contents: read + pull-requests: read + checks: read + actions: read + statuses: read +jobs: + shadow: + uses: Comfy-Org/github-workflows/.github/workflows/review-readiness-shadow.yml@ + with: + snapshot_and_policy_json: ${{ inputs.snapshot_and_policy_json }} +``` diff --git a/scripts/review-readiness/README.md b/scripts/review-readiness/README.md index 874f3e00..995a4388 100644 --- a/scripts/review-readiness/README.md +++ b/scripts/review-readiness/README.md @@ -89,3 +89,18 @@ escape the recheck. It is advisory evidence, not a merge gate. ```sh python3 scripts/review-readiness/collect_reviews.py < snapshot-and-policy.json ``` + +## Integrated shadow report + +`shadow_report.py` runs the real risk grader with `REPORT_ONLY=1` and no +publication options, then collects GitHub reviews for the same supplied snapshot. +It runs the grader in a temporary directory and emits one JSON report to stdout. +A changed or unavailable final snapshot discards the risk grade too. The report +remains advisory: missing checks and incomplete findings prevent `READY`. + +The reusable workflow `review-readiness-shadow.yml` retains that report for +30 days. Its caller provides the snapshot and policy; the report records the +provided tested SHA but does not verify it. The target repository must match +the caller repository. The workflow checks out its tools at its own immutable +workflow revision and does not execute PR code. Only use a reviewed, merged +provider revision in a caller. See the caller guide for trust and rollout limits. diff --git a/scripts/review-readiness/shadow_report.py b/scripts/review-readiness/shadow_report.py new file mode 100644 index 00000000..d7b16d1e --- /dev/null +++ b/scripts/review-readiness/shadow_report.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Create an advisory report using the report-only grader and review collector.""" + +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile + +import collect_reviews +import from_risk +import readiness + +RISK_DIR = Path(__file__).resolve().parents[1] / 'pr-risk' + + +def grade(snapshot): + with tempfile.TemporaryDirectory(prefix='readiness-risk-') as directory: + env = os.environ.copy() + env.pop('GITHUB_OUTPUT', None) + env.update({ + 'REPO': snapshot['repo'], 'PR_NUMBERS': str(snapshot['pr']), + 'BASE_REF': '', 'TOOL_DIR': str(RISK_DIR), 'RESULTS': 'pr-risk-results.jsonl', + 'REPORT_ONLY': '1', 'DRY_RUN': '1', 'PUBLISH_CHECK': '0', 'PUBLISH_COMMENT': '0', + 'WAIT_MINUTES': '0', 'JOB_TIMEOUT_MINUTES': '10', 'MAX_TARGETS': '1', + 'MAP_PATH': '.github/risk.json', 'RB_PATH': '.github/risk-runbooks.json', + 'FLEET_LOGINS': '', 'BOT_LOGINS': '', 'LABEL_MAP': '', + 'SELF_CONTEXT': os.environ.get('GITHUB_WORKFLOW', ''), + 'SELF_RUN_ID': os.environ.get('GITHUB_RUN_ID', ''), + }) + try: + proc = subprocess.run(['bash', str(RISK_DIR / 'grade-targets.sh')], cwd=directory, + env=env, capture_output=True, text=True, timeout=600) + if proc.returncode: + return None + lines = (Path(directory) / 'pr-risk-results.jsonl').read_text().splitlines() + if len(lines) != 1: + return None + summary = json.loads(lines[0]) + if not isinstance(summary, dict): + return None + # The target number is validated by readiness before it reaches this path. + record_path = Path(directory) / f"record-{snapshot['pr']}.json" + record = json.loads(record_path.read_text()) if record_path.exists() else None + return {'summary': summary, 'record': record} + except (OSError, ValueError, subprocess.SubprocessError): + return None + + +def build(data, client, run_risk=grade, expected_repo=None): + if not isinstance(data, dict): + raise ValueError('input must be an object') + # Validate the snapshot and policy before sending anything to GitHub. + readiness.evaluate({'snapshot': data.get('snapshot'), 'policy': data.get('policy'), + 'checks_complete': False, 'checks': []}) + snapshot = data['snapshot'] + if expected_repo is not None and snapshot['repo'].casefold() != expected_repo.casefold(): + raise ValueError('target repository does not match caller') + report = run_risk(snapshot) + risk = from_risk.evaluate_report({**data, 'report': report})['risk'] + result = collect_reviews.collect(data, client) + if result['collection']['snapshot'] != 'matched': + risk = {'status': 'unknown', 'tier': None} + result['risk'] = risk + result['risk_collection'] = 'observed' if report is not None else 'unavailable' + return result + + +def main(): + try: + data = json.load(sys.stdin) + readiness.evaluate({'snapshot': data.get('snapshot'), 'policy': data.get('policy'), + 'checks_complete': False, 'checks': []}) + snapshot = data['snapshot'] + result = build(data, collect_reviews.GitHub(snapshot['repo'], snapshot['pr']), + expected_repo=os.environ.get('SHADOW_EXPECTED_REPO')) + except (ValueError, TypeError, AttributeError, json.JSONDecodeError) as exc: + print(f'invalid shadow input: {exc}', file=sys.stderr) + return 2 + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/scripts/review-readiness/tests/test_shadow_report.py b/scripts/review-readiness/tests/test_shadow_report.py new file mode 100644 index 00000000..855acc2a --- /dev/null +++ b/scripts/review-readiness/tests/test_shadow_report.py @@ -0,0 +1,86 @@ +import importlib.util +import json +from pathlib import Path +import sys +import unittest +from unittest.mock import patch + +MODULE_DIR = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(MODULE_DIR)) +spec = importlib.util.spec_from_file_location('shadow_report', MODULE_DIR / 'shadow_report.py') +shadow = importlib.util.module_from_spec(spec) +spec.loader.exec_module(shadow) +from test_collect_reviews import FakeClient, SNAPSHOT, POLICY +from test_from_risk import risk_record + + +def risk(): + record = risk_record() + record.update(repo=SNAPSHOT['repo'], pr=SNAPSHOT['pr'], head_sha=SNAPSHOT['head_sha'], author=SNAPSHOT['author']) + return {'summary': {'pr': SNAPSHOT['pr'], 'base_sha': SNAPSHOT['base_sha'], + 'tier': record['risk']['tier'], 'status': 'graded'}, 'record': record} + + +def input_data(): + return {'snapshot': SNAPSHOT, 'policy': POLICY} + + +class ShadowTests(unittest.TestCase): + def test_connects_real_risk_grade_to_review_evaluator(self): + result = shadow.build(input_data(), FakeClient(), lambda snapshot: risk()) + self.assertEqual(result['state'], 'WAITING') + self.assertEqual(result['risk']['status'], 'observed') + self.assertEqual(result['collection']['reviews'], 'observed') + self.assertIn('finding evidence incomplete', result['reasons']) + + def test_known_open_review_thread_still_blocks(self): + client = FakeClient() + client.pages[0]['nodes'] = [{'isResolved': False, 'isOutdated': False}] + result = shadow.build(input_data(), client, lambda snapshot: risk()) + self.assertEqual(result['state'], 'BLOCKED') + self.assertEqual(result['risk']['status'], 'observed') + + def test_changed_snapshot_discards_risk_and_reviews(self): + client = FakeClient() + client.last = {**client.pr, 'head': {'sha': 'd' * 40}} + result = shadow.build(input_data(), client, lambda snapshot: risk()) + self.assertEqual(result['state'], 'WAITING') + self.assertEqual(result['risk']['status'], 'unknown') + self.assertEqual(result['collection']['snapshot'], 'changed') + + def test_missing_grade_is_explicit_and_other_evidence_still_works(self): + client = FakeClient() + client.pages[0]['nodes'] = [{'isResolved': False, 'isOutdated': False}] + result = shadow.build(input_data(), client, lambda snapshot: None) + self.assertEqual(result['state'], 'BLOCKED') + self.assertEqual(result['risk']['status'], 'unknown') + self.assertEqual(result['risk_collection'], 'unavailable') + + def test_stale_grade_never_displays_as_current(self): + report = risk() + report['summary']['base_sha'] = 'd' * 40 + result = shadow.build(input_data(), FakeClient(), lambda snapshot: report) + self.assertEqual(result['risk']['status'], 'stale') + self.assertIsNone(result['risk']['tier']) + + def test_grader_is_forced_to_report_only_in_an_isolated_directory(self): + r = risk() + def invoke(args, *, cwd, env, **kwargs): + self.assertEqual(env['REPORT_ONLY'], '1') + self.assertEqual(env['DRY_RUN'], '1') + self.assertEqual(env['PUBLISH_CHECK'], '0') + self.assertEqual(env['PUBLISH_COMMENT'], '0') + self.assertNotIn('GITHUB_OUTPUT', env) + Path(cwd, 'pr-risk-results.jsonl').write_text(json.dumps(r['summary']) + '\n') + Path(cwd, f"record-{SNAPSHOT['pr']}.json").write_text(json.dumps(r['record'])) + return type('Proc', (), {'returncode': 0})() + with patch.object(shadow.subprocess, 'run', side_effect=invoke): + self.assertEqual(shadow.grade(SNAPSHOT)['summary']['status'], 'graded') + + def test_wrong_expected_repo_rejected(self): + with self.assertRaises(ValueError): + shadow.build(input_data(), FakeClient(), lambda snapshot: risk(), expected_repo='other/repo') + + +if __name__ == '__main__': + unittest.main()