diff --git a/.github/workflows/test-review-readiness.yml b/.github/workflows/test-review-readiness.yml index fac0b1ee..8e2b645f 100644 --- a/.github/workflows/test-review-readiness.yml +++ b/.github/workflows/test-review-readiness.yml @@ -49,12 +49,12 @@ jobs: set -euo pipefail payload=$(jq -cn --arg repo "$TARGET_REPO" --argjson pr "$TARGET_PR" \ --arg author "$AUTHOR" --arg base "$BASE_SHA" --arg head "$HEAD_SHA" \ - '{snapshot:{repo:$repo,pr:$pr,author:$author,base_sha:$base,head_sha:$head,tested_sha:$head,policy_version:"synthetic-ci"},policy:{version:"synthetic-ci",required_checks:[{name:"unit",revision:"tested"}],required_human_approvals:1}}') + '{snapshot:{repo:$repo,pr:$pr,author:$author,base_sha:$base,head_sha:$head,tested_sha:$head,policy_version:"synthetic-ci"},policy:{version:"synthetic-ci",required_checks:[{name:"test",revision:"tested"}],required_human_approvals:1,actions_checks:[{name:"test",workflow_path:".github/workflows/test-review-readiness.yml"}]}}') printf 'json=%s\n' "$payload" >> "$GITHUB_OUTPUT" shadow: if: ${{ github.event_name == 'pull_request' }} - needs: prepare-shadow + needs: [prepare-shadow, test] permissions: contents: read pull-requests: read @@ -75,4 +75,4 @@ jobs: name: review-readiness-shadow-${{ github.run_id }}-${{ github.run_attempt }} - name: Verify report run: | - jq -e '(.state == "WAITING" or .state == "BLOCKED") and .collection.snapshot == "matched" and .risk_collection == "observed"' readiness-report.json + jq -e '(.state == "WAITING" or .state == "BLOCKED") and .collection.snapshot == "matched" and .risk_collection == "observed" and .collection.actions_checks == "observed" and (.actions_checks_matched | index("test") != null)' readiness-report.json diff --git a/scripts/review-readiness/README.md b/scripts/review-readiness/README.md index 995a4388..23d464b9 100644 --- a/scripts/review-readiness/README.md +++ b/scripts/review-readiness/README.md @@ -104,3 +104,24 @@ provided tested SHA but does not verify it. The target repository must match the caller repository. The workflow checks out its tools at its own immutable workflow revision and does not execute PR code. Only use a reviewed, merged provider revision in a caller. See the caller guide for trust and rollout limits. + +## GitHub Actions failure evidence + +A policy may add `actions_checks`, with a required check `name` and an exact +`workflow_path` for each source. The collector reads all check-run pages and +links each candidate to its GitHub Actions job and pull-request workflow run. +The GitHub Actions app, run attempt, workflow path, PR number, base and head must +match; the latest unambiguous matching job's reported failure blocks. + +`actions_checks_matched` lists the matched names. An unreadable source, partial +inventory, unstarted or ambiguous newer job remains unknown; older failures do +not replace it. The collector rechecks the PR snapshot after collection and +discards these signals when it changes. Reads are not atomic and do not establish +reviewer permissions or a merge candidate. + +This is **negative-only evidence**: a successful check run is attached to the PR +head, but that does not prove what revision its job actually tested. It never +satisfies a required passing check or produces `READY`. Caller-supplied policy +must be authorized separately. The pure evaluator accepts scoped +`reported_failures` and includes their proof links, but does not itself verify +the producer or link contents. diff --git a/scripts/review-readiness/collect_actions.py b/scripts/review-readiness/collect_actions.py new file mode 100644 index 00000000..88707813 --- /dev/null +++ b/scripts/review-readiness/collect_actions.py @@ -0,0 +1,94 @@ +"""Find scoped GitHub Actions failures without claiming tested-revision proof.""" + +from datetime import datetime +import re + +import readiness +from evidence_errors import EvidenceError + +ACTIONS_APP_ID = 15368 +FAILURES = {'failure', 'timed_out', 'action_required', 'startup_failure'} + + +def selectors(policy): + rules = policy.get('actions_checks', []) + if not isinstance(rules, list) or len(rules) > 20: + raise ValueError('actions_checks must be a list of at most 20 checks') + required = {item['name'] for item in policy['required_checks']} + result = {} + for item in rules: + if not isinstance(item, dict): + raise ValueError('invalid actions check') + name, path = item.get('name'), item.get('workflow_path') + if (not isinstance(name, str) or name not in required or name in result + or not isinstance(path, str) + or not re.fullmatch(r'\.github/workflows/[A-Za-z0-9_-]+\.ya?ml', path)): + raise ValueError('actions check needs a required name and a workflow path') + result[name] = path + return result + + +def run_matches(run, snapshot, path, attempt): + if (not isinstance(run, dict) or run.get('event') != 'pull_request' + or run.get('path') != path or run.get('head_sha') != snapshot['head_sha'] + or run.get('run_attempt') != attempt): + return False + prs = run.get('pull_requests') + if not isinstance(prs, list): + return False + return any(isinstance(pr, dict) and pr.get('number') == snapshot['pr'] + and isinstance(pr.get('base'), dict) and pr['base'].get('sha') == snapshot['base_sha'] + and isinstance(pr.get('head'), dict) and pr['head'].get('sha') == snapshot['head_sha'] + for pr in prs) + + +def collect(snapshot, rules, client): + if not rules: + return [], 'not_requested', [] + checks = client.list_check_runs(snapshot['head_sha']) + if not isinstance(checks, list): + raise ValueError('invalid checks') + signals = [] + matched = [] + partial = False + scope = {key: snapshot[key] for key in readiness.SCOPE_KEYS} + for name, workflow_path in rules.items(): + relevant = [c for c in checks if isinstance(c, dict) and c.get('name') == name + and isinstance(c.get('app'), dict) and c['app'].get('id') == ACTIONS_APP_ID] + if len(relevant) > 100: + partial = True + continue + candidates = [] + try: + for check in relevant: + check_id = check.get('id') + if type(check_id) is not int or check_id <= 0 or check.get('head_sha') != snapshot['head_sha']: + raise ValueError('invalid check') + started = datetime.fromisoformat(check['started_at'].replace('Z', '+00:00')) + if started.utcoffset() is None: + raise ValueError('invalid check time') + job = client.job(check_id) + if (not isinstance(job, dict) or job.get('id') != check_id or job.get('name') != name + or job.get('head_sha') != snapshot['head_sha'] + or job.get('status') != check.get('status') + or job.get('conclusion') != check.get('conclusion') + or type(job.get('run_id')) is not int or type(job.get('run_attempt')) is not int): + raise ValueError('inconsistent job') + run = client.run(job['run_id']) + if not isinstance(run, dict) or run.get('id') != job['run_id']: + raise ValueError('inconsistent run') + if run_matches(run, snapshot, workflow_path, job['run_attempt']): + candidates.append((started, check, job['run_id'])) + candidates.sort(key=lambda item: item[0], reverse=True) + if len(candidates) > 1 and candidates[0][0] == candidates[1][0]: + raise ValueError('ambiguous latest check') + if candidates: + matched.append(name) + _, latest, run_id = candidates[0] + if latest['status'] == 'completed' and latest['conclusion'] in FAILURES: + proof = f"https://github.com/{snapshot['repo']}/actions/runs/{run_id}/job/{latest['id']}" + signals.append({'scope': scope, 'name': name, 'proof': proof}) + except (EvidenceError, ValueError, TypeError, KeyError, AttributeError): + # A missing or malformed page/job/run cannot make an older job current. + partial = True + return signals, 'partial' if partial else 'observed', matched diff --git a/scripts/review-readiness/collect_reviews.py b/scripts/review-readiness/collect_reviews.py index 0e093779..4b751008 100644 --- a/scripts/review-readiness/collect_reviews.py +++ b/scripts/review-readiness/collect_reviews.py @@ -7,6 +7,8 @@ import sys import readiness +import collect_actions +from evidence_errors import EvidenceError THREAD_QUERY = ''' query($owner: String!, $name: String!, $pr: Int!, $cursor: String) { @@ -22,10 +24,6 @@ ''' -class EvidenceError(Exception): - pass - - class GitHub: def __init__(self, repo, pr): self.owner, self.name = repo.split('/') @@ -49,6 +47,28 @@ def list_reviews(self): raise EvidenceError('invalid review pages') return [review for page in pages for review in page] + def list_check_runs(self, head): + path = f'repos/{self.owner}/{self.name}/commits/{head}/check-runs?per_page=100&filter=all' + pages = self._api(['--paginate', '--slurp', path]) + if not isinstance(pages, list) or not pages: + raise EvidenceError('invalid check pages') + checks = [] + totals = set() + for page in pages: + if not isinstance(page, dict) or type(page.get('total_count')) is not int or not isinstance(page.get('check_runs'), list): + raise EvidenceError('invalid check pages') + totals.add(page['total_count']) + checks.extend(page['check_runs']) + if len(totals) != 1 or len(checks) != next(iter(totals)): + raise EvidenceError('incomplete check pages') + return checks + + def job(self, job_id): + return self._api([f'repos/{self.owner}/{self.name}/actions/jobs/{job_id}']) + + def run(self, run_id): + return self._api([f'repos/{self.owner}/{self.name}/actions/runs/{run_id}']) + def threads(self, cursor): args = ['graphql', '-f', f'query={THREAD_QUERY}', '-f', f'owner={self.owner}', '-f', f'name={self.name}', '-F', f'pr={self.pr}'] @@ -128,10 +148,12 @@ def collect(data, client): base = {'snapshot': data.get('snapshot'), 'policy': data.get('policy'), 'checks_complete': False, 'checks': [], 'reviews': None, 'findings': None} readiness.evaluate(base) + rules = collect_actions.selectors(base['policy']) snapshot = base['snapshot'] scope = {key: snapshot[key] for key in readiness.SCOPE_KEYS} proof = f"https://github.com/{snapshot['repo']}/pull/{snapshot['pr']}" - status = {'snapshot': 'unavailable', 'reviews': 'not_attempted', 'threads': 'not_attempted'} + matched_checks = [] + status = {'snapshot': 'unavailable', 'reviews': 'not_attempted', 'threads': 'not_attempted', 'actions_checks': 'not_attempted'} try: if matches(client.pull(), snapshot): status['snapshot'] = 'matched' @@ -148,19 +170,24 @@ def collect(data, client): status['threads'] = 'observed' except EvidenceError: status['threads'] = 'partial' + try: + base['reported_failures'], status['actions_checks'], matched_checks = collect_actions.collect(snapshot, rules, client) + except (EvidenceError, ValueError, TypeError): + status['actions_checks'] = 'unavailable' if not matches(client.pull(), snapshot): status['snapshot'] = 'changed' - base['reviews'], base['findings'] = None, None + base['reviews'], base['findings'], base['reported_failures'] = None, None, [] else: status['snapshot'] = 'changed' except EvidenceError: status['snapshot'] = 'unavailable' - base['reviews'], base['findings'] = None, None + base['reviews'], base['findings'], base['reported_failures'] = None, None, [] if status['snapshot'] != 'matched': - for key in ('reviews', 'threads'): + for key in ('reviews', 'threads', 'actions_checks'): if status[key] != 'not_attempted': status[key] = 'discarded' result = readiness.evaluate(base) + result['actions_checks_matched'] = matched_checks if status['snapshot'] == 'matched' else [] result['collection'] = status return result diff --git a/scripts/review-readiness/evidence_errors.py b/scripts/review-readiness/evidence_errors.py new file mode 100644 index 00000000..6db76aff --- /dev/null +++ b/scripts/review-readiness/evidence_errors.py @@ -0,0 +1,2 @@ +class EvidenceError(Exception): + """A remote evidence source could not be read reliably.""" diff --git a/scripts/review-readiness/readiness.py b/scripts/review-readiness/readiness.py index e1e01131..7e470fb8 100644 --- a/scripts/review-readiness/readiness.py +++ b/scripts/review-readiness/readiness.py @@ -106,6 +106,17 @@ def evaluate(data): if _proof(proof): proofs.add(proof) results[name].append((result, _proof(proof))) + # A reported failure can block, but does not prove which revision ran and + # cannot satisfy a required successful check. + for item in _list(data.get("reported_failures", []), "reported_failures"): + item = _object(item, "reported failure") + if _scope(item.get("scope")) != scope: + continue + name = _text(item.get("name"), "reported failure.name") + if name in required and _proof(item.get("proof")): + blocked.append(f"required check reported failure: {name}") + proofs.add(item["proof"]) + for name, found in results.items(): if any(result in ("failure", "error") for result, _ in found): blocked.append(f"required check failed: {name}") diff --git a/scripts/review-readiness/shadow_report.py b/scripts/review-readiness/shadow_report.py index d7b16d1e..8fec1284 100644 --- a/scripts/review-readiness/shadow_report.py +++ b/scripts/review-readiness/shadow_report.py @@ -9,6 +9,7 @@ import tempfile import collect_reviews +import collect_actions import from_risk import readiness @@ -54,6 +55,7 @@ def build(data, client, run_risk=grade, expected_repo=None): # Validate the snapshot and policy before sending anything to GitHub. readiness.evaluate({'snapshot': data.get('snapshot'), 'policy': data.get('policy'), 'checks_complete': False, 'checks': []}) + collect_actions.selectors(data['policy']) snapshot = data['snapshot'] if expected_repo is not None and snapshot['repo'].casefold() != expected_repo.casefold(): raise ValueError('target repository does not match caller') diff --git a/scripts/review-readiness/tests/test_check_failures.py b/scripts/review-readiness/tests/test_check_failures.py new file mode 100644 index 00000000..dcf67ba0 --- /dev/null +++ b/scripts/review-readiness/tests/test_check_failures.py @@ -0,0 +1,169 @@ +import json +import os +import subprocess +import tempfile +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import collect_reviews +import readiness +import shadow_report +from test_collect_reviews import FakeClient, SNAPSHOT, POLICY, HEAD, BASE, review + + +def policy(): + return {**POLICY, 'actions_checks': [{'name': 'unit', 'workflow_path': '.github/workflows/ci.yml'}]} + + +def check(identifier=101, state='completed', conclusion='failure', started='2026-01-01T00:00:00Z'): + return {'id': identifier, 'name': 'unit', 'head_sha': HEAD, 'status': state, + 'conclusion': conclusion, 'started_at': started, 'app': {'id': 15368}} + + +class CheckClient(FakeClient): + def __init__(self): + super().__init__() + self.checks = [check()] + self.job_data = {'id': 101, 'name': 'unit', 'head_sha': HEAD, 'run_id': 500, + 'run_attempt': 1, 'status': 'completed', 'conclusion': 'failure'} + self.run_data = {'id': 500, 'event': 'pull_request', 'run_attempt': 1, + 'path': '.github/workflows/ci.yml', 'head_sha': HEAD, + 'pull_requests': [{'number': 42, 'base': {'sha': BASE}, 'head': {'sha': HEAD}}]} + + def list_check_runs(self, head): + return self.checks + + def job(self, job_id): + return {**self.job_data, 'id': job_id} + + def run(self, run_id): + return self.run_data + + +def run(client): + return collect_reviews.collect({'snapshot': SNAPSHOT, 'policy': policy()}, client) + + +class CheckFailuresTests(unittest.TestCase): + def test_matching_failed_actions_job_blocks_despite_missing_positive_evidence(self): + result = run(CheckClient()) + self.assertEqual(result['state'], 'BLOCKED') + self.assertIn('required check reported failure: unit', result['reasons']) + self.assertIn('check inventory incomplete', result['reasons']) + self.assertEqual(result['collection']['actions_checks'], 'observed') + self.assertEqual(result['actions_checks_matched'], ['unit']) + + def test_success_does_not_prove_what_was_tested(self): + client = CheckClient() + client.checks[0]['conclusion'] = 'success' + client.job_data['conclusion'] = 'success' + result = run(client) + self.assertEqual(result['state'], 'WAITING') + self.assertIn('required check incomplete: unit', result['reasons']) + + def test_old_base_head_wrong_workflow_and_wrong_app_do_not_block(self): + for mutation in ('base', 'head', 'workflow', 'app', 'attempt'): + client = CheckClient() + if mutation == 'base': client.run_data['pull_requests'][0]['base']['sha'] = 'd' * 40 + if mutation == 'head': client.run_data['pull_requests'][0]['head']['sha'] = 'd' * 40 + if mutation == 'workflow': client.run_data['path'] = '.github/workflows/other.yml' + if mutation == 'app': client.checks[0]['app']['id'] = 123 + if mutation == 'attempt': client.job_data['run_attempt'] = 2 + self.assertEqual(run(client)['state'], 'WAITING', mutation) + + def test_newer_pending_check_supersedes_older_failure(self): + client = CheckClient() + client.checks.append(check(102, 'in_progress', None, '2026-01-01T00:01:00Z')) + def job(job_id): + return {**client.job_data, 'id': job_id, 'status': 'in_progress' if job_id == 102 else 'completed', + 'conclusion': None if job_id == 102 else 'failure'} + client.job = job + self.assertEqual(run(client)['state'], 'WAITING') + + def test_api_failure_is_visible_and_cannot_count_as_clean(self): + client = CheckClient() + def fail(): raise collect_reviews.EvidenceError('unavailable') + client.list_check_runs = lambda head: fail() + result = run(client) + self.assertEqual(result['state'], 'WAITING') + self.assertEqual(result['collection']['actions_checks'], 'unavailable') + + def test_real_cli_processes_paginated_github_check_response(self): + with tempfile.TemporaryDirectory() as directory: + fixture = CheckClient() + data = {'pull': fixture.pr, 'reviews': [[review()]], 'checks': fixture.checks, + 'job': fixture.job_data, 'run': fixture.run_data} + path = Path(directory) / 'fixture.json' + path.write_text(json.dumps(data)) + gh = Path(directory) / 'gh' + gh.write_text("""#!/usr/bin/env python3 +import json, os, sys +f = json.load(open(os.environ['FIXTURE_PATH'])) +args = sys.argv[1:] +if 'graphql' in args: + value = {'data': {'repository': {'pullRequest': {'reviewThreads': {'nodes': [], 'pageInfo': {'hasNextPage': False, 'endCursor': None}}}}}} +elif any('check-runs?' in arg for arg in args): + assert '--paginate' in args and '--slurp' in args and any('filter=all' in arg for arg in args) + value = [{'total_count': len(f['checks']), 'check_runs': f['checks']}] +elif any('/reviews?' in arg for arg in args): value = f['reviews'] +elif any('/actions/jobs/' in arg for arg in args): value = f['job'] +elif any('/actions/runs/' in arg for arg in args): value = f['run'] +else: value = f['pull'] +print(json.dumps(value)) +""") + gh.chmod(0o755) + env = {**os.environ, 'PATH': directory + os.pathsep + os.environ['PATH'], 'FIXTURE_PATH': str(path)} + proc = subprocess.run([sys.executable, str(Path(collect_reviews.__file__))], + input=json.dumps({'snapshot': SNAPSHOT, 'policy': policy()}), + capture_output=True, text=True, env=env, check=True) + result = json.loads(proc.stdout) + self.assertEqual(result['state'], 'BLOCKED') + self.assertIn('required check reported failure: unit', result['reasons']) + + def test_inconsistent_pages_fail_closed(self): + client = collect_reviews.GitHub(SNAPSHOT['repo'], SNAPSHOT['pr']) + client._api = lambda args: [{'total_count': 2, 'check_runs': [check()]}] + with self.assertRaises(collect_reviews.EvidenceError): + client.list_check_runs(HEAD) + + def test_indeterminate_newer_job_cannot_revive_old_failure(self): + client = CheckClient() + client.checks.append(check(102, 'queued', None, None)) + result = run(client) + self.assertEqual(result['state'], 'WAITING') + self.assertEqual(result['collection']['actions_checks'], 'partial') + + def test_snapshot_change_discards_check_failure(self): + client = CheckClient() + client.last = {**client.pr, 'head': {'sha': 'd' * 40}} + result = run(client) + self.assertEqual(result['state'], 'WAITING') + self.assertEqual(result['collection']['actions_checks'], 'discarded') + self.assertEqual(result['actions_checks_matched'], []) + + def test_invalid_source_policy_rejected_before_grading(self): + data = {'snapshot': SNAPSHOT, 'policy': {**POLICY, 'actions_checks': [{'name': 'unit', 'workflow_path': '../unsafe.yml'}]}} + with self.assertRaises(ValueError): + shadow_report.build(data, CheckClient(), lambda snapshot: self.fail('unexpected risk read')) + + def test_no_selector_makes_no_api_calls(self): + client = CheckClient() + client.list_check_runs = lambda head: self.fail('unexpected API read') + result = collect_reviews.collect({'snapshot': SNAPSHOT, 'policy': POLICY}, client) + self.assertEqual(result['state'], 'WAITING') + + def test_reported_failure_never_satisfies_success_or_unrelated_check(self): + scope = {k: SNAPSHOT[k] for k in readiness.SCOPE_KEYS} + data = {'snapshot': SNAPSHOT, 'policy': POLICY, 'checks_complete': False, 'checks': [], + 'reported_failures': [{'scope': scope, 'name': 'other', 'proof': 'https://example.com/job'}]} + self.assertEqual(readiness.evaluate(data)['state'], 'WAITING') + data['reported_failures'][0]['name'] = 'unit' + self.assertEqual(readiness.evaluate(data)['state'], 'BLOCKED') + data['reported_failures'][0]['scope'] = {**scope, 'base_sha': 'd' * 40} + self.assertEqual(readiness.evaluate(data)['state'], 'WAITING') + + +if __name__ == '__main__': + unittest.main()