Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .github/workflows/test-review-readiness.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,12 +49,12 @@ jobs:
set -euo pipefail
payload=$(jq -cn --arg repo "$TARGET_REPO" --argjson pr "$TARGET_PR" \
--arg author "$AUTHOR" --arg base "$BASE_SHA" --arg head "$HEAD_SHA" \
'{snapshot:{repo:$repo,pr:$pr,author:$author,base_sha:$base,head_sha:$head,tested_sha:$head,policy_version:"synthetic-ci"},policy:{version:"synthetic-ci",required_checks:[{name:"unit",revision:"tested"}],required_human_approvals:1}}')
'{snapshot:{repo:$repo,pr:$pr,author:$author,base_sha:$base,head_sha:$head,tested_sha:$head,policy_version:"synthetic-ci"},policy:{version:"synthetic-ci",required_checks:[{name:"test",revision:"tested"}],required_human_approvals:1,actions_checks:[{name:"test",workflow_path:".github/workflows/test-review-readiness.yml"}]}}')
printf 'json=%s\n' "$payload" >> "$GITHUB_OUTPUT"

shadow:
if: ${{ github.event_name == 'pull_request' }}
needs: prepare-shadow
needs: [prepare-shadow, test]
permissions:
contents: read
pull-requests: read
Expand All @@ -75,4 +75,4 @@ jobs:
name: review-readiness-shadow-${{ github.run_id }}-${{ github.run_attempt }}
- name: Verify report
run: |
jq -e '(.state == "WAITING" or .state == "BLOCKED") and .collection.snapshot == "matched" and .risk_collection == "observed"' readiness-report.json
jq -e '(.state == "WAITING" or .state == "BLOCKED") and .collection.snapshot == "matched" and .risk_collection == "observed" and .collection.actions_checks == "observed" and (.actions_checks_matched | index("test") != null)' readiness-report.json
21 changes: 21 additions & 0 deletions scripts/review-readiness/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -104,3 +104,24 @@ provided tested SHA but does not verify it. The target repository must match
the caller repository. The workflow checks out its tools at its own immutable
workflow revision and does not execute PR code. Only use a reviewed, merged
provider revision in a caller. See the caller guide for trust and rollout limits.

## GitHub Actions failure evidence

A policy may add `actions_checks`, with a required check `name` and an exact
`workflow_path` for each source. The collector reads all check-run pages and
links each candidate to its GitHub Actions job and pull-request workflow run.
The GitHub Actions app, run attempt, workflow path, PR number, base and head must
match; the latest unambiguous matching job's reported failure blocks.

`actions_checks_matched` lists the matched names. An unreadable source, partial
inventory, unstarted or ambiguous newer job remains unknown; older failures do
not replace it. The collector rechecks the PR snapshot after collection and
discards these signals when it changes. Reads are not atomic and do not establish
reviewer permissions or a merge candidate.

This is **negative-only evidence**: a successful check run is attached to the PR
head, but that does not prove what revision its job actually tested. It never
satisfies a required passing check or produces `READY`. Caller-supplied policy
must be authorized separately. The pure evaluator accepts scoped
`reported_failures` and includes their proof links, but does not itself verify
the producer or link contents.
94 changes: 94 additions & 0 deletions scripts/review-readiness/collect_actions.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
"""Find scoped GitHub Actions failures without claiming tested-revision proof."""

from datetime import datetime
import re

import readiness
from evidence_errors import EvidenceError

ACTIONS_APP_ID = 15368
FAILURES = {'failure', 'timed_out', 'action_required', 'startup_failure'}


def selectors(policy):
rules = policy.get('actions_checks', [])
if not isinstance(rules, list) or len(rules) > 20:
raise ValueError('actions_checks must be a list of at most 20 checks')
required = {item['name'] for item in policy['required_checks']}
result = {}
for item in rules:
if not isinstance(item, dict):
raise ValueError('invalid actions check')
name, path = item.get('name'), item.get('workflow_path')
if (not isinstance(name, str) or name not in required or name in result
or not isinstance(path, str)
or not re.fullmatch(r'\.github/workflows/[A-Za-z0-9_-]+\.ya?ml', path)):
raise ValueError('actions check needs a required name and a workflow path')
result[name] = path
return result


def run_matches(run, snapshot, path, attempt):
if (not isinstance(run, dict) or run.get('event') != 'pull_request'
or run.get('path') != path or run.get('head_sha') != snapshot['head_sha']
or run.get('run_attempt') != attempt):
return False
prs = run.get('pull_requests')
if not isinstance(prs, list):
return False
return any(isinstance(pr, dict) and pr.get('number') == snapshot['pr']
and isinstance(pr.get('base'), dict) and pr['base'].get('sha') == snapshot['base_sha']
and isinstance(pr.get('head'), dict) and pr['head'].get('sha') == snapshot['head_sha']
for pr in prs)


def collect(snapshot, rules, client):
if not rules:
return [], 'not_requested', []
checks = client.list_check_runs(snapshot['head_sha'])
if not isinstance(checks, list):
raise ValueError('invalid checks')
signals = []
matched = []
partial = False
scope = {key: snapshot[key] for key in readiness.SCOPE_KEYS}
for name, workflow_path in rules.items():
relevant = [c for c in checks if isinstance(c, dict) and c.get('name') == name
and isinstance(c.get('app'), dict) and c['app'].get('id') == ACTIONS_APP_ID]
if len(relevant) > 100:
partial = True
continue
candidates = []
try:
for check in relevant:
check_id = check.get('id')
if type(check_id) is not int or check_id <= 0 or check.get('head_sha') != snapshot['head_sha']:
raise ValueError('invalid check')
started = datetime.fromisoformat(check['started_at'].replace('Z', '+00:00'))
if started.utcoffset() is None:
raise ValueError('invalid check time')
job = client.job(check_id)
if (not isinstance(job, dict) or job.get('id') != check_id or job.get('name') != name
or job.get('head_sha') != snapshot['head_sha']
or job.get('status') != check.get('status')
or job.get('conclusion') != check.get('conclusion')
or type(job.get('run_id')) is not int or type(job.get('run_attempt')) is not int):
raise ValueError('inconsistent job')
run = client.run(job['run_id'])
if not isinstance(run, dict) or run.get('id') != job['run_id']:
raise ValueError('inconsistent run')
if run_matches(run, snapshot, workflow_path, job['run_attempt']):
candidates.append((started, check, job['run_id']))
candidates.sort(key=lambda item: item[0], reverse=True)
if len(candidates) > 1 and candidates[0][0] == candidates[1][0]:
raise ValueError('ambiguous latest check')
if candidates:
matched.append(name)
_, latest, run_id = candidates[0]
if latest['status'] == 'completed' and latest['conclusion'] in FAILURES:
proof = f"https://github.com/{snapshot['repo']}/actions/runs/{run_id}/job/{latest['id']}"
signals.append({'scope': scope, 'name': name, 'proof': proof})
except (EvidenceError, ValueError, TypeError, KeyError, AttributeError):
# A missing or malformed page/job/run cannot make an older job current.
partial = True
return signals, 'partial' if partial else 'observed', matched
43 changes: 35 additions & 8 deletions scripts/review-readiness/collect_reviews.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
import sys

import readiness
import collect_actions
from evidence_errors import EvidenceError

THREAD_QUERY = '''
query($owner: String!, $name: String!, $pr: Int!, $cursor: String) {
Expand All @@ -22,10 +24,6 @@
'''


class EvidenceError(Exception):
pass


class GitHub:
def __init__(self, repo, pr):
self.owner, self.name = repo.split('/')
Expand All @@ -49,6 +47,28 @@ def list_reviews(self):
raise EvidenceError('invalid review pages')
return [review for page in pages for review in page]

def list_check_runs(self, head):
path = f'repos/{self.owner}/{self.name}/commits/{head}/check-runs?per_page=100&filter=all'
pages = self._api(['--paginate', '--slurp', path])
if not isinstance(pages, list) or not pages:
raise EvidenceError('invalid check pages')
checks = []
totals = set()
for page in pages:
if not isinstance(page, dict) or type(page.get('total_count')) is not int or not isinstance(page.get('check_runs'), list):
raise EvidenceError('invalid check pages')
totals.add(page['total_count'])
checks.extend(page['check_runs'])
if len(totals) != 1 or len(checks) != next(iter(totals)):
raise EvidenceError('incomplete check pages')
return checks

def job(self, job_id):
return self._api([f'repos/{self.owner}/{self.name}/actions/jobs/{job_id}'])

def run(self, run_id):
return self._api([f'repos/{self.owner}/{self.name}/actions/runs/{run_id}'])

def threads(self, cursor):
args = ['graphql', '-f', f'query={THREAD_QUERY}', '-f', f'owner={self.owner}',
'-f', f'name={self.name}', '-F', f'pr={self.pr}']
Expand Down Expand Up @@ -128,10 +148,12 @@ def collect(data, client):
base = {'snapshot': data.get('snapshot'), 'policy': data.get('policy'),
'checks_complete': False, 'checks': [], 'reviews': None, 'findings': None}
readiness.evaluate(base)
rules = collect_actions.selectors(base['policy'])
snapshot = base['snapshot']
scope = {key: snapshot[key] for key in readiness.SCOPE_KEYS}
proof = f"https://github.com/{snapshot['repo']}/pull/{snapshot['pr']}"
status = {'snapshot': 'unavailable', 'reviews': 'not_attempted', 'threads': 'not_attempted'}
matched_checks = []
status = {'snapshot': 'unavailable', 'reviews': 'not_attempted', 'threads': 'not_attempted', 'actions_checks': 'not_attempted'}
try:
if matches(client.pull(), snapshot):
status['snapshot'] = 'matched'
Expand All @@ -148,19 +170,24 @@ def collect(data, client):
status['threads'] = 'observed'
except EvidenceError:
status['threads'] = 'partial'
try:
base['reported_failures'], status['actions_checks'], matched_checks = collect_actions.collect(snapshot, rules, client)
except (EvidenceError, ValueError, TypeError):
status['actions_checks'] = 'unavailable'
if not matches(client.pull(), snapshot):
status['snapshot'] = 'changed'
base['reviews'], base['findings'] = None, None
base['reviews'], base['findings'], base['reported_failures'] = None, None, []
else:
status['snapshot'] = 'changed'
except EvidenceError:
status['snapshot'] = 'unavailable'
base['reviews'], base['findings'] = None, None
base['reviews'], base['findings'], base['reported_failures'] = None, None, []
if status['snapshot'] != 'matched':
for key in ('reviews', 'threads'):
for key in ('reviews', 'threads', 'actions_checks'):
if status[key] != 'not_attempted':
status[key] = 'discarded'
result = readiness.evaluate(base)
result['actions_checks_matched'] = matched_checks if status['snapshot'] == 'matched' else []
result['collection'] = status
return result

Expand Down
2 changes: 2 additions & 0 deletions scripts/review-readiness/evidence_errors.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
class EvidenceError(Exception):
"""A remote evidence source could not be read reliably."""
11 changes: 11 additions & 0 deletions scripts/review-readiness/readiness.py
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,17 @@ def evaluate(data):
if _proof(proof):
proofs.add(proof)
results[name].append((result, _proof(proof)))
# A reported failure can block, but does not prove which revision ran and
# cannot satisfy a required successful check.
for item in _list(data.get("reported_failures", []), "reported_failures"):
item = _object(item, "reported failure")
if _scope(item.get("scope")) != scope:
continue
name = _text(item.get("name"), "reported failure.name")
if name in required and _proof(item.get("proof")):
blocked.append(f"required check reported failure: {name}")
proofs.add(item["proof"])

for name, found in results.items():
if any(result in ("failure", "error") for result, _ in found):
blocked.append(f"required check failed: {name}")
Expand Down
2 changes: 2 additions & 0 deletions scripts/review-readiness/shadow_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import tempfile

import collect_reviews
import collect_actions
import from_risk
import readiness

Expand Down Expand Up @@ -54,6 +55,7 @@ def build(data, client, run_risk=grade, expected_repo=None):
# Validate the snapshot and policy before sending anything to GitHub.
readiness.evaluate({'snapshot': data.get('snapshot'), 'policy': data.get('policy'),
'checks_complete': False, 'checks': []})
collect_actions.selectors(data['policy'])
snapshot = data['snapshot']
if expected_repo is not None and snapshot['repo'].casefold() != expected_repo.casefold():
raise ValueError('target repository does not match caller')
Expand Down
Loading
Loading