diff --git a/scripts/review-readiness/README.md b/scripts/review-readiness/README.md index daed8a19..874f3e00 100644 --- a/scripts/review-readiness/README.md +++ b/scripts/review-readiness/README.md @@ -61,3 +61,31 @@ not run automatically in a workflow or publish a check. ```sh python3 scripts/review-readiness/from_risk.py < report-envelope.json ``` + +## GitHub review inventory + +`collect_reviews.py` accepts `snapshot` and `policy` on standard input and uses +`gh api` to read the PR, its paginated reviews and its paginated review threads. +It checks that the PR is open and its base, head and author match before and +again after collection. A changed or unavailable snapshot discards the collected +review and thread evidence. `collection` reports which reads were observed, +partial, unavailable, or discarded. + +Only the latest submitted approval, dismissal, or change request per actor can +count, and only when its review records the supplied head SHA. GitHub `User` and +`Bot` identities are distinguished; unknown or malformed identities make review +collection unavailable. Comments and unsubmitted reviews do not replace a +submitted decision. Current unresolved, non-outdated review threads block even +if a later thread page fails. A partial thread read is not a complete inventory. + +The collector never marks the finding inventory complete: GitHub review threads +are only one potential source. It also does not collect checks, verify reviewer +permissions or branch protection, prove a tested revision, publish a report, or +verify that the caller's policy is authorized. Consequently this entry point can +return `BLOCKED` or `WAITING`, but never `READY`. Separate GitHub reads are not +atomic: review state can change during collection, and an ABA branch change can +escape the recheck. It is advisory evidence, not a merge gate. + +```sh +python3 scripts/review-readiness/collect_reviews.py < snapshot-and-policy.json +``` diff --git a/scripts/review-readiness/collect_reviews.py b/scripts/review-readiness/collect_reviews.py new file mode 100644 index 00000000..0e093779 --- /dev/null +++ b/scripts/review-readiness/collect_reviews.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +"""Collect advisory GitHub review evidence for a caller-supplied snapshot.""" + +from datetime import datetime +import json +import subprocess +import sys + +import readiness + +THREAD_QUERY = ''' +query($owner: String!, $name: String!, $pr: Int!, $cursor: String) { + repository(owner: $owner, name: $name) { + pullRequest(number: $pr) { + reviewThreads(first: 100, after: $cursor) { + pageInfo { hasNextPage endCursor } + nodes { isResolved isOutdated } + } + } + } +} +''' + + +class EvidenceError(Exception): + pass + + +class GitHub: + def __init__(self, repo, pr): + self.owner, self.name = repo.split('/') + self.path = f'repos/{repo}/pulls/{pr}' + self.pr = pr + + def _api(self, args): + try: + result = subprocess.run(['gh', 'api', *args], capture_output=True, + text=True, timeout=120, check=True) + return json.loads(result.stdout) + except (subprocess.SubprocessError, OSError, ValueError) as exc: + raise EvidenceError('GitHub request failed') from exc + + def pull(self): + return self._api([self.path]) + + def list_reviews(self): + pages = self._api(['--paginate', '--slurp', f'{self.path}/reviews?per_page=100']) + if not isinstance(pages, list) or not pages or any(not isinstance(page, list) for page in pages): + raise EvidenceError('invalid review pages') + return [review for page in pages for review in page] + + def threads(self, cursor): + args = ['graphql', '-f', f'query={THREAD_QUERY}', '-f', f'owner={self.owner}', + '-f', f'name={self.name}', '-F', f'pr={self.pr}'] + args += ['-F', 'cursor=null'] if cursor is None else ['-f', f'cursor={cursor}'] + try: + return self._api(args)['data']['repository']['pullRequest']['reviewThreads'] + except (KeyError, TypeError) as exc: + raise EvidenceError('invalid thread response') from exc + + +def matches(pull, snapshot): + try: + return (pull['state'] == 'open' and pull['base']['sha'] == snapshot['base_sha'] + and pull['head']['sha'] == snapshot['head_sha'] + and pull['user']['login'].casefold() == snapshot['author'].casefold()) + except (KeyError, TypeError, AttributeError): + return False + + +def approvals(reviews, head): + if not isinstance(reviews, list): + raise EvidenceError('invalid reviews') + latest = {} + for item in reviews: + try: + state = item['state'] + if state in ('COMMENTED', 'PENDING'): + continue + if state not in ('APPROVED', 'CHANGES_REQUESTED', 'DISMISSED'): + raise EvidenceError('unknown review state') + user = item['user'] + login, kind = user['login'], user['type'] + timestamp = datetime.fromisoformat(item['submitted_at'].replace('Z', '+00:00')) + review_id = item['id'] + if (not isinstance(login, str) or not login.strip() or kind not in ('User', 'Bot') + or timestamp.utcoffset() is None or type(review_id) is not int + or not isinstance(item['commit_id'], str)): + raise EvidenceError('invalid reviewer identity') + key = login.casefold() + order = (timestamp, review_id) + if key not in latest or order > latest[key][0]: + latest[key] = (order, item) + except (KeyError, TypeError, AttributeError, ValueError) as exc: + raise EvidenceError('invalid review') from exc + return [{'actor': item['user']['login'], 'human': item['user']['type'] == 'User', + 'state': item['state'].lower()} + for _, item in latest.values() if item['commit_id'] == head] + + +def thread_findings(client): + cursor = None + seen = set() + for _ in range(100): + page = client.threads(cursor) + try: + nodes, info = page['nodes'], page['pageInfo'] + if not isinstance(nodes, list) or type(info['hasNextPage']) is not bool: + raise EvidenceError('invalid thread page') + for node in nodes: + if type(node['isResolved']) is not bool or type(node['isOutdated']) is not bool: + raise EvidenceError('invalid thread state') + if not node['isOutdated']: + yield {'state': 'resolved' if node['isResolved'] else 'open'} + if not info['hasNextPage']: + return + cursor = info['endCursor'] + if not isinstance(cursor, str) or not cursor or cursor in seen: + raise EvidenceError('invalid thread cursor') + seen.add(cursor) + except (KeyError, TypeError) as exc: + raise EvidenceError('invalid thread page') from exc + raise EvidenceError('thread page limit reached') + + +def collect(data, client): + # Validate the complete input contract before any network call. + base = {'snapshot': data.get('snapshot'), 'policy': data.get('policy'), + 'checks_complete': False, 'checks': [], 'reviews': None, 'findings': None} + readiness.evaluate(base) + snapshot = base['snapshot'] + scope = {key: snapshot[key] for key in readiness.SCOPE_KEYS} + proof = f"https://github.com/{snapshot['repo']}/pull/{snapshot['pr']}" + status = {'snapshot': 'unavailable', 'reviews': 'not_attempted', 'threads': 'not_attempted'} + try: + if matches(client.pull(), snapshot): + status['snapshot'] = 'matched' + try: + base['reviews'] = {'scope': scope, 'complete': True, 'proof': proof, + 'approvals': approvals(client.list_reviews(), snapshot['head_sha'])} + status['reviews'] = 'observed' + except EvidenceError: + status['reviews'] = 'unavailable' + base['findings'] = {'scope': scope, 'complete': False, 'proof': proof, 'items': []} + try: + for item in thread_findings(client): + base['findings']['items'].append(item) + status['threads'] = 'observed' + except EvidenceError: + status['threads'] = 'partial' + if not matches(client.pull(), snapshot): + status['snapshot'] = 'changed' + base['reviews'], base['findings'] = None, None + else: + status['snapshot'] = 'changed' + except EvidenceError: + status['snapshot'] = 'unavailable' + base['reviews'], base['findings'] = None, None + if status['snapshot'] != 'matched': + for key in ('reviews', 'threads'): + if status[key] != 'not_attempted': + status[key] = 'discarded' + result = readiness.evaluate(base) + result['collection'] = status + return result + + +def main(): + try: + data = json.load(sys.stdin) + # Evaluate before constructing a client, so repo and PR are validated. + readiness.evaluate({'snapshot': data.get('snapshot'), 'policy': data.get('policy'), + 'checks_complete': False, 'checks': []}) + snapshot = data['snapshot'] + result = collect(data, GitHub(snapshot['repo'], snapshot['pr'])) + except (ValueError, TypeError, AttributeError, json.JSONDecodeError) as exc: + print(f'invalid readiness input: {exc}', file=sys.stderr) + return 2 + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/scripts/review-readiness/tests/test_collect_reviews.py b/scripts/review-readiness/tests/test_collect_reviews.py new file mode 100644 index 00000000..5d18db9c --- /dev/null +++ b/scripts/review-readiness/tests/test_collect_reviews.py @@ -0,0 +1,191 @@ +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +MODULE_DIR = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(MODULE_DIR)) +spec = importlib.util.spec_from_file_location('collect_reviews', MODULE_DIR / 'collect_reviews.py') +collector = importlib.util.module_from_spec(spec) +spec.loader.exec_module(collector) + +BASE, HEAD, TESTED = 'a' * 40, 'b' * 40, 'c' * 40 +SNAPSHOT = {'repo': 'test/repo', 'pr': 42, 'base_sha': BASE, 'head_sha': HEAD, + 'tested_sha': TESTED, 'policy_version': 'v1', 'author': 'author'} +POLICY = {'version': 'v1', 'required_checks': [{'name': 'unit', 'revision': 'tested'}], + 'required_human_approvals': 1} + + +def review(actor='reviewer', state='APPROVED', sha=HEAD, number=1, kind='User'): + return {'id': number, 'user': {'login': actor, 'type': kind}, 'state': state, + 'commit_id': sha, 'submitted_at': f'2026-01-01T00:00:{number:02d}Z'} + + +class FakeClient: + def __init__(self): + self.pr = {'state': 'open', 'user': {'login': 'author'}, + 'base': {'sha': BASE}, 'head': {'sha': HEAD}} + self.last = self.pr + self.reviews = [review()] + self.pages = [{'nodes': [], 'pageInfo': {'hasNextPage': False, 'endCursor': None}}] + self.calls = [] + + def pull(self): + self.calls.append('pull') + return self.pr if self.calls.count('pull') == 1 else self.last + + def list_reviews(self): + self.calls.append('reviews') + return self.reviews + + def threads(self, cursor): + self.calls.append(('threads', cursor)) + return self.pages[0 if cursor is None else 1] + + +def run(client): + return collector.collect({'snapshot': SNAPSHOT, 'policy': POLICY}, client) + + +class CollectReviewsTests(unittest.TestCase): + def test_current_human_approval_is_collected_but_other_sources_remain_incomplete(self): + result = run(FakeClient()) + self.assertEqual(result['state'], 'WAITING') + self.assertNotIn('required human approvals incomplete', result['reasons']) + self.assertIn('finding evidence incomplete', result['reasons']) + self.assertIn('check inventory incomplete', result['reasons']) + + def test_paginated_unresolved_current_thread_blocks(self): + client = FakeClient() + client.pages = [ + {'nodes': [], 'pageInfo': {'hasNextPage': True, 'endCursor': 'next'}}, + {'nodes': [{'isResolved': False, 'isOutdated': False}], + 'pageInfo': {'hasNextPage': False, 'endCursor': None}}, + ] + result = run(client) + self.assertEqual(result['state'], 'BLOCKED') + self.assertIn(('threads', 'next'), client.calls) + + def test_outdated_or_resolved_threads_do_not_block(self): + client = FakeClient() + client.pages[0]['nodes'] = [{'isResolved': False, 'isOutdated': True}, + {'isResolved': True, 'isOutdated': False}] + self.assertEqual(run(client)['state'], 'WAITING') + + def test_dismissed_or_old_head_approval_does_not_count(self): + for reviews in ([review(sha='d' * 40)], [review(), review(state='DISMISSED', number=2)]): + client = FakeClient() + client.reviews = reviews + self.assertIn('required human approvals incomplete', run(client)['reasons']) + + def test_latest_review_state_and_identity(self): + client = FakeClient() + client.reviews = [review(state='CHANGES_REQUESTED'), review(number=2), + review(actor='robot', kind='Bot', number=3)] + self.assertEqual(run(client)['state'], 'WAITING') + client.reviews = [review(), review(state='CHANGES_REQUESTED', number=2)] + self.assertEqual(run(client)['state'], 'BLOCKED') + + def test_snapshot_change_discards_all_collected_evidence(self): + client = FakeClient() + client.pages[0]['nodes'] = [{'isResolved': False, 'isOutdated': False}] + client.last = {**client.pr, 'head': {'sha': 'd' * 40}} + result = run(client) + self.assertEqual(result['state'], 'WAITING') + self.assertIn('review evidence missing', result['reasons']) + self.assertIn('finding evidence missing', result['reasons']) + self.assertEqual(result['collection']['snapshot'], 'changed') + + def test_failed_or_looping_pagination_cannot_claim_complete_inventory(self): + client = FakeClient() + client.pages = [{'nodes': [], 'pageInfo': {'hasNextPage': True, 'endCursor': 'next'}}, + {'nodes': [], 'pageInfo': {'hasNextPage': True, 'endCursor': 'next'}}] + result = run(client) + self.assertEqual(result['state'], 'WAITING') + self.assertIn('finding evidence incomplete', result['reasons']) + self.assertEqual(result['collection']['threads'], 'partial') + + def test_known_open_thread_still_blocks_when_later_page_fails(self): + client = FakeClient() + client.pages = [ + {'nodes': [{'isResolved': False, 'isOutdated': False}], + 'pageInfo': {'hasNextPage': True, 'endCursor': 'next'}}, + {'nodes': [], 'pageInfo': {'hasNextPage': True, 'endCursor': 'next'}}, + ] + self.assertEqual(run(client)['state'], 'BLOCKED') + + def test_cli_uses_paginated_github_responses(self): + with tempfile.TemporaryDirectory() as directory: + gh = Path(directory) / 'gh' + fixture = {'pull': FakeClient().pr, 'reviews': [[review()]], + 'pages': [{'nodes': [], 'pageInfo': {'hasNextPage': True, 'endCursor': 'next'}}, + {'nodes': [{'isResolved': False, 'isOutdated': False}], + 'pageInfo': {'hasNextPage': False, 'endCursor': None}}]} + fixture_path = Path(directory) / 'fixture.json' + fixture_path.write_text(json.dumps(fixture)) + gh.write_text("""#!/usr/bin/env python3 +import json, os, sys +f = json.load(open(os.environ['FIXTURE_PATH'])) +if 'graphql' in sys.argv: + page = f['pages'][int('cursor=next' in sys.argv)] + value = {'data': {'repository': {'pullRequest': {'reviewThreads': page}}}} +elif '--slurp' in sys.argv: + value = f['reviews'] +else: + value = f['pull'] +print(json.dumps(value)) +""") + gh.chmod(0o755) + env = {**os.environ, 'PATH': directory + os.pathsep + os.environ['PATH'], + 'FIXTURE_PATH': str(fixture_path)} + proc = subprocess.run([sys.executable, str(MODULE_DIR / 'collect_reviews.py')], + input=json.dumps({'snapshot': SNAPSHOT, 'policy': POLICY}), + text=True, capture_output=True, env=env, check=True) + result = json.loads(proc.stdout) + self.assertEqual(result['state'], 'BLOCKED') + self.assertIn('unresolved finding', result['reasons']) + self.assertNotIn('required human approvals incomplete', result['reasons']) + + def test_unbounded_thread_pagination_is_marked_partial(self): + class Endless(FakeClient): + def threads(self, cursor): + self.calls.append(('threads', cursor)) + return {'nodes': [], 'pageInfo': {'hasNextPage': True, 'endCursor': str(len(self.calls))}} + client = Endless() + result = run(client) + self.assertEqual(result['collection']['threads'], 'partial') + self.assertLessEqual(len([call for call in client.calls if isinstance(call, tuple)]), 100) + + def test_unavailable_reviews_do_not_hide_known_open_threads(self): + class UnavailableReviews(FakeClient): + def list_reviews(self): + raise collector.EvidenceError('unavailable') + client = UnavailableReviews() + client.pages[0]['nodes'] = [{'isResolved': False, 'isOutdated': False}] + result = run(client) + self.assertEqual(result['state'], 'BLOCKED') + self.assertEqual(result['collection']['reviews'], 'unavailable') + self.assertIn('review evidence missing', result['reasons']) + + def test_failed_final_snapshot_discards_evidence(self): + class UnavailableFinal(FakeClient): + def pull(self): + if self.calls.count('pull'): + raise collector.EvidenceError('unavailable') + return super().pull() + result = run(UnavailableFinal()) + self.assertEqual(result['collection']['snapshot'], 'unavailable') + self.assertIn('review evidence missing', result['reasons']) + + def test_invalid_api_identity_is_not_a_human_approval(self): + client = FakeClient() + client.reviews = [review(kind='Unknown')] + self.assertIn('review evidence missing', run(client)['reasons']) + + +if __name__ == '__main__': + unittest.main()