diff --git a/docs/callers/review-readiness-shadow.md b/docs/callers/review-readiness-shadow.md index 7912d58d..428052a9 100644 --- a/docs/callers/review-readiness-shadow.md +++ b/docs/callers/review-readiness-shadow.md @@ -15,8 +15,9 @@ in the caller's `uses:`. The provider tools load at the workflow's own revision, never from the PR. This workflow runs only when explicitly called. The report records the supplied `tested_sha` but cannot prove that tests actually -ran on it. It does not verify reviewer permissions or cover every finding source, so -review and finding evidence remains incomplete. +ran on it. It checks current repository base permissions for human approvals, but does not +verify code-owner rules or cover every finding source. Missing permission data +keeps approval evidence incomplete. Missing checks and incomplete findings prevent `READY`; known current failures can still produce `BLOCKED`. Data reads are not atomic, so a review change or an ABA revision change can race collection. It is not an authorization to merge or diff --git a/scripts/review-readiness/README.md b/scripts/review-readiness/README.md index 02a26444..50a9d981 100644 --- a/scripts/review-readiness/README.md +++ b/scripts/review-readiness/README.md @@ -74,17 +74,22 @@ partial, unavailable, or discarded. Only the latest submitted approval, dismissal, or change request per actor can count, and only when its review records the supplied head SHA. GitHub `User` and `Bot` identities are distinguished; unknown or malformed identities make review -collection unavailable. Review records are reported as incomplete even when the -list was read successfully, because the collector does not check reviewer -permissions. Comments and unsubmitted reviews do not replace a submitted decision. Current unresolved, non-outdated review threads block even +collection unavailable. For each current human non-author approval, the collector +reads the reviewer's current repository permission and counts only write, +maintain or admin base access. Missing, malformed or unavailable permission +responses make approval evidence incomplete; a successfully read lower role does +not count. It checks at most 100 approvals and reports `reviewer_permissions` as +observed, partial, not needed, not attempted or discarded. Current change requests +still block even if a permission read fails. Comments and unsubmitted reviews do +not replace a submitted decision. Current unresolved, non-outdated review threads block even if a later thread page fails. A partial thread read is not a complete inventory. The collector never marks the finding inventory complete: GitHub review threads -are only one potential source. It also does not collect checks, verify reviewer -permissions or branch protection, prove a tested revision, publish a report, or +are only one potential source. It also does not verify branch protection or code-owner requirements, prove a +tested revision, publish a report, or verify that the caller's policy is authorized. Consequently this entry point can return `BLOCKED` or `WAITING`, but never `READY`. Separate GitHub reads are not -atomic: review state can change during collection, and an ABA branch change can +atomic: reviews or permissions can change during collection, and an ABA branch change can escape the recheck. It is advisory evidence, not a merge gate. ```sh diff --git a/scripts/review-readiness/collect_reviews.py b/scripts/review-readiness/collect_reviews.py index 689e59d4..123fa4e1 100644 --- a/scripts/review-readiness/collect_reviews.py +++ b/scripts/review-readiness/collect_reviews.py @@ -3,6 +3,7 @@ from datetime import datetime import json +import re import subprocess import sys @@ -47,6 +48,11 @@ def list_reviews(self): raise EvidenceError('invalid review pages') return [review for page in pages for review in page] + def permission(self, login): + if not isinstance(login, str) or not re.fullmatch(r'[A-Za-z0-9-]{1,39}', login): + raise EvidenceError('invalid reviewer login') + return self._api([f'repos/{self.owner}/{self.name}/collaborators/{login}/permission']) + def list_check_runs(self, head): path = f'repos/{self.owner}/{self.name}/commits/{head}/check-runs?per_page=100&filter=all' pages = self._api(['--paginate', '--slurp', path]) @@ -118,6 +124,37 @@ def approvals(reviews, head): for _, item in latest.values() if item['commit_id'] == head] +def eligible_approvals(items, author, client): + """Filter approvals by current base permission, keeping blockers visible.""" + eligible = [] + complete = True + candidates = [item for item in items if item['human'] and item['state'] == 'approved' + and item['actor'].casefold() != author.casefold()] + status = 'observed' if candidates else 'not_needed' + if len(candidates) > 100: + candidates = [] + complete = False + status = 'partial' + eligible.extend(item for item in items if item['state'] != 'approved') + for item in candidates: + try: + data = client.permission(item['actor']) + user = data['user'] + role = data['permission'] + if (not isinstance(user, dict) or user.get('type') != 'User' + or not isinstance(user.get('login'), str) + or user['login'].casefold() != item['actor'].casefold() + or role not in ('admin', 'write', 'maintain', 'read', 'triage', 'none')): + raise EvidenceError('invalid reviewer permission') + if role in ('admin', 'write', 'maintain'): + eligible.append(item) + except (EvidenceError, KeyError, TypeError): + complete = False + status = 'partial' + break + return eligible, complete, status + + def thread_findings(client): cursor = None seen = set() @@ -153,13 +190,15 @@ def collect(data, client): scope = {key: snapshot[key] for key in readiness.SCOPE_KEYS} proof = f"https://github.com/{snapshot['repo']}/pull/{snapshot['pr']}" matched_checks = [] - status = {'snapshot': 'unavailable', 'reviews': 'not_attempted', 'threads': 'not_attempted', 'actions_checks': 'not_attempted'} + status = {'snapshot': 'unavailable', 'reviews': 'not_attempted', 'reviewer_permissions': 'not_attempted', 'threads': 'not_attempted', 'actions_checks': 'not_attempted'} try: if matches(client.pull(), snapshot): status['snapshot'] = 'matched' try: - base['reviews'] = {'scope': scope, 'complete': False, 'proof': proof, - 'approvals': approvals(client.list_reviews(), snapshot['head_sha'])} + items = approvals(client.list_reviews(), snapshot['head_sha']) + eligible, complete, status['reviewer_permissions'] = eligible_approvals(items, snapshot['author'], client) + base['reviews'] = {'scope': scope, 'complete': complete, 'proof': proof, + 'approvals': eligible} status['reviews'] = 'observed' except EvidenceError: status['reviews'] = 'unavailable' @@ -183,7 +222,7 @@ def collect(data, client): status['snapshot'] = 'unavailable' base['reviews'], base['findings'], base['reported_failures'] = None, None, [] if status['snapshot'] != 'matched': - for key in ('reviews', 'threads', 'actions_checks'): + for key in ('reviews', 'reviewer_permissions', 'threads', 'actions_checks'): if status[key] != 'not_attempted': status[key] = 'discarded' result = readiness.evaluate(base) diff --git a/scripts/review-readiness/tests/test_collect_reviews.py b/scripts/review-readiness/tests/test_collect_reviews.py index e5253bf7..3e1cf729 100644 --- a/scripts/review-readiness/tests/test_collect_reviews.py +++ b/scripts/review-readiness/tests/test_collect_reviews.py @@ -33,6 +33,7 @@ def __init__(self): self.reviews = [review()] self.pages = [{'nodes': [], 'pageInfo': {'hasNextPage': False, 'endCursor': None}}] self.calls = [] + self.permission_data = {'permission': 'write', 'user': {'login': 'reviewer', 'type': 'User'}} def pull(self): self.calls.append('pull') @@ -42,6 +43,10 @@ def list_reviews(self): self.calls.append('reviews') return self.reviews + def permission(self, login): + self.calls.append(('permission', login)) + return self.permission_data + def threads(self, cursor): self.calls.append(('threads', cursor)) return self.pages[0 if cursor is None else 1] @@ -52,13 +57,66 @@ def run(client): class CollectReviewsTests(unittest.TestCase): - def test_current_human_approval_remains_incomplete_without_permission_check(self): + def test_review_permission_read_is_separate_from_other_incomplete_evidence(self): result = run(FakeClient()) self.assertEqual(result['state'], 'WAITING') - self.assertIn('required human approvals incomplete', result['reasons']) + self.assertNotIn('required human approvals incomplete', result['reasons']) self.assertIn('finding evidence incomplete', result['reasons']) self.assertIn('check inventory incomplete', result['reasons']) + def test_eligible_current_approval_is_recognized_after_permission_check(self): + client = FakeClient() + result = run(client) + self.assertIn(('permission', 'reviewer'), client.calls) + self.assertEqual(result['collection']['reviewer_permissions'], 'observed') + self.assertNotIn('required human approvals incomplete', result['reasons']) + self.assertEqual(result['state'], 'WAITING') + + def test_ineligible_or_unavailable_permissions_do_not_count_as_approval(self): + for value in ('read', 'none', 'invalid', None): + client = FakeClient() + client.permission_data['permission'] = value + result = run(client) + self.assertIn('required human approvals incomplete', result['reasons'], value) + + def test_permission_failure_stops_further_lookups(self): + client = FakeClient() + calls = [] + def unavailable(login): + calls.append(login) + raise collector.EvidenceError('unavailable') + client.permission = unavailable + client.reviews = [review(), review(actor='other', number=2)] + result = run(client) + self.assertEqual(calls, ['reviewer']) + self.assertEqual(result['collection']['reviewer_permissions'], 'partial') + self.assertIn('required human approvals incomplete', result['reasons']) + + def test_permission_for_a_different_identity_does_not_count(self): + client = FakeClient() + client.permission_data['user']['login'] = 'someone-else' + self.assertIn('required human approvals incomplete', run(client)['reasons']) + + def test_permission_queries_are_bounded(self): + client = FakeClient() + items = [{'actor': f'user-{n}', 'human': True, 'state': 'approved'} for n in range(101)] + eligible, complete, status = collector.eligible_approvals(items, 'author', client) + self.assertEqual(eligible, []) + self.assertFalse(complete) + self.assertEqual(status, 'partial') + self.assertFalse(any(isinstance(call, tuple) and call[0] == 'permission' for call in client.calls)) + + def test_permission_failure_does_not_hide_change_requests(self): + client = FakeClient() + client.reviews = [review(), review(actor='other', state='CHANGES_REQUESTED', number=2)] + def unavailable(login): raise collector.EvidenceError('unavailable') + client.permission = unavailable + result = run(client) + self.assertEqual(result['state'], 'BLOCKED') + self.assertIn('human review requests changes', result['reasons']) + self.assertEqual(result['collection']['reviewer_permissions'], 'partial') + self.assertIn('required human approvals incomplete', result['reasons']) + def test_paginated_unresolved_current_thread_blocks(self): client = FakeClient() client.pages = [ @@ -133,6 +191,8 @@ def test_cli_uses_paginated_github_responses(self): if 'graphql' in sys.argv: page = f['pages'][int('cursor=next' in sys.argv)] value = {'data': {'repository': {'pullRequest': {'reviewThreads': page}}}} +elif any('/collaborators/' in arg for arg in sys.argv): + value = {'permission': 'write', 'user': {'login': 'reviewer', 'type': 'User'}} elif '--slurp' in sys.argv: value = f['reviews'] else: @@ -148,7 +208,7 @@ def test_cli_uses_paginated_github_responses(self): result = json.loads(proc.stdout) self.assertEqual(result['state'], 'BLOCKED') self.assertIn('unresolved finding', result['reasons']) - self.assertIn('required human approvals incomplete', result['reasons']) + self.assertNotIn('required human approvals incomplete', result['reasons']) def test_unbounded_thread_pagination_is_marked_partial(self): class Endless(FakeClient): @@ -158,7 +218,7 @@ def threads(self, cursor): client = Endless() result = run(client) self.assertEqual(result['collection']['threads'], 'partial') - self.assertLessEqual(len([call for call in client.calls if isinstance(call, tuple)]), 100) + self.assertLessEqual(len([call for call in client.calls if isinstance(call, tuple) and call[0] == 'threads']), 100) def test_unavailable_reviews_do_not_hide_known_open_threads(self): class UnavailableReviews(FakeClient):