Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions docs/callers/review-readiness-shadow.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,9 @@ in the caller's `uses:`. The provider tools load at the workflow's own revision,
never from the PR. This workflow runs only when explicitly called.

The report records the supplied `tested_sha` but cannot prove that tests actually
ran on it. It does not verify reviewer permissions or cover every finding source, so
review and finding evidence remains incomplete.
ran on it. It checks current repository base permissions for human approvals, but does not
verify code-owner rules or cover every finding source. Missing permission data
keeps approval evidence incomplete.
Missing checks and incomplete findings prevent `READY`; known current failures
can still produce `BLOCKED`. Data reads are not atomic, so a review change or an
ABA revision change can race collection. It is not an authorization to merge or
Expand Down
17 changes: 11 additions & 6 deletions scripts/review-readiness/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -74,17 +74,22 @@ partial, unavailable, or discarded.
Only the latest submitted approval, dismissal, or change request per actor can
count, and only when its review records the supplied head SHA. GitHub `User` and
`Bot` identities are distinguished; unknown or malformed identities make review
collection unavailable. Review records are reported as incomplete even when the
list was read successfully, because the collector does not check reviewer
permissions. Comments and unsubmitted reviews do not replace a submitted decision. Current unresolved, non-outdated review threads block even
collection unavailable. For each current human non-author approval, the collector
reads the reviewer's current repository permission and counts only write,
maintain or admin base access. Missing, malformed or unavailable permission
responses make approval evidence incomplete; a successfully read lower role does
not count. It checks at most 100 approvals and reports `reviewer_permissions` as
observed, partial, not needed, not attempted or discarded. Current change requests
still block even if a permission read fails. Comments and unsubmitted reviews do
not replace a submitted decision. Current unresolved, non-outdated review threads block even
if a later thread page fails. A partial thread read is not a complete inventory.

The collector never marks the finding inventory complete: GitHub review threads
are only one potential source. It also does not collect checks, verify reviewer
permissions or branch protection, prove a tested revision, publish a report, or
are only one potential source. It also does not verify branch protection or code-owner requirements, prove a
tested revision, publish a report, or
verify that the caller's policy is authorized. Consequently this entry point can
return `BLOCKED` or `WAITING`, but never `READY`. Separate GitHub reads are not
atomic: review state can change during collection, and an ABA branch change can
atomic: reviews or permissions can change during collection, and an ABA branch change can
escape the recheck. It is advisory evidence, not a merge gate.

```sh
Expand Down
47 changes: 43 additions & 4 deletions scripts/review-readiness/collect_reviews.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

from datetime import datetime
import json
import re
import subprocess
import sys

Expand Down Expand Up @@ -47,6 +48,11 @@ def list_reviews(self):
raise EvidenceError('invalid review pages')
return [review for page in pages for review in page]

def permission(self, login):
if not isinstance(login, str) or not re.fullmatch(r'[A-Za-z0-9-]{1,39}', login):
raise EvidenceError('invalid reviewer login')
return self._api([f'repos/{self.owner}/{self.name}/collaborators/{login}/permission'])

def list_check_runs(self, head):
path = f'repos/{self.owner}/{self.name}/commits/{head}/check-runs?per_page=100&filter=all'
pages = self._api(['--paginate', '--slurp', path])
Expand Down Expand Up @@ -118,6 +124,37 @@ def approvals(reviews, head):
for _, item in latest.values() if item['commit_id'] == head]


def eligible_approvals(items, author, client):
"""Filter approvals by current base permission, keeping blockers visible."""
eligible = []
complete = True
candidates = [item for item in items if item['human'] and item['state'] == 'approved'
and item['actor'].casefold() != author.casefold()]
status = 'observed' if candidates else 'not_needed'
if len(candidates) > 100:
candidates = []
complete = False
status = 'partial'
eligible.extend(item for item in items if item['state'] != 'approved')
for item in candidates:
try:
data = client.permission(item['actor'])
user = data['user']
role = data['permission']
if (not isinstance(user, dict) or user.get('type') != 'User'
or not isinstance(user.get('login'), str)
or user['login'].casefold() != item['actor'].casefold()
or role not in ('admin', 'write', 'maintain', 'read', 'triage', 'none')):
raise EvidenceError('invalid reviewer permission')
if role in ('admin', 'write', 'maintain'):
eligible.append(item)
except (EvidenceError, KeyError, TypeError):
complete = False
status = 'partial'
break
return eligible, complete, status


def thread_findings(client):
cursor = None
seen = set()
Expand Down Expand Up @@ -153,13 +190,15 @@ def collect(data, client):
scope = {key: snapshot[key] for key in readiness.SCOPE_KEYS}
proof = f"https://github.com/{snapshot['repo']}/pull/{snapshot['pr']}"
matched_checks = []
status = {'snapshot': 'unavailable', 'reviews': 'not_attempted', 'threads': 'not_attempted', 'actions_checks': 'not_attempted'}
status = {'snapshot': 'unavailable', 'reviews': 'not_attempted', 'reviewer_permissions': 'not_attempted', 'threads': 'not_attempted', 'actions_checks': 'not_attempted'}
try:
if matches(client.pull(), snapshot):
status['snapshot'] = 'matched'
try:
base['reviews'] = {'scope': scope, 'complete': False, 'proof': proof,
'approvals': approvals(client.list_reviews(), snapshot['head_sha'])}
items = approvals(client.list_reviews(), snapshot['head_sha'])
eligible, complete, status['reviewer_permissions'] = eligible_approvals(items, snapshot['author'], client)
base['reviews'] = {'scope': scope, 'complete': complete, 'proof': proof,
'approvals': eligible}
status['reviews'] = 'observed'
except EvidenceError:
status['reviews'] = 'unavailable'
Expand All @@ -183,7 +222,7 @@ def collect(data, client):
status['snapshot'] = 'unavailable'
base['reviews'], base['findings'], base['reported_failures'] = None, None, []
if status['snapshot'] != 'matched':
for key in ('reviews', 'threads', 'actions_checks'):
for key in ('reviews', 'reviewer_permissions', 'threads', 'actions_checks'):
if status[key] != 'not_attempted':
status[key] = 'discarded'
result = readiness.evaluate(base)
Expand Down
68 changes: 64 additions & 4 deletions scripts/review-readiness/tests/test_collect_reviews.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ def __init__(self):
self.reviews = [review()]
self.pages = [{'nodes': [], 'pageInfo': {'hasNextPage': False, 'endCursor': None}}]
self.calls = []
self.permission_data = {'permission': 'write', 'user': {'login': 'reviewer', 'type': 'User'}}

def pull(self):
self.calls.append('pull')
Expand All @@ -42,6 +43,10 @@ def list_reviews(self):
self.calls.append('reviews')
return self.reviews

def permission(self, login):
self.calls.append(('permission', login))
return self.permission_data

def threads(self, cursor):
self.calls.append(('threads', cursor))
return self.pages[0 if cursor is None else 1]
Expand All @@ -52,13 +57,66 @@ def run(client):


class CollectReviewsTests(unittest.TestCase):
def test_current_human_approval_remains_incomplete_without_permission_check(self):
def test_review_permission_read_is_separate_from_other_incomplete_evidence(self):
result = run(FakeClient())
self.assertEqual(result['state'], 'WAITING')
self.assertIn('required human approvals incomplete', result['reasons'])
self.assertNotIn('required human approvals incomplete', result['reasons'])
self.assertIn('finding evidence incomplete', result['reasons'])
self.assertIn('check inventory incomplete', result['reasons'])

def test_eligible_current_approval_is_recognized_after_permission_check(self):
client = FakeClient()
result = run(client)
self.assertIn(('permission', 'reviewer'), client.calls)
self.assertEqual(result['collection']['reviewer_permissions'], 'observed')
self.assertNotIn('required human approvals incomplete', result['reasons'])
self.assertEqual(result['state'], 'WAITING')

def test_ineligible_or_unavailable_permissions_do_not_count_as_approval(self):
for value in ('read', 'none', 'invalid', None):
client = FakeClient()
client.permission_data['permission'] = value
result = run(client)
self.assertIn('required human approvals incomplete', result['reasons'], value)

def test_permission_failure_stops_further_lookups(self):
client = FakeClient()
calls = []
def unavailable(login):
calls.append(login)
raise collector.EvidenceError('unavailable')
client.permission = unavailable
client.reviews = [review(), review(actor='other', number=2)]
result = run(client)
self.assertEqual(calls, ['reviewer'])
self.assertEqual(result['collection']['reviewer_permissions'], 'partial')
self.assertIn('required human approvals incomplete', result['reasons'])

def test_permission_for_a_different_identity_does_not_count(self):
client = FakeClient()
client.permission_data['user']['login'] = 'someone-else'
self.assertIn('required human approvals incomplete', run(client)['reasons'])

def test_permission_queries_are_bounded(self):
client = FakeClient()
items = [{'actor': f'user-{n}', 'human': True, 'state': 'approved'} for n in range(101)]
eligible, complete, status = collector.eligible_approvals(items, 'author', client)
self.assertEqual(eligible, [])
self.assertFalse(complete)
self.assertEqual(status, 'partial')
self.assertFalse(any(isinstance(call, tuple) and call[0] == 'permission' for call in client.calls))

def test_permission_failure_does_not_hide_change_requests(self):
client = FakeClient()
client.reviews = [review(), review(actor='other', state='CHANGES_REQUESTED', number=2)]
def unavailable(login): raise collector.EvidenceError('unavailable')
client.permission = unavailable
result = run(client)
self.assertEqual(result['state'], 'BLOCKED')
self.assertIn('human review requests changes', result['reasons'])
self.assertEqual(result['collection']['reviewer_permissions'], 'partial')
self.assertIn('required human approvals incomplete', result['reasons'])

def test_paginated_unresolved_current_thread_blocks(self):
client = FakeClient()
client.pages = [
Expand Down Expand Up @@ -133,6 +191,8 @@ def test_cli_uses_paginated_github_responses(self):
if 'graphql' in sys.argv:
page = f['pages'][int('cursor=next' in sys.argv)]
value = {'data': {'repository': {'pullRequest': {'reviewThreads': page}}}}
elif any('/collaborators/' in arg for arg in sys.argv):
value = {'permission': 'write', 'user': {'login': 'reviewer', 'type': 'User'}}
elif '--slurp' in sys.argv:
value = f['reviews']
else:
Expand All @@ -148,7 +208,7 @@ def test_cli_uses_paginated_github_responses(self):
result = json.loads(proc.stdout)
self.assertEqual(result['state'], 'BLOCKED')
self.assertIn('unresolved finding', result['reasons'])
self.assertIn('required human approvals incomplete', result['reasons'])
self.assertNotIn('required human approvals incomplete', result['reasons'])

def test_unbounded_thread_pagination_is_marked_partial(self):
class Endless(FakeClient):
Expand All @@ -158,7 +218,7 @@ def threads(self, cursor):
client = Endless()
result = run(client)
self.assertEqual(result['collection']['threads'], 'partial')
self.assertLessEqual(len([call for call in client.calls if isinstance(call, tuple)]), 100)
self.assertLessEqual(len([call for call in client.calls if isinstance(call, tuple) and call[0] == 'threads']), 100)

def test_unavailable_reviews_do_not_hide_known_open_threads(self):
class UnavailableReviews(FakeClient):
Expand Down
Loading