From 41a6d43ee2ee16443564b9e649c1168fdfed6a32 Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:24:47 +0800 Subject: [PATCH] fix: require evidence of human review identity --- docs/callers/review-readiness-shadow.md | 7 ++++--- scripts/review-readiness/README.md | 12 +++++++----- scripts/review-readiness/collect_reviews.py | 3 ++- .../tests/test_collect_reviews.py | 16 ++++++++++++---- 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/docs/callers/review-readiness-shadow.md b/docs/callers/review-readiness-shadow.md index 428052a9..6fb9c91b 100644 --- a/docs/callers/review-readiness-shadow.md +++ b/docs/callers/review-readiness-shadow.md @@ -15,9 +15,10 @@ in the caller's `uses:`. The provider tools load at the workflow's own revision, never from the PR. This workflow runs only when explicitly called. The report records the supplied `tested_sha` but cannot prove that tests actually -ran on it. It checks current repository base permissions for human approvals, but does not -verify code-owner rules or cover every finding source. Missing permission data -keeps approval evidence incomplete. +ran on it. It checks repository base permissions for User approvals, but cannot +establish whether those accounts belong to humans. It does not count them as +verified human approvals, verify code-owner rules, or cover every finding source. +Missing permission data also keeps approval evidence incomplete. Missing checks and incomplete findings prevent `READY`; known current failures can still produce `BLOCKED`. Data reads are not atomic, so a review change or an ABA revision change can race collection. It is not an authorization to merge or diff --git a/scripts/review-readiness/README.md b/scripts/review-readiness/README.md index 50a9d981..904bb1c2 100644 --- a/scripts/review-readiness/README.md +++ b/scripts/review-readiness/README.md @@ -74,11 +74,13 @@ partial, unavailable, or discarded. Only the latest submitted approval, dismissal, or change request per actor can count, and only when its review records the supplied head SHA. GitHub `User` and `Bot` identities are distinguished; unknown or malformed identities make review -collection unavailable. For each current human non-author approval, the collector -reads the reviewer's current repository permission and counts only write, -maintain or admin base access. Missing, malformed or unavailable permission -responses make approval evidence incomplete; a successfully read lower role does -not count. It checks at most 100 approvals and reports `reviewer_permissions` as +collection unavailable. A GitHub `User` may be a machine account, so its type and +repository permission do not establish a human identity. The collector checks the +current base permission of current non-author `User` approvals, but never counts +them as verified human approvals: write, maintain, or admin access leaves human +identity incomplete, and lower roles do not qualify. Missing, malformed or +unavailable permission responses also make approval evidence incomplete. It checks +at most 100 approvals and reports `reviewer_permissions` as observed, partial, not needed, not attempted or discarded. Current change requests still block even if a permission read fails. Comments and unsubmitted reviews do not replace a submitted decision. Current unresolved, non-outdated review threads block even diff --git a/scripts/review-readiness/collect_reviews.py b/scripts/review-readiness/collect_reviews.py index 123fa4e1..4078739d 100644 --- a/scripts/review-readiness/collect_reviews.py +++ b/scripts/review-readiness/collect_reviews.py @@ -147,7 +147,8 @@ def eligible_approvals(items, author, client): or role not in ('admin', 'write', 'maintain', 'read', 'triage', 'none')): raise EvidenceError('invalid reviewer permission') if role in ('admin', 'write', 'maintain'): - eligible.append(item) + # GitHub User accounts can also be automation; role is not proof of a human. + complete = False except (EvidenceError, KeyError, TypeError): complete = False status = 'partial' diff --git a/scripts/review-readiness/tests/test_collect_reviews.py b/scripts/review-readiness/tests/test_collect_reviews.py index 3e1cf729..7ab1aed4 100644 --- a/scripts/review-readiness/tests/test_collect_reviews.py +++ b/scripts/review-readiness/tests/test_collect_reviews.py @@ -57,19 +57,27 @@ def run(client): class CollectReviewsTests(unittest.TestCase): + def test_github_user_type_and_write_permission_do_not_prove_human_identity(self): + client = FakeClient() + client.reviews = [review(actor='service-account')] + client.permission_data['user']['login'] = 'service-account' + result = run(client) + self.assertEqual(result['collection']['reviewer_permissions'], 'observed') + self.assertIn('required human approvals incomplete', result['reasons']) + def test_review_permission_read_is_separate_from_other_incomplete_evidence(self): result = run(FakeClient()) self.assertEqual(result['state'], 'WAITING') - self.assertNotIn('required human approvals incomplete', result['reasons']) + self.assertIn('required human approvals incomplete', result['reasons']) self.assertIn('finding evidence incomplete', result['reasons']) self.assertIn('check inventory incomplete', result['reasons']) - def test_eligible_current_approval_is_recognized_after_permission_check(self): + def test_current_approval_stays_unverified_after_permission_check(self): client = FakeClient() result = run(client) self.assertIn(('permission', 'reviewer'), client.calls) self.assertEqual(result['collection']['reviewer_permissions'], 'observed') - self.assertNotIn('required human approvals incomplete', result['reasons']) + self.assertIn('required human approvals incomplete', result['reasons']) self.assertEqual(result['state'], 'WAITING') def test_ineligible_or_unavailable_permissions_do_not_count_as_approval(self): @@ -208,7 +216,7 @@ def test_cli_uses_paginated_github_responses(self): result = json.loads(proc.stdout) self.assertEqual(result['state'], 'BLOCKED') self.assertIn('unresolved finding', result['reasons']) - self.assertNotIn('required human approvals incomplete', result['reasons']) + self.assertIn('required human approvals incomplete', result['reasons']) def test_unbounded_thread_pagination_is_marked_partial(self): class Endless(FakeClient):