From 3abdb256f8fea649e6ae3abe9a4ee2eab4fa0201 Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Wed, 28 Oct 2026 17:53:31 +0800 Subject: [PATCH 1/2] feat: compare adjudicated reviewer outputs --- .github/cursor-review/README.md | 75 ++++++++ .github/cursor-review/reviewer-eval.py | 166 ++++++++++++++++++ .../cursor-review/tests/test_reviewer_eval.py | 131 ++++++++++++++ 3 files changed, 372 insertions(+) create mode 100644 .github/cursor-review/reviewer-eval.py create mode 100644 .github/cursor-review/tests/test_reviewer_eval.py diff --git a/.github/cursor-review/README.md b/.github/cursor-review/README.md index cf9c3715..9f9c1168 100644 --- a/.github/cursor-review/README.md +++ b/.github/cursor-review/README.md @@ -509,3 +509,78 @@ review for everyone is a normal PR to this directory: Consumers pinned to a SHA pick up changes when they bump the SHA; consumers on a floating major tag pick them up on the next run. + +## Offline reviewer evaluation + +`reviewer-eval.py` compares two reviewer versions against the **same** curated, +human-adjudicated bug and clean cases. It reads a JSON object from stdin and +writes a paired comparison to stdout; it does not run a model, publish a review, +or change GitHub. There is no bundled or implied production benchmark. + +Each case identifies `id`, `repo`, `base_sha`, `head_sha`, and a `truth` object +with `kind` (`bug`, `clean`, or `unknown`), `proof`, and `bug_ids`. A confirmed +bug case has one or more unique bug IDs; a confirmed clean case has none. Truth +and all judgments need HTTPS evidence links. A case has `baseline` and +`candidate` runs. Each run carries `repo`, `base_sha`, `head_sha`, +`reviewer_version`, `run_proof`, the actual structured `record` from +`review-output-mcp.py`, `record_sha256`, `adjudication_complete`, +`adjudication_proof`, and `judgments`. Top-level `baseline_version` and +`candidate_version` select the exact versions being compared. + +A judgment has the zero-based finding `index`, `verdict` (`bug` or +`false_positive`), the matching `bug_id` (null for a false positive), and a +`proof` link. Each finding needs one judgment. The record digest is SHA-256 of +UTF-8 JSON serialized with sorted keys, no extra whitespace and unescaped Unicode: + +```sh +python3 -c 'import hashlib,json,sys; r=json.load(open(sys.argv[1],encoding="utf-8")); print(hashlib.sha256(json.dumps(r,sort_keys=True,separators=(",",":"),ensure_ascii=False).encode()).hexdigest())' findings.json +python3 .github/cursor-review/reviewer-eval.py < adjudicated-cases.json +``` + +Only pairs where **both** versions finished on the specified repo/base/head, +the records match their adjudicated digests, and all findings are judged are +scored. Unknown truth, stale or failed reviews, missing proof, changed records, +and incomplete judgments are listed as excluded; they are never counted as +clean or as missed bugs. Invalid or contradictory labels fail the command. + +The output includes denominators, confirmed bugs found, false positives, +duplicates, clean-case alarms, recall, finding precision and clean-case alarm +rate. A zero denominator produces null. Each bug counts at most once toward +recall; repeated findings are reported as duplicates and lower precision. The +caller must verify the identities, source provenance, labels and linked evidence: +the tool validates link syntax and record integrity, not the truth or authenticity +of what they assert. Results are descriptive for the paired cases only and do not +set an acceptance threshold or authorize a pilot. + +For example, this **synthetic** confirmed-bug case has a clean baseline review +and a clean candidate review. Both missed `bug-1`; the output has one paired case +and zero of one confirmed bugs found for each version. The links and SHAs are +illustrative, not evidence of a real bug. + +```json +{ + "baseline_version": "v1", + "candidate_version": "v2", + "cases": [{ + "id": "example-1", + "repo": "example/repo", + "base_sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "head_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "truth": {"kind": "bug", "proof": "https://example.com/bug", "bug_ids": ["bug-1"]}, + "baseline": { + "repo": "example/repo", "base_sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "head_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "reviewer_version": "v1", + "run_proof": "https://example.com/run-1", "record": {"status": "ok", "findings": []}, + "record_sha256": "a1542dd12dffb7f59221e1935389c4e233c664776f1786ae407b150ea153d93f", + "adjudication_complete": true, "adjudication_proof": "https://example.com/judgment-1", "judgments": [] + }, + "candidate": { + "repo": "example/repo", "base_sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "head_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "reviewer_version": "v2", + "run_proof": "https://example.com/run-2", "record": {"status": "ok", "findings": []}, + "record_sha256": "a1542dd12dffb7f59221e1935389c4e233c664776f1786ae407b150ea153d93f", + "adjudication_complete": true, "adjudication_proof": "https://example.com/judgment-2", "judgments": [] + } + }] +} +``` diff --git a/.github/cursor-review/reviewer-eval.py b/.github/cursor-review/reviewer-eval.py new file mode 100644 index 00000000..af2cef0b --- /dev/null +++ b/.github/cursor-review/reviewer-eval.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +"""Compare two structured review outputs against human-adjudicated cases.""" + +import hashlib +import importlib.util +import json +from pathlib import Path +import re +import sys +from urllib.parse import urlsplit + +spec = importlib.util.spec_from_file_location('review_output', Path(__file__).with_name('review-output-mcp.py')) +review_output = importlib.util.module_from_spec(spec) +spec.loader.exec_module(review_output) +SHA = re.compile(r'^[a-f0-9]{40}$') +DIGEST = re.compile(r'^[a-f0-9]{64}$') + + +class Incomplete(Exception): + pass + + +def proof(value): + if not isinstance(value, str): + return False + try: + url = urlsplit(value) + return url.scheme == 'https' and bool(url.hostname) and not url.username and not url.password + except ValueError: + return False + + +def text(value, field): + if not isinstance(value, str) or not value.strip(): + raise ValueError(f'{field} must be a nonempty string') + return value + + +def digest(record): + encoded = json.dumps(record, sort_keys=True, separators=(',', ':'), ensure_ascii=False).encode('utf-8') + return hashlib.sha256(encoded).hexdigest() + + +def truth_for(case): + truth = case.get('truth') + if not isinstance(truth, dict): + raise Incomplete('truth unavailable') + kind = truth.get('kind') + if kind == 'unknown': + raise Incomplete('truth unavailable') + if kind not in ('bug', 'clean') or not proof(truth.get('proof')): + raise ValueError('truth needs a kind and an HTTPS proof') + bugs = truth.get('bug_ids') + if (not isinstance(bugs, list) or any(not isinstance(x, str) or not x.strip() for x in bugs) + or len(set(bugs)) != len(bugs) or (kind == 'bug') != bool(bugs)): + raise ValueError('bug cases require unique bug ids; clean cases require none') + return kind, set(bugs) + + +def adjudicated(run, case, reviewer_version, bug_ids): + if (not isinstance(run, dict) or run.get('repo') != case['repo'] + or run.get('head_sha') != case['head_sha'] + or run.get('base_sha') != case['base_sha'] + or run.get('reviewer_version') != reviewer_version or not proof(run.get('run_proof'))): + raise Incomplete('review missing or stale') + record = run.get('record') + if not isinstance(record, dict) or record.get('status') != 'ok': + raise Incomplete('review did not finish') + expected = run.get('record_sha256') + if not isinstance(expected, str) or not DIGEST.fullmatch(expected) or expected != digest(record): + raise Incomplete('record digest missing or changed') + if run.get('adjudication_complete') is not True or not proof(run.get('adjudication_proof')): + raise Incomplete('adjudication incomplete') + findings = record.get('findings') + labels = run.get('judgments') + if not isinstance(findings, list) or not isinstance(labels, list): + raise Incomplete('findings or judgments unavailable') + for finding in findings: + try: + review_output.validate_finding(finding, allow_repeat=True) + except (ValueError, TypeError, KeyError) as exc: + raise Incomplete('invalid structured finding') from exc + judgments = {} + for label in labels: + if not isinstance(label, dict): + raise ValueError('invalid judgment') + index = label.get('index') + if type(index) is not int or index < 0 or index >= len(findings) or index in judgments: + raise ValueError('judgment indices must be unique and in range') + verdict, bug_id = label.get('verdict'), label.get('bug_id') + if verdict not in ('bug', 'false_positive') or not proof(label.get('proof')): + raise Incomplete('judgment unavailable') + if (verdict == 'bug' and bug_id not in bug_ids) or (verdict == 'false_positive' and bug_id is not None): + raise ValueError('judgment does not match confirmed bugs') + judgments[index] = (verdict, bug_id) + if len(judgments) != len(findings): + raise Incomplete('judgment unavailable') + bug_matches = [bug for verdict, bug in judgments.values() if verdict == 'bug'] + return {'found': len(set(bug_matches)), 'duplicates': len(bug_matches) - len(set(bug_matches)), + 'false_positives': sum(verdict == 'false_positive' for verdict, _ in judgments.values()), + 'findings': len(findings)} + + +def empty_metrics(): + return {'bugs_total': 0, 'bugs_found': 0, 'false_positive_findings': 0, 'duplicate_findings': 0, + 'findings_total': 0, 'clean_cases': 0, 'clean_cases_with_findings': 0} + + +def ratio(numerator, denominator): + return numerator / denominator if denominator else None + + +def evaluate(data): + if not isinstance(data, dict) or not isinstance(data.get('cases'), list): + raise ValueError('cases must be a list') + versions = {name: text(data.get(f'{name}_version'), f'{name}_version') for name in ('baseline', 'candidate')} + results = {'versions': versions, 'paired_cases': 0, 'excluded': [], 'baseline': empty_metrics(), 'candidate': empty_metrics()} + seen = set() + for case in data['cases']: + if not isinstance(case, dict): + raise ValueError('case must be an object') + identifier = text(case.get('id'), 'case.id') + if identifier in seen: + raise ValueError('case ids must be unique') + seen.add(identifier) + if not re.fullmatch(r'[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+', text(case.get('repo'), 'case.repo')): + raise ValueError('case.repo must be owner/name') + if not all(isinstance(case.get(k), str) and SHA.fullmatch(case[k]) for k in ('base_sha', 'head_sha')): + raise ValueError('case needs base and head SHAs') + try: + kind, bugs = truth_for(case) + pair = {name: adjudicated(case.get(name), case, versions[name], bugs) for name in ('baseline', 'candidate')} + except Incomplete as exc: + results['excluded'].append({'case': identifier, 'reason': str(exc)}) + continue + results['paired_cases'] += 1 + for name, measurement in pair.items(): + metrics = results[name] + metrics['bugs_total'] += len(bugs) + metrics['bugs_found'] += measurement['found'] + metrics['false_positive_findings'] += measurement['false_positives'] + metrics['duplicate_findings'] += measurement['duplicates'] + metrics['findings_total'] += measurement['findings'] + if kind == 'clean': + metrics['clean_cases'] += 1 + metrics['clean_cases_with_findings'] += int(measurement['findings'] > 0) + for name in ('baseline', 'candidate'): + metrics = results[name] + metrics['bug_recall'] = ratio(metrics['bugs_found'], metrics['bugs_total']) + metrics['finding_precision'] = ratio(metrics['bugs_found'], metrics['findings_total']) + metrics['clean_case_alarm_rate'] = ratio(metrics['clean_cases_with_findings'], metrics['clean_cases']) + return results + + +def main(): + try: + result = evaluate(json.load(sys.stdin)) + except (ValueError, TypeError, json.JSONDecodeError) as exc: + print(f'invalid evaluation input: {exc}', file=sys.stderr) + return 2 + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/.github/cursor-review/tests/test_reviewer_eval.py b/.github/cursor-review/tests/test_reviewer_eval.py new file mode 100644 index 00000000..69936fb9 --- /dev/null +++ b/.github/cursor-review/tests/test_reviewer_eval.py @@ -0,0 +1,131 @@ +import hashlib +import importlib.util +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import types +import unittest + +ROOT = Path(__file__).resolve().parents[1] + +def load(name, filename): + spec = importlib.util.spec_from_file_location(name, ROOT / filename) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + +mcp = load('review_output', 'review-output-mcp.py') +evaluator = load('reviewer_eval', 'reviewer-eval.py') +HEAD = 'b' * 40 +BASE = 'a' * 40 +PROOF = 'https://example.com/evidence' +FINDING = {'file': 'src/app.py', 'line': 5, 'side': 'RIGHT', 'severity': 'high', 'body': 'An issue.'} + + +def record(findings): + with tempfile.TemporaryDirectory() as directory: + args = types.SimpleNamespace(mode='judge', out=str(Path(directory) / 'findings.json'), model='test', review_type='judge') + mcp.call_tool(args, 'cursor_review_submit_final', {'findings': findings}) + return json.loads(Path(args.out).read_text()) + + +def run(findings, judgments): + output = record(findings) + digest = hashlib.sha256(json.dumps(output, sort_keys=True, separators=(',', ':'), ensure_ascii=False).encode()).hexdigest() + return {'repo': 'example/repo', 'base_sha': BASE, 'head_sha': HEAD, 'reviewer_version': 'v1', 'run_proof': PROOF, 'record': output, 'record_sha256': digest, + 'adjudication_complete': True, 'adjudication_proof': PROOF, 'judgments': judgments} + + +def judgment(index, verdict, bug=None): + return {'index': index, 'verdict': verdict, 'bug_id': bug, 'proof': PROOF} + + +def case(kind='bug'): + findings = [FINDING] if kind == 'bug' else [] + labels = [judgment(0, 'bug', 'issue-1')] if kind == 'bug' else [] + return {'id': kind, 'repo': 'example/repo', 'base_sha': BASE, 'head_sha': HEAD, + 'truth': {'kind': kind, 'proof': PROOF, 'bug_ids': ['issue-1'] if kind == 'bug' else []}, + 'baseline': run(findings, labels), 'candidate': run(findings, labels)} + + +class ReviewerEvaluationTests(unittest.TestCase): + def test_real_output_scores_matched_bug_and_clean_case(self): + result = evaluator.evaluate({'baseline_version': 'v1', 'candidate_version': 'v1', 'cases': [case(), case('clean')]}) + self.assertEqual(result['paired_cases'], 2) + self.assertEqual(result['baseline']['bugs_found'], 1) + self.assertEqual(result['candidate']['bugs_total'], 1) + self.assertEqual(result['candidate']['clean_cases_with_findings'], 0) + self.assertEqual(result['candidate']['clean_cases'], 1) + + def test_missed_bug_and_false_positive_are_distinct(self): + bug = case() + bug['candidate'] = run([], []) + clean = case('clean') + clean['candidate'] = run([FINDING], [judgment(0, 'false_positive')]) + result = evaluator.evaluate({'baseline_version': 'v1', 'candidate_version': 'v1', 'cases': [bug, clean]}) + self.assertEqual(result['candidate']['bugs_found'], 0) + self.assertEqual(result['candidate']['bugs_total'], 1) + self.assertEqual(result['candidate']['false_positive_findings'], 1) + self.assertEqual(result['candidate']['clean_cases_with_findings'], 1) + + def test_incomplete_or_failed_review_is_excluded_from_both_sides(self): + for mode in ('unfinished', 'unjudged', 'wrong_head', 'changed_record'): + item = case() + if mode == 'unfinished': item['candidate']['record']['status'] = 'error' + if mode == 'unjudged': item['candidate']['judgments'] = [] + if mode == 'wrong_head': item['candidate']['head_sha'] = 'c' * 40 + if mode == 'changed_record': item['candidate']['record']['findings'][0]['body'] = 'Changed.' + result = evaluator.evaluate({'baseline_version': 'v1', 'candidate_version': 'v1', 'cases': [item]}) + self.assertEqual(result['paired_cases'], 0, mode) + self.assertEqual(result['candidate']['bugs_total'], 0, mode) + self.assertEqual(result['excluded'][0]['case'], 'bug') + + def test_other_base_and_mixed_reviewer_versions_are_excluded(self): + for field, value in [('base_sha', 'd' * 40), ('repo', 'other/repo'), ('reviewer_version', 'other'), ('run_proof', None)]: + item = case() + item['candidate'][field] = value + result = evaluator.evaluate({'baseline_version': 'v1', 'candidate_version': 'v1', 'cases': [item]}) + self.assertEqual(result['paired_cases'], 0, field) + + def test_duplicate_bug_findings_do_not_inflate_detection(self): + item = case() + item['candidate'] = run([FINDING, FINDING], [judgment(0, 'bug', 'issue-1'), judgment(1, 'bug', 'issue-1')]) + result = evaluator.evaluate({'baseline_version': 'v1', 'candidate_version': 'v1', 'cases': [item]}) + self.assertEqual(result['candidate']['bugs_found'], 1) + self.assertEqual(result['candidate']['duplicate_findings'], 1) + + def test_invalid_truth_or_unknown_bug_is_rejected(self): + item = case('clean') + item['candidate'] = run([FINDING], [judgment(0, 'bug', 'invented')]) + with self.assertRaises(ValueError): evaluator.evaluate({'baseline_version': 'v1', 'candidate_version': 'v1', 'cases': [item]}) + item = case() + item['truth']['proof'] = None + with self.assertRaises(ValueError): evaluator.evaluate({'baseline_version': 'v1', 'candidate_version': 'v1', 'cases': [item]}) + + def test_unknown_truth_and_empty_denominators_are_explicit(self): + item = case() + item['truth']['kind'] = 'unknown' + result = evaluator.evaluate({'baseline_version': 'v1', 'candidate_version': 'v1', 'cases': [item]}) + self.assertEqual(result['excluded'], [{'case': 'bug', 'reason': 'truth unavailable'}]) + self.assertIsNone(result['baseline']['bug_recall']) + self.assertIsNone(result['baseline']['finding_precision']) + self.assertIsNone(result['baseline']['clean_case_alarm_rate']) + + def test_documented_example_is_runnable(self): + text = (ROOT / 'README.md').read_text() + example = text.split('For example, this **synthetic** confirmed-bug case', 1)[1].split('```json\n', 1)[1].split('\n```', 1)[0] + result = evaluator.evaluate(json.loads(example)) + self.assertEqual(result['paired_cases'], 1) + self.assertEqual(result['candidate']['bugs_total'], 1) + self.assertEqual(result['candidate']['bugs_found'], 0) + + def test_cli_outputs_paired_comparison(self): + proc = subprocess.run([sys.executable, str(ROOT / 'reviewer-eval.py')], input=json.dumps({'baseline_version': 'v1', 'candidate_version': 'v1', 'cases': [case()]}), + text=True, capture_output=True, check=True) + self.assertEqual(json.loads(proc.stdout)['paired_cases'], 1) + + +if __name__ == '__main__': + unittest.main() From 896fd93816d51b7518cc948417f500f1f15e6166 Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Wed, 28 Oct 2026 17:55:37 +0800 Subject: [PATCH 2/2] fix: bind reviewer evaluations to pull requests --- .github/cursor-review/README.md | 10 +++++----- .github/cursor-review/reviewer-eval.py | 3 +++ .github/cursor-review/tests/test_reviewer_eval.py | 13 ++++++++++--- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/.github/cursor-review/README.md b/.github/cursor-review/README.md index 9f9c1168..1b3ff6fe 100644 --- a/.github/cursor-review/README.md +++ b/.github/cursor-review/README.md @@ -517,11 +517,11 @@ human-adjudicated bug and clean cases. It reads a JSON object from stdin and writes a paired comparison to stdout; it does not run a model, publish a review, or change GitHub. There is no bundled or implied production benchmark. -Each case identifies `id`, `repo`, `base_sha`, `head_sha`, and a `truth` object +Each case identifies `id`, `repo`, `pr`, `base_sha`, `head_sha`, and a `truth` object with `kind` (`bug`, `clean`, or `unknown`), `proof`, and `bug_ids`. A confirmed bug case has one or more unique bug IDs; a confirmed clean case has none. Truth and all judgments need HTTPS evidence links. A case has `baseline` and -`candidate` runs. Each run carries `repo`, `base_sha`, `head_sha`, +`candidate` runs. Each run carries `repo`, `pr`, `base_sha`, `head_sha`, `reviewer_version`, `run_proof`, the actual structured `record` from `review-output-mcp.py`, `record_sha256`, `adjudication_complete`, `adjudication_proof`, and `judgments`. Top-level `baseline_version` and @@ -563,19 +563,19 @@ illustrative, not evidence of a real bug. "candidate_version": "v2", "cases": [{ "id": "example-1", - "repo": "example/repo", + "repo": "example/repo", "pr": 42, "base_sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "head_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "truth": {"kind": "bug", "proof": "https://example.com/bug", "bug_ids": ["bug-1"]}, "baseline": { - "repo": "example/repo", "base_sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "repo": "example/repo", "pr": 42, "base_sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "head_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "reviewer_version": "v1", "run_proof": "https://example.com/run-1", "record": {"status": "ok", "findings": []}, "record_sha256": "a1542dd12dffb7f59221e1935389c4e233c664776f1786ae407b150ea153d93f", "adjudication_complete": true, "adjudication_proof": "https://example.com/judgment-1", "judgments": [] }, "candidate": { - "repo": "example/repo", "base_sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "repo": "example/repo", "pr": 42, "base_sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "head_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "reviewer_version": "v2", "run_proof": "https://example.com/run-2", "record": {"status": "ok", "findings": []}, "record_sha256": "a1542dd12dffb7f59221e1935389c4e233c664776f1786ae407b150ea153d93f", diff --git a/.github/cursor-review/reviewer-eval.py b/.github/cursor-review/reviewer-eval.py index af2cef0b..41fc0c23 100644 --- a/.github/cursor-review/reviewer-eval.py +++ b/.github/cursor-review/reviewer-eval.py @@ -59,6 +59,7 @@ def truth_for(case): def adjudicated(run, case, reviewer_version, bug_ids): if (not isinstance(run, dict) or run.get('repo') != case['repo'] + or type(run.get('pr')) is not int or run['pr'] != case['pr'] or run.get('head_sha') != case['head_sha'] or run.get('base_sha') != case['base_sha'] or run.get('reviewer_version') != reviewer_version or not proof(run.get('run_proof'))): @@ -125,6 +126,8 @@ def evaluate(data): seen.add(identifier) if not re.fullmatch(r'[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+', text(case.get('repo'), 'case.repo')): raise ValueError('case.repo must be owner/name') + if type(case.get('pr')) is not int or case['pr'] < 1: + raise ValueError('case.pr must be a positive integer') if not all(isinstance(case.get(k), str) and SHA.fullmatch(case[k]) for k in ('base_sha', 'head_sha')): raise ValueError('case needs base and head SHAs') try: diff --git a/.github/cursor-review/tests/test_reviewer_eval.py b/.github/cursor-review/tests/test_reviewer_eval.py index 69936fb9..179e96ab 100644 --- a/.github/cursor-review/tests/test_reviewer_eval.py +++ b/.github/cursor-review/tests/test_reviewer_eval.py @@ -34,7 +34,7 @@ def record(findings): def run(findings, judgments): output = record(findings) digest = hashlib.sha256(json.dumps(output, sort_keys=True, separators=(',', ':'), ensure_ascii=False).encode()).hexdigest() - return {'repo': 'example/repo', 'base_sha': BASE, 'head_sha': HEAD, 'reviewer_version': 'v1', 'run_proof': PROOF, 'record': output, 'record_sha256': digest, + return {'repo': 'example/repo', 'pr': 42, 'base_sha': BASE, 'head_sha': HEAD, 'reviewer_version': 'v1', 'run_proof': PROOF, 'record': output, 'record_sha256': digest, 'adjudication_complete': True, 'adjudication_proof': PROOF, 'judgments': judgments} @@ -45,7 +45,7 @@ def judgment(index, verdict, bug=None): def case(kind='bug'): findings = [FINDING] if kind == 'bug' else [] labels = [judgment(0, 'bug', 'issue-1')] if kind == 'bug' else [] - return {'id': kind, 'repo': 'example/repo', 'base_sha': BASE, 'head_sha': HEAD, + return {'id': kind, 'repo': 'example/repo', 'pr': 42, 'base_sha': BASE, 'head_sha': HEAD, 'truth': {'kind': kind, 'proof': PROOF, 'bug_ids': ['issue-1'] if kind == 'bug' else []}, 'baseline': run(findings, labels), 'candidate': run(findings, labels)} @@ -83,12 +83,19 @@ def test_incomplete_or_failed_review_is_excluded_from_both_sides(self): self.assertEqual(result['excluded'][0]['case'], 'bug') def test_other_base_and_mixed_reviewer_versions_are_excluded(self): - for field, value in [('base_sha', 'd' * 40), ('repo', 'other/repo'), ('reviewer_version', 'other'), ('run_proof', None)]: + for field, value in [('pr', 43), ('base_sha', 'd' * 40), ('repo', 'other/repo'), ('reviewer_version', 'other'), ('run_proof', None)]: item = case() item['candidate'][field] = value result = evaluator.evaluate({'baseline_version': 'v1', 'candidate_version': 'v1', 'cases': [item]}) self.assertEqual(result['paired_cases'], 0, field) + def test_invalid_pr_is_rejected(self): + for value in [None, True, 0, -1]: + item = case() + item['pr'] = value + with self.assertRaises(ValueError): + evaluator.evaluate({'baseline_version': 'v1', 'candidate_version': 'v1', 'cases': [item]}) + def test_duplicate_bug_findings_do_not_inflate_detection(self): item = case() item['candidate'] = run([FINDING, FINDING], [judgment(0, 'bug', 'issue-1'), judgment(1, 'bug', 'issue-1')])