diff --git a/.github/workflows/test-review-readiness.yml b/.github/workflows/test-review-readiness.yml new file mode 100644 index 00000000..5cf7d28a --- /dev/null +++ b/.github/workflows/test-review-readiness.yml @@ -0,0 +1,27 @@ +name: Test review readiness + +on: + pull_request: + paths: + - 'scripts/review-readiness/**' + - '.github/workflows/test-review-readiness.yml' + push: + branches: [main] + paths: + - 'scripts/review-readiness/**' + - '.github/workflows/test-review-readiness.yml' + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.12' + - run: python3 -m unittest discover -s scripts/review-readiness/tests -v diff --git a/scripts/review-readiness/README.md b/scripts/review-readiness/README.md new file mode 100644 index 00000000..daed8a19 --- /dev/null +++ b/scripts/review-readiness/README.md @@ -0,0 +1,63 @@ +# Review readiness evaluator + +`readiness.py` consumes one JSON object on standard input and writes one JSON +result. It does not fetch evidence, publish results, or modify GitHub. The caller +owns the snapshot, policy and evidence collection. `READY` is an advisory result, +not a merge authorization; GitHub rules remain authoritative. + +```sh +python3 scripts/review-readiness/readiness.py < evidence.json +python3 -m unittest discover -s scripts/review-readiness/tests -v +``` + +The snapshot identifies `repo`, `pr`, `base_sha`, `head_sha`, `tested_sha`, +`policy_version`, and the PR `author`. SHAs are lowercase 40-character hashes. +Each evidence scope repeats `repo`, `pr`, `base_sha`, `head_sha`, and +`policy_version`. Checks additionally name their tested SHA. The policy contains at least one required check object with a unique `name` and +`revision` set to `head` or `tested`, plus at least one human approval. Its +version must match the snapshot. The `tested` revision is supplied by the caller +and can be a merge candidate; `head` is the PR head. A GitHub check run attached +to a head SHA does not by itself prove which revision its job actually tested. + +Input fields: + +- `checks_complete`: whether the producer collected the complete check inventory. + `checks` is a list of scoped `name`, `result`, `tested_sha`, and `proof` entries. + Results are `success`, `failure`, `error`, `pending`, `cancelled`, `skipped`, `neutral`, or `unavailable`. +- `reviews`: a scoped object with `complete`, `proof`, and `approvals`. Each + approval has `actor`, `human`, and `state` (`approved`, `dismissed`, or + `changes_requested`). Supply the current effective review state for each actor. +- `findings`: a scoped object with `complete`, `proof`, and `items`; each item + is `open` or `resolved`. The producer must cover every finding source required + by the repo policy before calling the inventory complete. + +A proof is an HTTPS link. Its syntax is checked, but its contents, identity, +permissions, completeness and relationship to the tested revision are **not** +verified here. The caller must establish those facts, including the source and +identity of human approvals, before treating the result as evidence. Missing +inventories, incomplete inventories, absent proof, and stale scoped records cannot +produce `READY`. Current failures, change requests and open findings produce +`BLOCKED` even when other evidence is incomplete. Stale records do not block or +satisfy the current snapshot. Cancelled, skipped, neutral, or unavailable checks wait. Duplicate +conflicting reviews never count as an approval; actor comparisons ignore case. + +Malformed input or policy exits 2 without a result. Valid input exits 0 with +`state`, scope, tested SHA, reasons, and current evidence links. This evaluator +cannot detect an ABA revision change during collection, confirm GitHub's current +head at publication, establish the merge candidate, or decide which evidence the +repository requires. A trusted producer, repo-owned policy, publication recheck, +and integration tests are still required before a pilot or enforcement. + +## Risk report consumer + +`from_risk.py` accepts an object containing `snapshot`, `policy`, and `report` +with the report-only workflow's `summary` and graded `record`. It uses the +actual evaluator with empty check, review, and finding inventories, so its result +is always `WAITING`. The optional risk grade is classified as `observed`, +`stale`, or `unknown` and never substitutes for evidence. The caller must +independently establish that the supplied snapshot is current. This adapter does +not run automatically in a workflow or publish a check. + +```sh +python3 scripts/review-readiness/from_risk.py < report-envelope.json +``` diff --git a/scripts/review-readiness/from_risk.py b/scripts/review-readiness/from_risk.py new file mode 100644 index 00000000..13c1ab1f --- /dev/null +++ b/scripts/review-readiness/from_risk.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Evaluate readiness with only a report-only risk result as available input.""" + +import json +import sys + +import readiness + + +def evaluate_report(data): + if not isinstance(data, dict): + raise ValueError("input must be an object") + snapshot = data.get("snapshot") + result = readiness.evaluate({ + "snapshot": snapshot, + "policy": data.get("policy"), + "checks_complete": False, + "checks": [], + "reviews": None, + "findings": None, + }) + risk = {"status": "unknown", "tier": None} + report = data.get("report") + if isinstance(report, dict): + summary, record = report.get("summary"), report.get("record") + if isinstance(summary, dict) and isinstance(record, dict): + identity = (summary.get("pr"), summary.get("base_sha"), record.get("repo"), + record.get("pr"), record.get("head_sha"), record.get("author")) + expected = (snapshot["pr"], snapshot["base_sha"], snapshot["repo"], + snapshot["pr"], snapshot["head_sha"], snapshot["author"]) + if all(isinstance(value, (str, int)) and not isinstance(value, bool) and value for value in identity): + matches = identity[:-1] == expected[:-1] and str(identity[-1]).casefold() == expected[-1].casefold() + grade = record.get("risk") + if not matches: + risk["status"] = "stale" + elif (summary.get("status") == "graded" and isinstance(grade, dict) + and grade.get("status") == "ok" and grade.get("tier") in ("R0", "R1", "R2", "R3") + and summary.get("tier") == grade.get("tier")): + risk = {"status": "observed", "tier": grade["tier"]} + result["risk"] = risk + return result + + +def main(): + try: + result = evaluate_report(json.load(sys.stdin)) + except (ValueError, TypeError, json.JSONDecodeError) as exc: + print(f"invalid readiness input: {exc}", file=sys.stderr) + return 2 + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/review-readiness/readiness.py b/scripts/review-readiness/readiness.py new file mode 100644 index 00000000..e1e01131 --- /dev/null +++ b/scripts/review-readiness/readiness.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Evaluate caller-supplied evidence without reading or writing external systems.""" + +import json +import re +import sys +from urllib.parse import urlsplit + +SHA = re.compile(r"^[0-9a-f]{40}$") +SCOPE_KEYS = ("repo", "pr", "base_sha", "head_sha", "policy_version") +CHECK_RESULTS = {"success", "failure", "error", "pending", "cancelled", "unavailable", "skipped", "neutral"} + + +def _object(value, name): + if not isinstance(value, dict): + raise ValueError(f"{name} must be an object") + return value + + +def _text(value, name): + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{name} must be a nonempty string") + return value + + +def _sha(value, name): + if not isinstance(value, str) or not SHA.fullmatch(value): + raise ValueError(f"{name} must be a lowercase 40-character SHA") + return value + + +def _scope(value): + value = _object(value, "scope") + repo = _text(value.get("repo"), "scope.repo") + if not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", repo): + raise ValueError("scope.repo must be owner/name") + pr = value.get("pr") + if type(pr) is not int or pr < 1: + raise ValueError("scope.pr must be a positive integer") + return (repo, pr, _sha(value.get("base_sha"), "base_sha"), + _sha(value.get("head_sha"), "head_sha"), + _text(value.get("policy_version"), "policy_version")) + + +def _proof(value): + if not isinstance(value, str): + return False + try: + url = urlsplit(value) + return url.scheme == "https" and bool(url.hostname) and not url.username and not url.password + except ValueError: + return False + + +def _list(value, name): + if not isinstance(value, list): + raise ValueError(f"{name} must be a list") + return value + + +def _complete(value, name): + if not isinstance(value, bool): + raise ValueError(f"{name} must be a boolean") + return value + + +def evaluate(data): + data = _object(data, "input") + snapshot = _object(data.get("snapshot"), "snapshot") + scope = _scope(snapshot) + tested_sha = _sha(snapshot.get("tested_sha"), "tested_sha") + author = _text(snapshot.get("author"), "snapshot.author").casefold() + policy = _object(data.get("policy"), "policy") + if _text(policy.get("version"), "policy.version") != scope[4]: + raise ValueError("policy version does not match snapshot") + required = {} + for check in _list(policy.get("required_checks"), "required_checks"): + check = _object(check, "required check") + name = _text(check.get("name"), "required check.name") + revision = check.get("revision") + if name in required or revision not in ("head", "tested"): + raise ValueError("required checks need unique names and a head or tested revision") + required[name] = scope[3] if revision == "head" else tested_sha + if not required: + raise ValueError("required_checks must not be empty") + count = policy.get("required_human_approvals") + if type(count) is not int or count < 1: + raise ValueError("required_human_approvals must be a positive integer") + + blocked, waiting = [], [] + proofs = set() + results = {name: [] for name in required} + if not _complete(data.get("checks_complete", False), "checks_complete"): + waiting.append("check inventory incomplete") + for item in _list(data.get("checks", []), "checks"): + item = _object(item, "check") + if _scope(item.get("scope")) != scope: + continue + name = _text(item.get("name"), "check.name") + result = item.get("result") + if result not in CHECK_RESULTS: + raise ValueError("invalid check result") + item_tested = _sha(item.get("tested_sha"), "check.tested_sha") + if name in results and item_tested == required[name]: + proof = item.get("proof") + if _proof(proof): + proofs.add(proof) + results[name].append((result, _proof(proof))) + for name, found in results.items(): + if any(result in ("failure", "error") for result, _ in found): + blocked.append(f"required check failed: {name}") + elif not found or any(result != "success" or not proof for result, proof in found): + waiting.append(f"required check incomplete: {name}") + + reviews = data.get("reviews") + if reviews is None: + waiting.append("review evidence missing") + else: + reviews = _object(reviews, "reviews") + if _scope(reviews.get("scope")) != scope: + waiting.append("required human approvals incomplete") + else: + complete = _complete(reviews.get("complete"), "reviews.complete") + if _proof(reviews.get("proof")): + proofs.add(reviews["proof"]) + actors = {} + for approval in _list(reviews.get("approvals"), "approvals"): + approval = _object(approval, "approval") + actor = _text(approval.get("actor"), "approval.actor").casefold() + human = _complete(approval.get("human"), "approval.human") + state = approval.get("state") + if state not in ("approved", "dismissed", "changes_requested"): + raise ValueError("invalid review state") + if human and actor != author: + actors.setdefault(actor, set()).add(state) + if any("changes_requested" in states for states in actors.values()): + blocked.append("human review requests changes") + approved = sum(states == {"approved"} for states in actors.values()) + if not complete or not _proof(reviews.get("proof")) or approved < count: + waiting.append("required human approvals incomplete") + + findings = data.get("findings") + if findings is None: + waiting.append("finding evidence missing") + else: + findings = _object(findings, "findings") + if _scope(findings.get("scope")) != scope: + waiting.append("finding evidence incomplete") + else: + complete = _complete(findings.get("complete"), "findings.complete") + if _proof(findings.get("proof")): + proofs.add(findings["proof"]) + items = _list(findings.get("items"), "findings.items") + for item in items: + if _object(item, "finding").get("state") not in ("open", "resolved"): + raise ValueError("invalid finding state") + if any(item["state"] == "open" for item in items): + blocked.append("unresolved finding") + if not complete or not _proof(findings.get("proof")): + waiting.append("finding evidence incomplete") + + state = "BLOCKED" if blocked else "WAITING" if waiting else "READY" + return {"state": state, "scope": dict(zip(SCOPE_KEYS, scope)), + "tested_sha": tested_sha, "reasons": blocked + waiting, "proofs": sorted(proofs)} + + +def main(): + try: + result = evaluate(json.load(sys.stdin)) + except (ValueError, TypeError, json.JSONDecodeError) as exc: + print(f"invalid readiness input: {exc}", file=sys.stderr) + return 2 + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/review-readiness/tests/test_from_risk.py b/scripts/review-readiness/tests/test_from_risk.py new file mode 100644 index 00000000..13545f4f --- /dev/null +++ b/scripts/review-readiness/tests/test_from_risk.py @@ -0,0 +1,89 @@ +import importlib.util +import json +from pathlib import Path +import subprocess +import sys +import unittest + +ROOT = Path(__file__).resolve().parents[3] +MODULE_DIR = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(MODULE_DIR)) +SPEC = importlib.util.spec_from_file_location("from_risk", MODULE_DIR / "from_risk.py") +consumer = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(consumer) + +BASE = "a" * 40 +HEAD = "b" * 40 +TESTED = "c" * 40 + + +def context(): + return { + "snapshot": {"repo": "test/repo", "pr": 42, "author": "dev", "base_sha": BASE, + "head_sha": HEAD, "tested_sha": TESTED, "policy_version": "v1"}, + "policy": {"version": "v1", "required_checks": [{"name": "unit", "revision": "tested"}], + "required_human_approvals": 1}, + } + + +def risk_record(): + raw = {"repo": "test/repo", "pr": 42, "title": "docs", "author": "dev", + "author_association": "MEMBER", "is_fork": False, "labels": [], + "head_ref": "feature", "head_sha": HEAD, "additions": 1, "deletions": 0, + "changed_files": 1, "checks_state": "SUCCESS", "checks_status": "ok", + "provenance_status": "ok", "changed_paths_status": "ok", + "changed_paths": [{"path": "README.md", "additions": 1, "deletions": 0, + "change_type": "MODIFIED"}]} + run = subprocess.run(["bash", str(ROOT / "scripts/pr-risk/grade-pr-risk.sh"), "--stdin"], + input=json.dumps(raw), text=True, capture_output=True, check=True) + return json.loads(run.stdout) + + +def envelope(): + record = risk_record() + return {**context(), "report": {"summary": {"pr": 42, "status": "graded", "tier": record["risk"]["tier"], + "base_sha": BASE}, "record": record}} + + +class RiskConsumerTests(unittest.TestCase): + def test_real_grader_output_never_substitutes_for_review_evidence(self): + result = consumer.evaluate_report(envelope()) + self.assertEqual(result["state"], "WAITING") + self.assertEqual(result["risk"]["status"], "observed") + self.assertIn("review evidence missing", result["reasons"]) + self.assertIn("finding evidence missing", result["reasons"]) + + def test_stale_report_is_not_presented_as_current(self): + data = envelope() + data["report"]["summary"]["base_sha"] = "d" * 40 + result = consumer.evaluate_report(data) + self.assertEqual(result["state"], "WAITING") + self.assertEqual(result["risk"]["status"], "stale") + self.assertIsNone(result["risk"]["tier"]) + + def test_unknown_or_inconsistent_grade_stays_unknown(self): + for change in ("ungraded", "wrong_tier"): + data = envelope() + if change == "ungraded": + data["report"]["summary"]["status"] = "ungraded" + else: + data["report"]["summary"]["tier"] = "R3" + result = consumer.evaluate_report(data) + self.assertEqual(result["state"], "WAITING") + self.assertEqual(result["risk"]["status"], "unknown") + self.assertIsNone(result["risk"]["tier"]) + + def test_missing_or_partial_report_is_unknown(self): + for report in (None, {}, {"summary": {}, "record": {}}, {"summary": None}): + result = consumer.evaluate_report({**context(), "report": report}) + self.assertEqual(result["state"], "WAITING") + self.assertEqual(result["risk"]["status"], "unknown") + + def test_cli_executes_consumer(self): + run = subprocess.run([sys.executable, str(MODULE_DIR / "from_risk.py")], + input=json.dumps(envelope()), text=True, capture_output=True, check=True) + self.assertEqual(json.loads(run.stdout)["state"], "WAITING") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/review-readiness/tests/test_readiness.py b/scripts/review-readiness/tests/test_readiness.py new file mode 100644 index 00000000..caf83be6 --- /dev/null +++ b/scripts/review-readiness/tests/test_readiness.py @@ -0,0 +1,163 @@ +import copy +import json +import subprocess +import sys +import importlib.util +from pathlib import Path +import unittest + +SPEC = importlib.util.spec_from_file_location("readiness", Path(__file__).resolve().parents[1] / "readiness.py") +module = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(module) + +BASE = "a" * 40 +HEAD = "b" * 40 +TESTED = "c" * 40 +SCOPE = {"repo": "example/repo", "pr": 42, "base_sha": BASE, "head_sha": HEAD, "policy_version": "v1"} +PROOF = "https://example.com/evidence" + + +def sample(): + return { + "snapshot": {**SCOPE, "tested_sha": TESTED, "author": "author"}, + "policy": {"version": "v1", "required_checks": [{"name": "build", "revision": "tested"}], "required_human_approvals": 1}, + "checks_complete": True, + "checks": [{"scope": SCOPE, "name": "build", "result": "success", "tested_sha": TESTED, "proof": PROOF}], + "reviews": {"scope": SCOPE, "complete": True, "proof": PROOF, "approvals": [{"actor": "reviewer", "human": True, "state": "approved"}]}, + "findings": {"scope": SCOPE, "complete": True, "proof": PROOF, "items": []}, + } + + +class ReadinessTests(unittest.TestCase): + def test_complete_current_evidence_is_ready(self): + self.assertEqual(module.evaluate(sample())["state"], "READY") + + def test_current_failure_blocks_even_with_incomplete_evidence(self): + data = sample() + data["checks"][0]["result"] = "failure" + data["reviews"]["complete"] = False + self.assertEqual(module.evaluate(data)["state"], "BLOCKED") + + def test_open_finding_blocks_even_when_inventory_incomplete(self): + data = sample() + data["findings"].update(complete=False, items=[{"state": "open"}]) + self.assertEqual(module.evaluate(data)["state"], "BLOCKED") + + def test_stale_failure_does_not_block_or_satisfy_current_check(self): + data = sample() + data["checks"][0]["scope"] = {**SCOPE, "head_sha": "d" * 40} + data["checks"][0]["result"] = "failure" + self.assertEqual(module.evaluate(data)["state"], "WAITING") + + def test_skipped_or_unavailable_check_waits(self): + for result in ("skipped", "neutral", "unavailable", "cancelled", "pending"): + data = sample() + data["checks"][0]["result"] = result + self.assertEqual(module.evaluate(data)["state"], "WAITING", result) + + def test_policy_can_require_both_head_and_merge_candidate_checks(self): + data = sample() + data["policy"]["required_checks"].append({"name": "preview", "revision": "head"}) + data["checks"].append({"scope": SCOPE, "name": "preview", "result": "success", "tested_sha": HEAD, "proof": PROOF}) + self.assertEqual(module.evaluate(data)["state"], "READY") + data["checks"][1]["tested_sha"] = TESTED + self.assertEqual(module.evaluate(data)["state"], "WAITING") + + def test_stale_payload_is_ignored_after_its_scope_is_known(self): + data = sample() + data["checks"][0].update(scope={**SCOPE, "head_sha": "d" * 40}, result="unrecognized") + data["reviews"].update(scope={**SCOPE, "head_sha": "d" * 40}, approvals=None) + data["findings"].update(scope={**SCOPE, "head_sha": "d" * 40}, items=None) + self.assertEqual(module.evaluate(data)["state"], "WAITING") + + def test_wrong_tested_revision_does_not_satisfy_check(self): + data = sample() + data["checks"][0]["tested_sha"] = "d" * 40 + self.assertEqual(module.evaluate(data)["state"], "WAITING") + + def test_stale_approval_and_unavailable_findings_wait(self): + for section in ("reviews", "findings"): + data = sample() + data[section]["scope"] = {**SCOPE, "policy_version": "old"} + self.assertEqual(module.evaluate(data)["state"], "WAITING") + data = sample() + data["findings"]["complete"] = False + self.assertEqual(module.evaluate(data)["state"], "WAITING") + + def test_self_and_bot_approvals_do_not_count(self): + data = sample() + data["reviews"]["approvals"] = [{"actor": "author", "human": True, "state": "approved"}, {"actor": "robot", "human": False, "state": "approved"}] + self.assertEqual(module.evaluate(data)["state"], "WAITING") + + def test_missing_or_inconsistent_policy_is_rejected(self): + for change in ({"required_checks": []}, {"required_human_approvals": 0}, {"version": "other"}): + data = sample() + data["policy"].update(change) + with self.assertRaises(ValueError): + module.evaluate(data) + + def test_conflicting_review_states_do_not_count_as_approval(self): + data = sample() + data["reviews"]["approvals"].append({"actor": "reviewer", "human": True, "state": "dismissed"}) + self.assertNotEqual(module.evaluate(data)["state"], "READY") + + def test_missing_proof_or_incomplete_checks_cannot_be_ready(self): + for target in ("check", "reviews", "findings"): + data = sample() + item = data["checks"][0] if target == "check" else data[target] + item.pop("proof") + self.assertEqual(module.evaluate(data)["state"], "WAITING") + data = sample() + data["checks_complete"] = False + self.assertEqual(module.evaluate(data)["state"], "WAITING") + + def test_other_pr_cannot_satisfy_same_revision(self): + data = sample() + data["checks"][0]["scope"] = {**SCOPE, "pr": 43} + self.assertEqual(module.evaluate(data)["state"], "WAITING") + + def test_cli_runs_real_evaluator_and_rejects_invalid_input(self): + script = str(Path(__file__).resolve().parents[1] / "readiness.py") + run = subprocess.run([sys.executable, script], input=json.dumps(sample()), text=True, capture_output=True, check=True) + self.assertEqual(json.loads(run.stdout)["state"], "READY") + invalid = subprocess.run([sys.executable, script], input="{}", text=True, capture_output=True) + self.assertEqual(invalid.returncode, 2) + self.assertEqual(invalid.stdout, "") + + def test_case_insensitive_self_and_duplicate_reviewers(self): + data = sample() + data["reviews"]["approvals"] = [{"actor": "AUTHOR", "human": True, "state": "approved"}] + self.assertEqual(module.evaluate(data)["state"], "WAITING") + data["policy"]["required_human_approvals"] = 2 + data["reviews"]["approvals"] = [{"actor": "Reviewer", "human": True, "state": "approved"}, {"actor": "reviewer", "human": True, "state": "approved"}] + self.assertEqual(module.evaluate(data)["state"], "WAITING") + + def test_output_keeps_current_evidence_links(self): + self.assertEqual(module.evaluate(sample())["proofs"], [PROOF]) + + def test_stale_finding_and_change_request_do_not_block_current_snapshot(self): + data = sample() + stale = {**SCOPE, "base_sha": "d" * 40} + data["findings"].update(scope=stale, items=[{"state": "open"}]) + data["reviews"].update(scope=stale, approvals=[{"actor": "reviewer", "human": True, "state": "changes_requested"}]) + self.assertEqual(module.evaluate(data)["state"], "WAITING") + + def test_current_change_request_blocks_incomplete_review_inventory(self): + data = sample() + data["reviews"].update(complete=False, approvals=[{"actor": "reviewer", "human": True, "state": "changes_requested"}]) + self.assertEqual(module.evaluate(data)["state"], "BLOCKED") + + def test_new_snapshot_with_no_evidence_waits(self): + data = sample() + data.update(checks=[], checks_complete=False, reviews=None, findings=None) + self.assertEqual(module.evaluate(data)["state"], "WAITING") + + def test_input_is_not_mutated(self): + data = sample() + before = copy.deepcopy(data) + module.evaluate(data) + self.assertEqual(data, before) + + +if __name__ == "__main__": + unittest.main()