From c7920ec28a393905aae44fb66bc5f66b7386e118 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 12:41:49 -0700 Subject: [PATCH 01/21] test: share authoritative GitHub review receipt fixture --- tests/execution/github_review/fakes.py | 33 +++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/tests/execution/github_review/fakes.py b/tests/execution/github_review/fakes.py index 1549ab78ee..7e35e4250e 100644 --- a/tests/execution/github_review/fakes.py +++ b/tests/execution/github_review/fakes.py @@ -3,14 +3,17 @@ from __future__ import annotations import asyncio -from dataclasses import dataclass +from dataclasses import dataclass, replace from pathlib import Path from typing import Any from autoskillit.core import ( DiffAnchorAuthority, GitHubReviewComment, + GitHubReviewReceipt, GitHubReviewRequest, + ReviewOperationState, + ReviewReconciliationResult, ReviewResponseClass, ) from autoskillit.execution import ( @@ -298,3 +301,31 @@ def _poster( review_comment_cap=review_comment_cap, wall_clock=clock, ) + + +def authoritative_receipt(**overrides: Any) -> GitHubReviewReceipt: + """Return the canonical server receipt used by verifier and publisher tests.""" + receipt = GitHubReviewReceipt( + schema_version=1, + operation_key="f" * 64, + repository="owner/repo", + pr_number=7, + head_sha="a" * 40, + logical_iteration="review-pr:1", + requested_event="COMMENT", + effective_event="COMMENT", + requested_body_digest="b" * 64, + effective_body_digest="b" * 64, + canonical_finding_digest="c" * 64, + state=ReviewOperationState.SUCCEEDED, + response_class=ReviewResponseClass.SUCCESS, + review_id=901, + comment_ids=(), + canonical_finding_count=0, + reconciliation_result=ReviewReconciliationResult.NOT_NEEDED, + finding_dispositions=(), + created_at=1.0, + updated_at=2.0, + final_attempt_digest="d" * 64, + ) + return replace(receipt, **overrides) From 89ae14f6e68fc42e8dac9807abe27f65fc6e1d32 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 12:44:07 -0700 Subject: [PATCH 02/21] fix: preserve server review receipts with publisher-owned bindings --- src/autoskillit/core/__init__.pyi | 1 + .../core/types/github/_type_github_review.py | 6 + src/autoskillit/server/tools/tools_pr_ops.py | 164 +++-- .../skills_extended/resolve-review/SKILL.md | 25 +- .../skills_extended/review-pr/SKILL.md | 57 +- src/autoskillit/smoke_utils/__init__.py | 2 + .../smoke_utils/review/__init__.py | 2 + .../smoke_utils/review/_publication.py | 287 +++++++-- .../arch/test_review_receipt_single_writer.py | 33 + .../core/types/test_github_review_contract.py | 6 + ...st_review_publication_receipt_contracts.py | 3 + .../test_review_receipt_production_seam.py | 395 ++++++++++++ tests/server/test_tools_pr_ops.py | 59 +- ...resolve_review_diff_context_consumption.py | 45 +- .../skills/test_resolve_review_local_mode.py | 2 +- .../test_review_pr_artifact_idempotency.py | 71 +- .../_review_publication_fixtures.py | 78 +++ .../test_experimental_review_publication.py | 432 +++++-------- tests/smoke_utils/test_facade.py | 2 + tests/smoke_utils/test_review_contracts.py | 18 + .../test_review_publication_durability.py | 606 ++++++++++++++++++ 21 files changed, 1841 insertions(+), 453 deletions(-) create mode 100644 tests/arch/test_review_receipt_single_writer.py create mode 100644 tests/server/test_review_receipt_production_seam.py create mode 100644 tests/smoke_utils/_review_publication_fixtures.py create mode 100644 tests/smoke_utils/test_review_publication_durability.py diff --git a/src/autoskillit/core/__init__.pyi b/src/autoskillit/core/__init__.pyi index e9445cf3b8..321a886e0c 100644 --- a/src/autoskillit/core/__init__.pyi +++ b/src/autoskillit/core/__init__.pyi @@ -1443,6 +1443,7 @@ from .types import extract_path_arg as extract_path_arg from .types import extract_positional_args as extract_positional_args from .types import extract_skill_name as extract_skill_name from .types import fleet_error as fleet_error +from .types import github_review_receipt_filename as github_review_receipt_filename from .types import group_by_pair as group_by_pair from .types import hookdef_session_scope as hookdef_session_scope from .types import is_canonical_integer_text as is_canonical_integer_text diff --git a/src/autoskillit/core/types/github/_type_github_review.py b/src/autoskillit/core/types/github/_type_github_review.py index bb19a40c70..d8679217b3 100644 --- a/src/autoskillit/core/types/github/_type_github_review.py +++ b/src/autoskillit/core/types/github/_type_github_review.py @@ -21,6 +21,7 @@ "ReviewOperationState", "ReviewReconciliationResult", "ReviewResponseClass", + "github_review_receipt_filename", "is_final_github_review_state", "is_valid_github_review_head_sha", "is_valid_github_review_logical_iteration", @@ -60,6 +61,11 @@ def is_valid_github_review_repository(value: object) -> bool: return isinstance(value, str) and _REPOSITORY_RE.fullmatch(value) is not None +def github_review_receipt_filename(pr_number: int) -> str: + """Return the server receipt filename for a pull request.""" + return f"batch_review_response_{pr_number}.json" + + class ReviewOperationState(StrEnum): DRY_RUN = "DRY_RUN" PREPARED = "PREPARED" diff --git a/src/autoskillit/server/tools/tools_pr_ops.py b/src/autoskillit/server/tools/tools_pr_ops.py index bad59fb714..e001d73b31 100644 --- a/src/autoskillit/server/tools/tools_pr_ops.py +++ b/src/autoskillit/server/tools/tools_pr_ops.py @@ -25,6 +25,7 @@ atomic_write, destination_location, get_logger, + github_review_receipt_filename, is_final_github_review_state, is_valid_github_review_head_sha, is_valid_github_review_logical_iteration, @@ -105,8 +106,91 @@ def _unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: return value -def _review_verification_result(success: bool) -> str: - return json.dumps({"reviews_posted": "true" if success else "false"}) +def _review_verification_result(rejection: str | None) -> str: + if rejection is None: + return json.dumps({"reviews_posted": "true"}) + return json.dumps({"reviews_posted": "false", "rejection_reason": rejection}) + + +def _load_receipt_payload(receipt: Path, root: Path) -> dict[str, object] | str: + try: + _, receipt_bytes = read_stable_contained_bytes( + receipt, + root, + max_size_bytes=1_000_000, + ) + except (ContainmentError, OSError): + return "receipt_unreadable" + try: + return _load_unique_json_object(receipt_bytes) + except (TypeError, ValueError): + return "receipt_malformed" + + +def _github_receipt_rejection( + *, + cwd: str, + receipt_path: str, + repository: str, + pr_number: int, + head_sha: str, + logical_iteration: str, + post_state: str, +) -> str | None: + if ( + not Path(cwd).is_absolute() + or not os.path.isdir(cwd) + or not is_valid_github_review_repository(repository) + or not isinstance(pr_number, int) + or isinstance(pr_number, bool) + or pr_number <= 0 + or not is_valid_github_review_head_sha(head_sha) + or not is_valid_github_review_logical_iteration(logical_iteration) + or not is_final_github_review_state(post_state) + ): + return "invalid_request" + receipt = Path(receipt_path) + if not receipt.is_absolute() or receipt.name != github_review_receipt_filename(pr_number): + return "invalid_request" + payload = _load_receipt_payload(receipt, Path(cwd) / ".autoskillit" / "temp") + if isinstance(payload, str): + return payload + operation_key = payload.get("operation_key") + if not is_valid_github_review_operation_key(operation_key): + return "receipt_malformed" + + poster = _get_ctx().github_review_poster + if poster is None: + return "ledger_receipt_missing" + authoritative = poster.verify_receipt(operation_key) + if authoritative is None: + return "ledger_receipt_missing" + if ( + authoritative.repository.casefold() != repository.casefold() + or authoritative.pr_number != pr_number + or authoritative.head_sha != head_sha + or authoritative.logical_iteration != logical_iteration + or authoritative.state.value != post_state + ): + return "identity_mismatch" + + artifact_wire = json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ) + authoritative_wire = json.dumps( + authoritative.to_dict(), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ) + if artifact_wire != authoritative_wire: + return "receipt_wire_mismatch" + return None async def _close_issues_sequentially( @@ -266,68 +350,25 @@ async def verify_review_receipt( if (gate := _require_enabled()) is not None: return gate if mode == "local": - return _review_verification_result(post_state == "LOCAL") + return _review_verification_result( + None if post_state == "LOCAL" else "invalid_request" + ) if mode != "github": - return _review_verification_result(False) - if ( - not Path(cwd).is_absolute() - or not os.path.isdir(cwd) - or not is_valid_github_review_repository(repository) - or not isinstance(pr_number, int) - or isinstance(pr_number, bool) - or pr_number <= 0 - or not is_valid_github_review_head_sha(head_sha) - or not is_valid_github_review_logical_iteration(logical_iteration) - or not is_final_github_review_state(post_state) - ): - return _review_verification_result(False) - receipt = Path(receipt_path) - if not receipt.is_absolute() or receipt.name != f"batch_review_response_{pr_number}.json": - return _review_verification_result(False) - root = Path(cwd) / ".autoskillit" / "temp" - _, receipt_bytes = read_stable_contained_bytes( - receipt, - root, - max_size_bytes=1_000_000, - ) - payload = _load_unique_json_object(receipt_bytes) - operation_key = payload.get("operation_key") - if not is_valid_github_review_operation_key(operation_key): - return _review_verification_result(False) - - poster = _get_ctx().github_review_poster - if poster is None: - return _review_verification_result(False) - authoritative = poster.verify_receipt(operation_key) - if authoritative is None: - return _review_verification_result(False) - artifact_wire = json.dumps( - payload, - sort_keys=True, - separators=(",", ":"), - ensure_ascii=False, - allow_nan=False, - ) - authoritative_wire = json.dumps( - authoritative.to_dict(), - sort_keys=True, - separators=(",", ":"), - ensure_ascii=False, - allow_nan=False, - ) - identity_matches = ( - authoritative.repository.casefold() == repository.casefold() - and authoritative.pr_number == pr_number - and authoritative.head_sha == head_sha - and authoritative.logical_iteration == logical_iteration - and authoritative.state.value == post_state - ) + return _review_verification_result("invalid_request") return _review_verification_result( - identity_matches and artifact_wire == authoritative_wire + _github_receipt_rejection( + cwd=cwd, + receipt_path=receipt_path, + repository=repository, + pr_number=pr_number, + head_sha=head_sha, + logical_iteration=logical_iteration, + post_state=post_state, + ) ) except Exception: logger.error("verify_review_receipt unhandled exception", exc_info=True) - return _review_verification_result(False) + return _review_verification_result("unexpected_error") @mcp.tool(tags={"autoskillit", "headless", "github"}, annotations={"readOnlyHint": True}) @@ -378,16 +419,17 @@ async def post_pr_review( ) root = (Path(cwd) / ".autoskillit" / "temp").resolve() destination = destination_location(Path(receipt_path)) + expected_name = github_review_receipt_filename(pr_number) if ( not Path(cwd).is_absolute() or not destination.is_relative_to(root) - or destination.name != f"batch_review_response_{pr_number}.json" + or destination.name != expected_name ): return _review_post_error( head_sha=head_sha, error=( "receipt_path must be an absolute contained path under " - f"{root} named batch_review_response_{pr_number}.json" + f"{root} named {expected_name}" ), state=ReviewOperationState.TERMINAL, response_class=ReviewResponseClass.CLIENT_ERROR, diff --git a/src/autoskillit/skills_extended/resolve-review/SKILL.md b/src/autoskillit/skills_extended/resolve-review/SKILL.md index c3ca1d7e2b..be379e1436 100644 --- a/src/autoskillit/skills_extended/resolve-review/SKILL.md +++ b/src/autoskillit/skills_extended/resolve-review/SKILL.md @@ -346,13 +346,18 @@ After saving the raw review responses, check for the handoff file from review-pr ```bash # Read review-pr artifacts from the iteration-scoped directory -REVIEW_PR_DIR="${AUTOSKILLIT_TEMP}/review-pr" +REVIEW_PR_DIR="{{AUTOSKILLIT_TEMP}}/review-pr" # If iter_N directory exists under review-pr/, use the latest iteration REVIEW_PR_ITER_DIR=$(ls -d "${REVIEW_PR_DIR}"/iter_* 2>/dev/null | sort -V | tail -1) REVIEW_PR_OUTPUT="${REVIEW_PR_ITER_DIR:-${REVIEW_PR_DIR}}" +REVIEW_PR_OUTPUT="$(realpath -- "$REVIEW_PR_OUTPUT")" DIFF_CONTEXT_PATH="${REVIEW_PR_OUTPUT}/diff_context_{pr_number}.json" +printf '%s\n' "$REVIEW_PR_OUTPUT" ``` +Paste the printed absolute path as a string literal in the Python helper call's +`output_dir` argument. + If the file exists: - Parse it as JSON - Build `diff_context_map: dict[tuple[str, int], dict]` where key is `(entry.path, entry.line)` @@ -364,12 +369,15 @@ If the file exists: - In `mode=local`, read `local_findings_{pr_number}.json` and `diff_context_{pr_number}.json` from the same `REVIEW_PR_OUTPUT`, then call `review_handoff_pair_error(diff_context, local_findings)`. -- In `mode=github`, pair `diff_context_{pr_number}.json` with the published - `batch_review_response_{pr_number}.json` receipt from the same - `REVIEW_PR_OUTPUT`, then call - `review_handoff_pair_error(diff_context, batch_review_response)`. Do not require - a local-findings artifact in GitHub mode. A helper error for either - mode-appropriate pair rejects the handoff and uses the existing fallback path. +- In `mode=github`, pair `diff_context_{pr_number}.json` with the publisher's + `github_review_binding_{pr_number}.json` marker from the same directory by calling + `github_review_handoff_error(output_dir=, + pr_number="{pr_number}")` from `autoskillit.smoke_utils`; it also checks the marker + against the server-written receipt. Do not require a local-findings artifact in + GitHub mode. On a non-null result, set `diff_context_map = {}`, print + `Pre-built context rejected: {error}`, skip schema processing and pre-built + review-level ingestion, and use the existing fallback path. Continue schema + validation and ingestion only when the helper returns `None`. - After that mode-appropriate handoff-pair authority check, require `schema_version == 2`. Every `context_entries` entry with an integer `line` must have a non-empty `anchor_digest`; a null line must omit it. Reject incomplete or @@ -381,6 +389,9 @@ If the file exists: stale, not permission to mint an anchor digest from this checkout. - Log: `"Loaded pre-built context for N findings from review-pr handoff (schema_version: {v})"` +The existing fallback reconstructs code context from review comments and live source +under Step 3.5's checks; it does not retry or redeliver the publication. + After the same pair-authority and v2 schema validation, load `review_level_findings` separately from line-anchored `context_entries`. Preserve each `candidate_id`, path, diagnostic line, message, severity, and dimension; deduplicate diff --git a/src/autoskillit/skills_extended/review-pr/SKILL.md b/src/autoskillit/skills_extended/review-pr/SKILL.md index 81ac473018..42243007cf 100644 --- a/src/autoskillit/skills_extended/review-pr/SKILL.md +++ b/src/autoskillit/skills_extended/review-pr/SKILL.md @@ -812,8 +812,9 @@ If an initially valid retained snapshot is now unavailable or differs, finalizat survivor sets from effect-producing consumers, permits only diagnostic raw/summary envelopes with empty survivors, and emits `stale_snapshot`. In that movement branch revalidation sets `FINAL_SNAPSHOT_STATE=stale` before any consumer is selected. Initial gate degradation remains -`authority_degraded` and emits `needs_human`. Do not publish diff context, local findings, receipts, -comments, reviews, or approvals from either state. On freshness, set +`authority_degraded` and emits `needs_human`. Do not publish diff context, local +findings, the GitHub binding marker, comments, reviews, or approvals from either state. +On freshness, set `COMMIT_ID="$METRICS_HEAD_SHA"` once and never query a later head to replace it. ### Step 4.5: Echo Primary Obligation @@ -962,7 +963,7 @@ Resolve `repository` from the canonical `nameWithOwner` value supplied by the ca require it to match `owner/repo`. Require a positive caller-supplied `pr_number`, and validate the caller-supplied `pr_head_sha` against `^[0-9a-f]{40}$` before publication. Require the caller-supplied `logical_iteration` to be namespaced and require `receipt_path` to be the -contained `batch_review_response_${pr_number}.json` destination under +contained `batch_review_response_${pr_number}.json` destination directly inside `{review_output_dir}` for this invocation. Reject a logical iteration that does not start with `review-pr:`. @@ -1034,7 +1035,7 @@ additional GitHub review write. Re-run the final freshness guard before every `diff_context` handoff publication. Derive `review_generation_id` from the snapshot authority and canonical normalized raw -ledger. Every artifact in a successful invocation carries the same +ledger. Every publisher-written artifact in a successful invocation carries the same `review_generation_id`, `_head_sha`, `_base_sha`, optional `_merge_base_sha`, and `annotation_generation_id`. @@ -1044,13 +1045,16 @@ to a fixed destination. Publish effect-bearing artifacts in this order: 1. diagnostic/raw ledger; 2. diff context; -3. GitHub receipt/publication records when applicable; +3. `github_review_binding_{pr_number}.json` last in GitHub mode as the GitHub generation + commit marker. The publisher derives it from the server-written + `batch_review_response_{pr_number}.json` and never writes, replaces, or deletes that + receipt; 4. `local_findings_{pr_number}.json last` in local mode as the local generation commit marker. On `stale_snapshot`, publish only bounded diagnostic raw and summary envelopes with -empty survivor sets. Do not publish diff context, local findings, receipts, comments, -reviews, or approvals. +empty survivor sets. Do not publish diff context, local findings, the GitHub binding +marker, comments, reviews, or approvals. The raw ledger contains immutable linked arrays named `candidate_records`, `validation_records`, `disposition_records`, `aggregation_records`, @@ -1076,7 +1080,7 @@ bash "{{AUTOSKILLIT_SCRIPTS}}/review_pr_gate.sh" revalidate "{authority_path}" Bind `FINAL_SNAPSHOT_STATE` using Step 2.7's failure rule, then assign `FINAL_SNAPSHOT_STATE == "fresh"` to the Python boolean `SNAPSHOT_IS_FRESH`. -Parse `RECEIPT_DOCUMENT` only when it is non-empty: +Re-finalize when the snapshot state changed: ```text if FINAL_SNAPSHOT_STATE != FINALIZED_SNAPSHOT_STATE: @@ -1097,8 +1101,6 @@ if FINAL_SNAPSHOT_STATE != FINALIZED_SNAPSHOT_STATE: ``` ```python -import json - from autoskillit.smoke_utils import ( prepare_experimental_review_publication, publish_experimental_review_artifacts, @@ -1137,7 +1139,6 @@ else: mode=MODE, snapshot_is_fresh=SNAPSHOT_IS_FRESH, handoff_metadata=HANDOFF_METADATA, - receipt=(json.loads(RECEIPT_DOCUMENT) if MODE == "github" and RECEIPT_DOCUMENT else None), ) if SNAPSHOT_IS_FRESH: publication_generation_id = PUBLICATION["artifacts"]["raw_findings"][ @@ -1145,18 +1146,30 @@ else: ] if publication_generation_id != REVIEW_GENERATION_ID: raise RuntimeError("publication generation changed after external effects") - PUBLICATION_RESULT = publish_experimental_review_artifacts( - publication=PUBLICATION, - output_dir="{review_output_dir}", - pr_number=str(pr_number), - ) + try: + PUBLICATION_RESULT = publish_experimental_review_artifacts( + publication=PUBLICATION, + output_dir="{review_output_dir}", + pr_number=str(pr_number), + ) + except ValueError as exc: + print(f"Publication skipped: {exc}") + PUBLICATION_RESULT = { + "state": "publication_skipped", + "error": str(exc), + "publication_records": [], + } ``` +The server receipt remains the effect authority, so continue to emit the verdict and +receipt tokens if publication is skipped. + This publisher is the only writer of the fixed raw-findings, diff-context, -GitHub-receipt, and local-findings paths. Its executable order is raw findings, -diff context, then the GitHub receipt; local mode instead publishes local findings -last. Every downstream finding is copied from `FINAL_REVIEW_FINDINGS`, retains -opaque fields, and carries both `file` and the normalized `path` alias plus +GitHub-binding, and local-findings paths. Its executable order is raw findings, +diff context, then the GitHub binding marker; local mode instead publishes local findings +last. The server is the only writer of the GitHub receipt. Every downstream finding is +copied from `FINAL_REVIEW_FINDINGS`, retains opaque fields, and carries both `file` and +the normalized `path` alias plus `body`, `side`, and `code_region`. **Write Raw Findings JSON (first):** @@ -1165,7 +1178,7 @@ As the first publication, build the complete raw ledger in memory using the sche specified below. The sole publisher renders it into a same-directory temporary path and atomically renames it to `{review_output_dir}raw_findings_{pr_number}.json`. Do not begin diff-context, -receipt, or local-findings publication until this rename succeeds. On +GitHub-binding, or local-findings publication until this rename succeeds. On `stale_snapshot`, the raw ledger is a bounded diagnostic envelope with empty survivor and publication sets. @@ -1232,7 +1245,7 @@ best-effort and its absence is handled gracefully by resolve-review. Do not independently render or rename this fixed path: the helper invocation above normalizes and publishes it in the same transaction as raw findings and the final -receipt/local marker. +GitHub binding/local marker. Version 2 requires a non-empty `anchor_digest` on every context entry whose `line` is an integer. For `line: null`, omit the digest and do not manufacture source bytes. diff --git a/src/autoskillit/smoke_utils/__init__.py b/src/autoskillit/smoke_utils/__init__.py index 07db7e4adb..52616b87a6 100644 --- a/src/autoskillit/smoke_utils/__init__.py +++ b/src/autoskillit/smoke_utils/__init__.py @@ -66,6 +66,7 @@ determine_experimental_review_verdict, evaluate_review_audit_slots, finalize_review_audit, + github_review_handoff_error, load_review_audit_manifest, normalize_local_review_finding, parse_auditor_findings_output, @@ -119,6 +120,7 @@ "extract_investigation", "fetch_merge_queue_data", "finalize_review_audit", + "github_review_handoff_error", "init_counter", "load_review_audit_manifest", "normalize_local_review_finding", diff --git a/src/autoskillit/smoke_utils/review/__init__.py b/src/autoskillit/smoke_utils/review/__init__.py index b96ffd674e..64d7a1bd13 100644 --- a/src/autoskillit/smoke_utils/review/__init__.py +++ b/src/autoskillit/smoke_utils/review/__init__.py @@ -29,6 +29,7 @@ parse_auditor_findings_output, ) from autoskillit.smoke_utils.review._publication import ( + github_review_handoff_error, normalize_local_review_finding, prepare_experimental_review_publication, publish_experimental_review_artifacts, @@ -54,6 +55,7 @@ "determine_experimental_review_verdict", "evaluate_review_audit_slots", "finalize_review_audit", + "github_review_handoff_error", "load_review_audit_manifest", "normalize_local_review_finding", "parse_auditor_findings_output", diff --git a/src/autoskillit/smoke_utils/review/_publication.py b/src/autoskillit/smoke_utils/review/_publication.py index 585561fee2..61ba9ec174 100644 --- a/src/autoskillit/smoke_utils/review/_publication.py +++ b/src/autoskillit/smoke_utils/review/_publication.py @@ -7,11 +7,24 @@ import hashlib import json import os +import re import tempfile from collections.abc import Mapping, Sequence from pathlib import Path +from typing import cast -from autoskillit.smoke_utils._review_contracts import _is_non_empty_string +from autoskillit.core import ( + ContainmentError, + GitHubReviewReceipt, + github_review_receipt_filename, + is_final_github_review_state, + read_stable_contained_bytes, +) +from autoskillit.core.runtime.private_file import fsync_directory, fsync_file +from autoskillit.smoke_utils._review_contracts import ( + _is_non_empty_string, + review_handoff_pair_error, +) from autoskillit.smoke_utils.review._constants import _bounded_utf8 _MAX_GITHUB_REVIEW_BODY_BYTES = 60 * 1024 @@ -24,8 +37,15 @@ "raw_findings": "raw_findings_{pr_number}.json", "diff_context": "diff_context_{pr_number}.json", "local_findings": "local_findings_{pr_number}.json", - "review_receipt": "batch_review_response_{pr_number}.json", + "github_review_binding": "github_review_binding_{pr_number}.json", } +_GENERATION_IDENTITY_KEYS = ( + "_head_sha", + "_base_sha", + "_merge_base_sha", + "annotation_generation_id", + "review_generation_id", +) def render_review_finding_body(finding: Mapping[str, object]) -> str: @@ -112,12 +132,11 @@ def prepare_experimental_review_publication( mode: str, snapshot_is_fresh: bool, handoff_metadata: Mapping[str, object] | None = None, - receipt: Mapping[str, object] | None = None, unpostable: Sequence[Mapping[str, object]] = (), review_level_findings: Sequence[Mapping[str, object]] = (), ) -> dict[str, object]: """Build one immutable publication generation or suppress stale effects.""" - if mode not in {"github", "local"}: + if mode not in ("github", "local"): raise ValueError(f"mode must be 'github' or 'local', got {mode!r}") head_sha = snapshot.get("head_sha", snapshot.get("_head_sha", "")) base_sha = snapshot.get("base_sha", snapshot.get("_base_sha", "")) @@ -195,6 +214,7 @@ def prepare_experimental_review_publication( } if not snapshot_is_fresh: return { + "mode": mode, "state": "stale_snapshot", "artifact_order": ["raw_findings"], "artifacts": {"raw_findings": raw_findings}, @@ -222,20 +242,98 @@ def prepare_experimental_review_publication( } artifacts["local_findings"] = local_findings artifact_order.append("local_findings") - elif receipt is not None: - normalized_receipt = json.loads( - json.dumps(dict(receipt), sort_keys=True, separators=(",", ":")) - ) - review_receipt = {**normalized_receipt, **identity} - artifacts["review_receipt"] = review_receipt - artifact_order.append("review_receipt") return { + "mode": mode, "state": "complete", "artifact_order": artifact_order, "artifacts": artifacts, } +def _load_github_review_receipt( + root: Path, + pr_number: str, + *, + head_sha: str, +) -> GitHubReviewReceipt: + if re.fullmatch(r"[1-9][0-9]*", pr_number) is None: + raise ValueError("pr_number must be canonical decimal") + try: + identifier = int(pr_number) + except ValueError as exc: + raise ValueError("pr_number must be canonical decimal") from exc + try: + _resolved, data = read_stable_contained_bytes( + root / github_review_receipt_filename(identifier), + root, + max_size_bytes=1_000_000, + ) + except (ContainmentError, OSError) as exc: + raise ValueError(f"receipt unreadable: {exc}") from exc + try: + receipt = GitHubReviewReceipt.from_wire(json.loads(data)) + except ( + KeyError, + TypeError, + ValueError, + OverflowError, + RecursionError, + ) as exc: + raise ValueError(f"receipt malformed: {exc}") from exc + if ( + receipt.pr_number != identifier + or not is_final_github_review_state(receipt.state.value) + or type(receipt.review_id) is not int + or receipt.review_id <= 0 + ): + raise ValueError("receipt is not a final successful review") + if receipt.head_sha != head_sha: + raise ValueError("receipt head does not match the review generation") + return receipt + + +def _publication_order( + publication: Mapping[str, object], + root: Path, + identifier: str, +) -> tuple[list[str], dict[str, object]]: + raw_order = publication.get("artifact_order") + raw_artifacts = publication.get("artifacts") + if not isinstance(raw_order, list) or not isinstance(raw_artifacts, Mapping): + raise ValueError("publication must contain artifact_order and artifacts") + order = [str(name) for name in raw_order] + if len(order) != len(set(order)) or set(order) != set(raw_artifacts): + raise ValueError("artifact_order must name every artifact exactly once") + if any(name not in _PUBLICATION_FILENAMES for name in order): + raise ValueError("publication contains an unknown artifact") + if not order or order[0] != "raw_findings": + raise ValueError("raw_findings must be the first publication") + if "local_findings" in order and order[-1] != "local_findings": + raise ValueError("local_findings must be the final publication marker") + if "github_review_binding" in order and order[-1] != "github_review_binding": + raise ValueError("github_review_binding must be the final publication marker") + mode = publication.get("mode") + if mode not in {"github", "local"}: + raise ValueError("publication mode must be 'github' or 'local'") + + artifacts: dict[str, object] = dict(raw_artifacts) + if publication.get("state") == "complete" and mode == "github": + if order != ["raw_findings", "diff_context"]: + raise ValueError("complete github publication requires raw findings and diff context") + diff_context = cast(Mapping[str, object], artifacts["diff_context"]) + receipt = _load_github_review_receipt( + root, + identifier, + head_sha=cast(str, diff_context["_head_sha"]), + ) + artifacts["github_review_binding"] = { + **{key: diff_context[key] for key in _GENERATION_IDENTITY_KEYS if key in diff_context}, + "receipt_operation_key": receipt.operation_key, + } + order.append("github_review_binding") + return order, artifacts + + def _write_temp_bytes(directory: Path, final_name: str, content: bytes) -> Path: fd, temporary_name = tempfile.mkstemp( dir=directory, @@ -252,19 +350,75 @@ def _write_temp_bytes(directory: Path, final_name: str, content: bytes) -> Path: try: with stream: stream.write(content) + fsync_file(temporary_path) except Exception: temporary_path.unlink(missing_ok=True) raise return temporary_path +def _restore_publication_files( + root: Path, + final_paths: Mapping[str, Path], + prior_bytes: Mapping[str, bytes | None], + changed: Sequence[str], + marker_names: tuple[str, ...], +) -> OSError | None: + try: + for name in marker_names: + if final_paths[name].exists(): + final_paths[name].unlink() + fsync_directory(root) + except OSError as error: + return error + + rollback_error: OSError | None = None + payload_names = [name for name in dict.fromkeys(changed) if name not in marker_names] + for name in payload_names: + try: + previous = prior_bytes[name] + if previous is None: + final_paths[name].unlink(missing_ok=True) + else: + rollback_path = _write_temp_bytes(root, final_paths[name].name, previous) + try: + os.replace(rollback_path, final_paths[name]) + finally: + rollback_path.unlink(missing_ok=True) + fsync_directory(root) + except OSError as error: + rollback_error = error + + marker_names_to_restore = ( + [name for name in marker_names if prior_bytes[name] is not None] + if rollback_error is None + else [] + ) + for name in marker_names_to_restore: + try: + rollback_path = _write_temp_bytes( + root, final_paths[name].name, prior_bytes[name] or b"" + ) + try: + os.replace(rollback_path, final_paths[name]) + finally: + rollback_path.unlink(missing_ok=True) + fsync_directory(root) + except OSError as error: + rollback_error = error + return rollback_error + + def publish_experimental_review_artifacts( *, publication: Mapping[str, object], output_dir: str, pr_number: str, ) -> dict[str, object]: - """Atomically publish a prepared generation and roll back partial renames.""" + """Publish artifacts using a marker derived at commit time from the sibling server receipt. + + The publisher never writes, replaces, or deletes the server receipt. + """ root = Path(output_dir) if not root.is_absolute(): raise ValueError(f"output_dir must be absolute, got {output_dir!r}") @@ -272,35 +426,22 @@ def publish_experimental_review_artifacts( if not identifier or Path(identifier).name != identifier or identifier in {".", ".."}: raise ValueError(f"pr_number must be a path-safe identifier, got {pr_number!r}") - raw_order = publication.get("artifact_order") - raw_artifacts = publication.get("artifacts") - if not isinstance(raw_order, list) or not isinstance(raw_artifacts, Mapping): - raise ValueError("publication must contain artifact_order and artifacts") - order = [str(name) for name in raw_order] - if len(order) != len(set(order)) or set(order) != set(raw_artifacts): - raise ValueError("artifact_order must name every artifact exactly once") - if any(name not in _PUBLICATION_FILENAMES for name in order): - raise ValueError("publication contains an unknown artifact") - if not order or order[0] != "raw_findings": - raise ValueError("raw_findings must be the first publication") - if "local_findings" in order and order[-1] != "local_findings": - raise ValueError("local_findings must be the final publication marker") - if "review_receipt" in order and order[-1] != "review_receipt": - raise ValueError("review_receipt must be published after raw findings and handoffs") + order, artifacts = _publication_order(publication, root, identifier) root.mkdir(parents=True, exist_ok=True) final_paths = { name: root / filename.format(pr_number=identifier) for name, filename in _PUBLICATION_FILENAMES.items() } - retired_names = [name for name in final_paths if name not in order] + marker_names = ("local_findings", "github_review_binding") + retired_names = [ + name for name in final_paths if name not in order and name not in marker_names + ] prior_bytes = { name: path.read_bytes() if path.exists() else None for name, path in final_paths.items() } documents = { - name: ( - json.dumps(raw_artifacts[name], sort_keys=True, separators=(",", ":")) + "\n" - ).encode() + name: (json.dumps(artifacts[name], sort_keys=True, separators=(",", ":")) + "\n").encode() for name in order } staged: dict[str, Path] = {} @@ -312,36 +453,44 @@ def publish_experimental_review_artifacts( final_paths[name].name, documents[name], ) - for name in order[:-1]: + + for name in marker_names: + if final_paths[name].exists(): + final_paths[name].unlink() + changed.append(name) + fsync_directory(root) + + marker_name = order[-1] if order[-1] in marker_names else None + payload_order = order[:-1] if marker_name is not None else order + for name in payload_order: os.replace(staged[name], final_paths[name]) staged.pop(name) changed.append(name) + fsync_directory(root) for name in retired_names: if final_paths[name].exists(): final_paths[name].unlink() changed.append(name) - marker_name = order[-1] - os.replace(staged[marker_name], final_paths[marker_name]) - staged.pop(marker_name) - changed.append(marker_name) + fsync_directory(root) + if marker_name is not None: + os.replace(staged[marker_name], final_paths[marker_name]) + staged.pop(marker_name) + changed.append(marker_name) + fsync_directory(root) except Exception as publication_error: - rollback_error: Exception | None = None for temporary_path in staged.values(): temporary_path.unlink(missing_ok=True) - for name in reversed(changed): - try: - previous = prior_bytes[name] - if previous is None: - final_paths[name].unlink(missing_ok=True) - else: - rollback_path = _write_temp_bytes( - root, - final_paths[name].name, - previous, - ) - os.replace(rollback_path, final_paths[name]) - except OSError as error: # pragma: no cover - exceptional filesystem failure - rollback_error = error + rollback_error = ( + _restore_publication_files( + root, + final_paths, + prior_bytes, + changed, + marker_names, + ) + if changed + else None + ) if rollback_error is not None: raise ExceptionGroup( "publication failed and rollback was incomplete", @@ -365,3 +514,39 @@ def publish_experimental_review_artifacts( "published_paths": {name: str(final_paths[name]) for name in order}, "publication_records": publication_records, } + + +def github_review_handoff_error(*, output_dir: str, pr_number: str) -> str | None: + """Return a stable reason when the GitHub binding cannot be paired to its receipt.""" + root = Path(output_dir) + if not root.is_absolute(): + return f"output_dir must be absolute, got {output_dir!r}" + + documents: dict[str, object] = {} + for name in ("diff_context", "github_review_binding"): + path = root / _PUBLICATION_FILENAMES[name].format(pr_number=pr_number) + try: + _resolved, data = read_stable_contained_bytes(path, root) + documents[name] = json.loads(data) + except (ContainmentError, OSError, ValueError, RecursionError) as exc: + return f"{name} unreadable: {exc}" + + pair_error = review_handoff_pair_error( + documents["diff_context"], + documents["github_review_binding"], + ) + if pair_error is not None: + return pair_error + + binding = cast(Mapping[str, object], documents["github_review_binding"]) + try: + receipt = _load_github_review_receipt( + root, + pr_number, + head_sha=cast(str, binding["_head_sha"]), + ) + except ValueError as exc: + return str(exc) + if binding.get("receipt_operation_key") != receipt.operation_key: + return "receipt operation key does not match the github review binding" + return None diff --git a/tests/arch/test_review_receipt_single_writer.py b/tests/arch/test_review_receipt_single_writer.py new file mode 100644 index 0000000000..22bda16920 --- /dev/null +++ b/tests/arch/test_review_receipt_single_writer.py @@ -0,0 +1,33 @@ +"""Keep the server receipt path and filename formatter in their owners.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.layer("arch"), pytest.mark.small] + +_SOURCE_ROOT = Path(__file__).resolve().parents[2] / "src" / "autoskillit" + + +def _files_containing(token: str) -> set[str]: + return { + path.relative_to(_SOURCE_ROOT).as_posix() + for path in _SOURCE_ROOT.rglob("*.py") + if token in path.read_text(encoding="utf-8") + } + + +def test_receipt_filename_literal_is_owned_by_its_core_definition() -> None: + assert _files_containing("batch_review_response_") == { + "core/types/github/_type_github_review.py" + } + + +def test_receipt_filename_formatter_has_only_the_three_declared_call_sites() -> None: + assert _files_containing("github_review_receipt_filename") == { + "core/types/github/_type_github_review.py", + "server/tools/tools_pr_ops.py", + "smoke_utils/review/_publication.py", + } diff --git a/tests/core/types/test_github_review_contract.py b/tests/core/types/test_github_review_contract.py index 59c816f1a3..0e2b295b8f 100644 --- a/tests/core/types/test_github_review_contract.py +++ b/tests/core/types/test_github_review_contract.py @@ -26,6 +26,7 @@ ReviewOperationState, ReviewReconciliationResult, ReviewResponseClass, + github_review_receipt_filename, is_final_github_review_state, is_valid_github_review_head_sha, is_valid_github_review_logical_iteration, @@ -56,6 +57,7 @@ "ReviewResponseClass", "ReviewReconciliationResult", "ReviewFindingDispositionKind", + "github_review_receipt_filename", "is_final_github_review_state", "is_valid_github_review_head_sha", "is_valid_github_review_logical_iteration", @@ -153,6 +155,10 @@ def test_contract_module_exports_exact_public_surface() -> None: assert getattr(_type_github_review, name) is getattr(sys.modules["autoskillit.core"], name) +def test_github_review_receipt_filename_formats_the_server_artifact_name() -> None: + assert github_review_receipt_filename(42) == "batch_review_response_42.json" + + def test_contract_module_imports_only_stdlib_and_anchor_sibling() -> None: """The review contracts depend only on stdlib and their IL-0 anchor type.""" source_path = Path(inspect.getsourcefile(_type_github_review) or "") diff --git a/tests/recipe/test_review_publication_receipt_contracts.py b/tests/recipe/test_review_publication_receipt_contracts.py index 0d3f5fdd07..8b09bfc5ba 100644 --- a/tests/recipe/test_review_publication_receipt_contracts.py +++ b/tests/recipe/test_review_publication_receipt_contracts.py @@ -175,6 +175,9 @@ def test_review_publisher_captures_receipt_identity( assert skill_inputs["pr_head_sha"] == "${{ context.pr_head_sha }}" assert skill_inputs["anchor_authority_path"] == ("${{ context.anchor_authority_path }}") assert skill_inputs["receipt_path"].endswith(f"/batch_review_response_{consumer.pr_ref}.json") + assert skill_inputs["receipt_path"].rsplit("/", 1)[0] == review["with"]["output_dir"].rstrip( + "/" + ) assert capture["review_operation_key"] == "${{ result.review_operation_key }}" assert capture["review_head_sha"] == "${{ result.review_head_sha }}" assert capture["review_post_state"] == "${{ result.review_post_state }}" diff --git a/tests/server/test_review_receipt_production_seam.py b/tests/server/test_review_receipt_production_seam.py new file mode 100644 index 0000000000..9f284beeff --- /dev/null +++ b/tests/server/test_review_receipt_production_seam.py @@ -0,0 +1,395 @@ +"""Composed production tests for server-owned review receipts.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any +from unittest.mock import AsyncMock, Mock + +import pytest + +import autoskillit.server.tools.tools_pr_ops as tools_pr_ops +from autoskillit.server.tools.tools_pr_ops import post_pr_review, verify_review_receipt +from autoskillit.smoke_utils import ( + prepare_experimental_review_publication, + publish_experimental_review_artifacts, +) +from tests.execution.github_review.fakes import ( + CreateOutcome, + ManualClock, + StatefulReviewGateway, + _poster, +) + +pytestmark = [pytest.mark.layer("server"), pytest.mark.anyio, pytest.mark.medium] + +_HEAD_SHA = "a" * 40 +_BASE_SHA = "b" * 40 +_LOGICAL_ITERATION = "review-pr:2" + + +def _review_paths(tmp_path: Path) -> tuple[Path, Path]: + output_dir = tmp_path / ".autoskillit" / "temp" / "review-pr" / "iter_2" + return output_dir, output_dir / "batch_review_response_42.json" + + +def _prepare_review_context( + tmp_path: Path, + tool_ctx_kitchen_open: Any, + monkeypatch: pytest.MonkeyPatch, + outcomes: list[CreateOutcome], +) -> tuple[ManualClock, Path, Path]: + clock = ManualClock() + gateway = StatefulReviewGateway(clock=clock, outcomes=outcomes) + poster = _poster(tmp_path / "ledger.sqlite3", gateway, clock) + monkeypatch.setattr(tool_ctx_kitchen_open, "github_review_poster", poster) + monkeypatch.setattr(tools_pr_ops, "_get_ctx", lambda: tool_ctx_kitchen_open) + monkeypatch.setattr(tools_pr_ops, "_notify", AsyncMock()) + return (clock, *_review_paths(tmp_path)) + + +async def _post_review( + tmp_path: Path, + receipt_path: Path, + *, + logical_iteration: str = _LOGICAL_ITERATION, +) -> dict[str, Any]: + anchor_authority_path = tmp_path / ".autoskillit" / "temp" / "anchor_authority_42.json" + return json.loads( + await post_pr_review( + cwd=str(tmp_path), + receipt_path=str(receipt_path), + anchor_authority_path=str(anchor_authority_path), + repository="octo/example", + pr_number=42, + head_sha=_HEAD_SHA, + logical_iteration=logical_iteration, + event="COMMENT", + body="Review body.", + comments=[], + dry_run=False, + ctx=Mock(), + ) + ) + + +async def _verify_review( + tmp_path: Path, + receipt_path: Path, + **overrides: object, +) -> dict[str, Any]: + arguments: dict[str, object] = { + "cwd": str(tmp_path), + "receipt_path": str(receipt_path), + "repository": "octo/example", + "pr_number": 42, + "head_sha": _HEAD_SHA, + "logical_iteration": _LOGICAL_ITERATION, + "mode": "github", + "post_state": "SUCCEEDED", + "ctx": Mock(), + } + arguments.update(overrides) + return json.loads(await verify_review_receipt(**arguments)) # type: ignore[arg-type] + + +def _publish( + output_dir: Path, + mode: str, + *, + written_at: str, + fresh: bool = True, +) -> dict[str, Any]: + publication = prepare_experimental_review_publication( + raw_ledger={"candidate_records": []}, + survivors=[], + snapshot={"head_sha": _HEAD_SHA, "base_sha": _BASE_SHA}, + annotation_generation_id="g" * 64, + mode=mode, + snapshot_is_fresh=fresh, + handoff_metadata={"written_at": written_at, "schema_version": 2}, + ) + return publish_experimental_review_artifacts( + publication=publication, + output_dir=str(output_dir), + pr_number="42", + ) + + +def _handoff_error(output_dir: Path) -> str | None: + from autoskillit.smoke_utils import github_review_handoff_error + + return github_review_handoff_error(output_dir=str(output_dir), pr_number="42") + + +@pytest.mark.parametrize( + ("outcomes", "expected_state"), + [ + ([CreateOutcome(200, commit=True)], "SUCCEEDED"), + ( + [CreateOutcome(502, data={"message": "Bad Gateway"}, commit=True)], + "RECONCILED", + ), + ], +) +async def test_server_receipt_survives_github_publication_and_verifies( + outcomes: list[CreateOutcome], + expected_state: str, + tool_ctx_kitchen_open: Any, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _, output_dir, receipt_path = _prepare_review_context( + tmp_path, tool_ctx_kitchen_open, monkeypatch, outcomes + ) + + posted = await _post_review(tmp_path, receipt_path) + + assert posted["state"] == expected_state + assert receipt_path.is_file() + assert receipt_path.stat().st_mode & 0o777 == 0o600 + receipt_bytes = receipt_path.read_bytes() + receipt_inode = receipt_path.stat().st_ino + publication = _publish(output_dir, "github", written_at="t0") + + assert receipt_path.read_bytes() == receipt_bytes + assert receipt_path.stat().st_ino == receipt_inode + assert await _verify_review(tmp_path, receipt_path, post_state=expected_state) == { + "reviews_posted": "true" + } + assert _handoff_error(output_dir) is None + assert publication["publication_records"][-1]["artifact"] == "github_review_binding" + + +async def test_replayed_post_after_publication_still_verifies_and_pairs( + tool_ctx_kitchen_open: Any, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _, output_dir, receipt_path = _prepare_review_context( + tmp_path, + tool_ctx_kitchen_open, + monkeypatch, + [CreateOutcome(200, commit=True)], + ) + await _post_review(tmp_path, receipt_path) + _publish(output_dir, "github", written_at="t0") + + replayed = await _post_review(tmp_path, receipt_path) + + assert replayed["replayed"] is True + assert await _verify_review(tmp_path, receipt_path) == {"reviews_posted": "true"} + assert _handoff_error(output_dir) is None + + +async def test_new_post_without_republication_is_rejected_by_consumer( + tool_ctx_kitchen_open: Any, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _, output_dir, receipt_path = _prepare_review_context( + tmp_path, + tool_ctx_kitchen_open, + monkeypatch, + [CreateOutcome(200, commit=True), CreateOutcome(200, commit=True)], + ) + await _post_review(tmp_path, receipt_path) + _publish(output_dir, "github", written_at="t0") + + second_post = await _post_review(tmp_path, receipt_path, logical_iteration="review-pr:3") + + assert second_post["state"] == "SUCCEEDED" + assert _handoff_error(output_dir) == ( + "receipt operation key does not match the github review binding" + ) + + +def _configure_verifier_rejection( + case: str, + tmp_path: Path, + receipt_path: Path, + clock: ManualClock, + tool_ctx_kitchen_open: Any, + monkeypatch: pytest.MonkeyPatch, +) -> tuple[Path, dict[str, object]]: + overrides: dict[str, object] = {} + if case == "binding_identity": + receipt = json.loads(receipt_path.read_text()) + binding = json.loads((receipt_path.parent / "github_review_binding_42.json").read_text()) + identity = {key: value for key, value in binding.items() if key != "receipt_operation_key"} + receipt_path.write_text( + json.dumps({**receipt, **identity}, sort_keys=True, separators=(",", ":")) + ) + receipt_path.chmod(0o600) + elif case == "review_id": + receipt = json.loads(receipt_path.read_text()) + receipt["review_id"] += 1 + receipt_path.write_text(json.dumps(receipt, sort_keys=True, separators=(",", ":"))) + receipt_path.chmod(0o600) + elif case == "head_sha": + overrides["head_sha"] = "c" * 40 + elif case == "empty_ledger": + empty_poster = _poster( + tmp_path / "empty-ledger.sqlite3", + StatefulReviewGateway(clock=clock), + clock, + ) + monkeypatch.setattr(tool_ctx_kitchen_open, "github_review_poster", empty_poster) + elif case == "wrong_pr_filename": + wrong_path = receipt_path.with_name("batch_review_response_43.json") + wrong_path.write_bytes(receipt_path.read_bytes()) + wrong_path.chmod(0o600) + receipt_path = wrong_path + elif case == "outside_root": + outside_path = tmp_path / "outside" / receipt_path.name + outside_path.parent.mkdir(parents=True) + outside_path.write_bytes(receipt_path.read_bytes()) + outside_path.chmod(0o600) + receipt_path = outside_path + elif case == "deleted": + receipt_path.unlink() + else: + receipt_path.write_bytes(b'{"operation_key":"x","operation_key":"y"}') + receipt_path.chmod(0o600) + return receipt_path, overrides + + +@pytest.mark.parametrize( + ("case", "reason"), + [ + ("binding_identity", "receipt_wire_mismatch"), + ("review_id", "receipt_wire_mismatch"), + ("head_sha", "identity_mismatch"), + ("empty_ledger", "ledger_receipt_missing"), + ("wrong_pr_filename", "invalid_request"), + ("outside_root", "receipt_unreadable"), + ("deleted", "receipt_unreadable"), + ("malformed", "receipt_malformed"), + ], +) +async def test_verifier_rejects_with_exact_reason( + case: str, + reason: str, + tool_ctx_kitchen_open: Any, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + clock, output_dir, receipt_path = _prepare_review_context( + tmp_path, + tool_ctx_kitchen_open, + monkeypatch, + [CreateOutcome(200, commit=True)], + ) + await _post_review(tmp_path, receipt_path) + _publish(output_dir, "github", written_at="t0") + verify_path, overrides = _configure_verifier_rejection( + case, tmp_path, receipt_path, clock, tool_ctx_kitchen_open, monkeypatch + ) + + assert await _verify_review(tmp_path, verify_path, **overrides) == { + "reviews_posted": "false", + "rejection_reason": reason, + } + + +async def test_mixed_generation_handoff_is_rejected( + tool_ctx_kitchen_open: Any, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _, output_dir, receipt_path = _prepare_review_context( + tmp_path, + tool_ctx_kitchen_open, + monkeypatch, + [CreateOutcome(200, commit=True)], + ) + await _post_review(tmp_path, receipt_path) + _publish(output_dir, "github", written_at="t0") + diff_context = output_dir / "diff_context_42.json" + generation_a = diff_context.read_bytes() + _publish(output_dir, "github", written_at="t1") + diff_context.write_bytes(generation_a) + + assert _handoff_error(output_dir) == ( + "paired review artifacts have mismatched review_generation_id" + ) + assert await _verify_review(tmp_path, receipt_path) == {"reviews_posted": "true"} + + +async def test_github_publish_without_server_receipt_fails_closed( + tool_ctx_kitchen_open: Any, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _, output_dir, _ = _prepare_review_context( + tmp_path, + tool_ctx_kitchen_open, + monkeypatch, + [CreateOutcome(200, commit=True)], + ) + import autoskillit.smoke_utils.review._publication as publication_module + + write_spy = Mock(wraps=publication_module._write_temp_bytes) + replace_spy = Mock(wraps=os.replace) + monkeypatch.setattr(publication_module, "_write_temp_bytes", write_spy) + monkeypatch.setattr(publication_module.os, "replace", replace_spy) + + with pytest.raises(ValueError, match=r"^receipt unreadable"): + _publish(output_dir, "github", written_at="t0") + + write_spy.assert_not_called() + replace_spy.assert_not_called() + assert not output_dir.exists() or not any(output_dir.iterdir()) + + +async def test_stale_publication_after_post_preserves_server_receipt( + tool_ctx_kitchen_open: Any, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _, output_dir, receipt_path = _prepare_review_context( + tmp_path, + tool_ctx_kitchen_open, + monkeypatch, + [CreateOutcome(200, commit=True)], + ) + await _post_review(tmp_path, receipt_path) + _publish(output_dir, "github", written_at="t0") + receipt_bytes = receipt_path.read_bytes() + receipt_inode = receipt_path.stat().st_ino + + _publish(output_dir, "github", written_at="t1", fresh=False) + + assert receipt_path.read_bytes() == receipt_bytes + assert receipt_path.stat().st_ino == receipt_inode + assert not (output_dir / "diff_context_42.json").exists() + assert not (output_dir / "github_review_binding_42.json").exists() + assert await _verify_review(tmp_path, receipt_path) == {"reviews_posted": "true"} + + +async def test_local_mode_never_touches_receipt_and_stays_explicit( + tool_ctx_kitchen_open: Any, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _, output_dir, receipt_path = _prepare_review_context( + tmp_path, + tool_ctx_kitchen_open, + monkeypatch, + [CreateOutcome(200, commit=True)], + ) + await _post_review(tmp_path, receipt_path) + receipt_bytes = receipt_path.read_bytes() + + _publish(output_dir, "local", written_at="t0") + + assert receipt_path.read_bytes() == receipt_bytes + assert await _verify_review(tmp_path, receipt_path, mode="local", post_state="LOCAL") == { + "reviews_posted": "true" + } + assert await _verify_review(tmp_path, receipt_path, mode="local", post_state="SUCCEEDED") == { + "reviews_posted": "false", + "rejection_reason": "invalid_request", + } diff --git a/tests/server/test_tools_pr_ops.py b/tests/server/test_tools_pr_ops.py index ee73c07675..aaa0f19299 100644 --- a/tests/server/test_tools_pr_ops.py +++ b/tests/server/test_tools_pr_ops.py @@ -16,7 +16,6 @@ GitHubReviewReceipt, GitHubReviewRequest, ReviewOperationState, - ReviewReconciliationResult, ReviewResponseClass, ) from autoskillit.pipeline.gate import DefaultGateState @@ -29,6 +28,7 @@ post_pr_review, verify_review_receipt, ) +from tests.execution.github_review.fakes import authoritative_receipt from tests.server._recipe_fixture_parameters import ( IMPLEMENTATION_READY_PARAMETERS, ) @@ -164,32 +164,6 @@ async def test_close_issues_sequentially_delays_between_calls() -> None: # --------------------------------------------------------------------------- -def _authoritative_receipt() -> GitHubReviewReceipt: - return GitHubReviewReceipt( - schema_version=1, - operation_key="f" * 64, - repository="owner/repo", - pr_number=7, - head_sha="a" * 40, - logical_iteration="review-pr:1", - requested_event="COMMENT", - effective_event="COMMENT", - requested_body_digest="b" * 64, - effective_body_digest="b" * 64, - canonical_finding_digest="c" * 64, - state=ReviewOperationState.SUCCEEDED, - response_class=ReviewResponseClass.SUCCESS, - review_id=901, - comment_ids=(), - canonical_finding_count=0, - reconciliation_result=ReviewReconciliationResult.NOT_NEEDED, - finding_dispositions=(), - created_at=1.0, - updated_at=2.0, - final_attempt_digest="d" * 64, - ) - - def _write_receipt(cwd: Path, payload: dict[str, object]) -> Path: path = cwd / ".autoskillit" / "temp" / "batch_review_response_7.json" path.parent.mkdir(parents=True, exist_ok=True) @@ -219,7 +193,7 @@ async def test_ledger_receipt_is_authoritative( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: - receipt = _authoritative_receipt() + receipt = authoritative_receipt() poster = _FakeReviewPoster() poster.receipts[receipt.operation_key] = receipt monkeypatch.setattr(tool_ctx_kitchen_open, "github_review_poster", poster) @@ -240,11 +214,14 @@ async def test_handcrafted_receipt_without_ledger_row_is_rejected( ) -> None: monkeypatch.setattr(tool_ctx_kitchen_open, "github_review_poster", _FakeReviewPoster()) monkeypatch.setattr(tools_pr_ops, "_get_ctx", lambda: tool_ctx_kitchen_open) - receipt_path = _write_receipt(tmp_path, _authoritative_receipt().to_dict()) + receipt_path = _write_receipt(tmp_path, authoritative_receipt().to_dict()) result = await _verify_receipt(tmp_path, receipt_path) - assert result == {"reviews_posted": "false"} + assert result == { + "reviews_posted": "false", + "rejection_reason": "ledger_receipt_missing", + } @pytest.mark.anyio @@ -264,7 +241,7 @@ async def test_receipt_identity_must_match_independently_supplied_values( parameter: str, value: object, ) -> None: - receipt = _authoritative_receipt() + receipt = authoritative_receipt() poster = _FakeReviewPoster() poster.receipts[receipt.operation_key] = receipt monkeypatch.setattr(tool_ctx_kitchen_open, "github_review_poster", poster) @@ -273,7 +250,10 @@ async def test_receipt_identity_must_match_independently_supplied_values( result = await _verify_receipt(tmp_path, receipt_path, **{parameter: value}) - assert result == {"reviews_posted": "false"} + assert result == { + "reviews_posted": "false", + "rejection_reason": "identity_mismatch", + } @pytest.mark.anyio @@ -293,7 +273,7 @@ async def test_receipt_contents_must_match_ledger_row_field_for_field( field: str, value: object, ) -> None: - receipt = _authoritative_receipt() + receipt = authoritative_receipt() poster = _FakeReviewPoster() poster.receipts[receipt.operation_key] = receipt monkeypatch.setattr(tool_ctx_kitchen_open, "github_review_poster", poster) @@ -304,7 +284,10 @@ async def test_receipt_contents_must_match_ledger_row_field_for_field( result = await _verify_receipt(tmp_path, receipt_path) - assert result == {"reviews_posted": "false"} + assert result == { + "reviews_posted": "false", + "rejection_reason": "receipt_wire_mismatch", + } @pytest.mark.anyio @@ -322,7 +305,13 @@ async def test_local_mode_gate_requires_explicit_local_marker( post_state=post_state, ) - assert result == {"reviews_posted": expected} + if expected == "true": + assert result == {"reviews_posted": expected} + else: + assert result == { + "reviews_posted": expected, + "rejection_reason": "invalid_request", + } @pytest.mark.anyio diff --git a/tests/skills/test_resolve_review_diff_context_consumption.py b/tests/skills/test_resolve_review_diff_context_consumption.py index 1f2e3e5db1..ae01d06b30 100644 --- a/tests/skills/test_resolve_review_diff_context_consumption.py +++ b/tests/skills/test_resolve_review_diff_context_consumption.py @@ -6,9 +6,11 @@ import pytest from autoskillit.smoke_utils import ( + github_review_handoff_error, prepare_experimental_review_publication, publish_experimental_review_artifacts, ) +from tests.execution.github_review.fakes import authoritative_receipt pytestmark = [pytest.mark.layer("skills"), pytest.mark.medium] @@ -86,7 +88,8 @@ def test_step2_uses_mode_appropriate_generation_pair() -> None: assert "In `mode=local`, read `local_findings_{pr_number}.json`" in normalized assert "In `mode=github`, pair `diff_context_{pr_number}.json`" in normalized - assert "`batch_review_response_{pr_number}.json` receipt" in normalized + assert "`github_review_binding_{pr_number}.json` marker" in normalized + assert "github_review_handoff_error(" in normalized assert "Do not require a local-findings artifact in GitHub mode" in normalized @@ -137,6 +140,7 @@ def test_diff_context_path_matches_review_pr_output_path(): """resolve-review's diff_context path must reference the review-pr handoff file.""" full = _skill_text() assert "diff_context_" in full and "REVIEW_PR_OUTPUT" in full + assert "${AUTOSKILLIT_TEMP}" not in full def test_step2_map_type_is_dict_of_dicts(): @@ -182,10 +186,19 @@ def test_v2_review_level_findings_enter_the_resolver_without_thread_identity() - step2 = _step2_section() step3 = _step3_section() - pair_check = step2.find("review_handoff_pair_error") + pair_check = step2.find("github_review_handoff_error(") schema_check = step2.find("schema_version == 2", pair_check) review_level_load = step2.find("review_level_findings", schema_check) assert 0 <= pair_check < schema_check < review_level_load + rejection = step2.find("On a non-null result", pair_check) + assert pair_check < rejection < schema_check + for instruction in ( + "diff_context_map = {}", + "Pre-built context rejected:", + "skip schema processing and pre-built review-level ingestion", + "use the existing fallback path", + ): + assert instruction in step2 for field in ("candidate_id", "path", "line", "message", "severity", "dimension"): assert field in step2 @@ -296,24 +309,38 @@ def test_actual_publisher_output_matches_resolve_review_path_line_boundary( "candidate_id": "deletion", }, ] + output_dir = tmp_path / mode + receipt = authoritative_receipt() if mode == "github" else None + pr_number = str(receipt.pr_number) if receipt is not None else "51" + if receipt is not None: + output_dir.mkdir(parents=True) + receipt_path = output_dir / f"batch_review_response_{pr_number}.json" + receipt_path.write_text( + json.dumps(receipt.to_dict(), sort_keys=True, separators=(",", ":")) + ) + publication = prepare_experimental_review_publication( raw_ledger={"candidate_records": findings}, survivors=findings, - snapshot={"head_sha": "head", "base_sha": "base", "merge_base_sha": "merge"}, + snapshot={ + "head_sha": receipt.head_sha if receipt is not None else "head", + "base_sha": "base", + "merge_base_sha": "merge", + }, annotation_generation_id="annotation", mode=mode, snapshot_is_fresh=True, - handoff_metadata={"pr_number": 51, "iteration": 3}, - receipt=( - {"posted": True, "http_status": 200, "commit_id": "head"} if mode == "github" else None - ), + handoff_metadata={"pr_number": int(pr_number), "iteration": 3}, ) published = publish_experimental_review_artifacts( publication=publication, - output_dir=str(tmp_path / mode), - pr_number="51", + output_dir=str(output_dir), + pr_number=pr_number, ) + if mode == "github": + assert github_review_handoff_error(output_dir=str(output_dir), pr_number=pr_number) is None + diff_context = json.loads(Path(published["published_paths"]["diff_context"]).read_text()) resolve_review_map = { (entry["path"], entry["line"]): dict(entry) for entry in diff_context["context_entries"] diff --git a/tests/skills/test_resolve_review_local_mode.py b/tests/skills/test_resolve_review_local_mode.py index eaa5e68eef..1c1ed675fc 100644 --- a/tests/skills/test_resolve_review_local_mode.py +++ b/tests/skills/test_resolve_review_local_mode.py @@ -349,7 +349,7 @@ def test_mode_specific_handoff_pairs_use_executable_identity_validator() -> None text = _skill_text() assert "review_handoff_pair_error(diff_context, local_findings)" in text - assert "review_handoff_pair_error(diff_context, batch_review_response)" in text + assert "github_review_handoff_error(" in text def test_local_mode_recognizes_overengineering_dimensions() -> None: diff --git a/tests/skills/test_review_pr_artifact_idempotency.py b/tests/skills/test_review_pr_artifact_idempotency.py index 4c4330ff86..f3e6a7f356 100644 --- a/tests/skills/test_review_pr_artifact_idempotency.py +++ b/tests/skills/test_review_pr_artifact_idempotency.py @@ -2,6 +2,9 @@ from __future__ import annotations +import ast +import builtins +import re from pathlib import Path import pytest @@ -30,6 +33,23 @@ _WINDOW = 600 _RECEIPT_PREFIX = "batch_review_response_" +_UPSTREAM_BINDINGS: dict[str, str] = { + "FINAL_SNAPSHOT_STATE": "Bind `FINAL_SNAPSHOT_STATE` from the one printed word", + "FINALIZED_SNAPSHOT_STATE": "Update `FINALIZED_SNAPSHOT_STATE` to the state passed", + "AUDIT_FINALIZATION": "AUDIT_FINALIZATION = finalize_review_audit(", + "FINAL_REVIEW_FINDINGS": "`FINAL_REVIEW_FINDINGS = FILTERED_FINDINGS = survivors`", + "UNPOSTABLE_FINDINGS": "`UNPOSTABLE_FINDINGS = unpostable`", + "REVIEW_LEVEL_FINDINGS": "`REVIEW_LEVEL_FINDINGS = review_level_findings`", + "RAW_LEDGER": "Before Step 6, finish `RAW_LEDGER` and `HANDOFF_METADATA` in memory", + "HANDOFF_METADATA": "Before Step 6, finish `RAW_LEDGER` and `HANDOFF_METADATA` in memory", + "GATE_AUTHORITY": "parse the one printed JSON object as `GATE_AUTHORITY`", + "ANNOTATION_GENERATION_ID": ( + "`ANNOTATION_GENERATION_ID` from its `snapshot` and generation fields" + ), + "MODE": 'MODE="github"', + "REVIEW_GENERATION_ID": "REVIEW_GENERATION_ID = PUBLICATION_SEED[", + "pr_number": "read -r pr_number pr_url", +} def _all_surrounding_windows(text: str, target: str) -> list[str]: @@ -47,6 +67,26 @@ def _all_surrounding_windows(text: str, target: str) -> list[str]: return windows +def _python_names_in_fence(source: str) -> tuple[set[str], set[str]]: + nodes = list(ast.walk(ast.parse(source))) + loaded = { + node.id for node in nodes if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load) + } + bound = { + node.id for node in nodes if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Store) + } + for node in nodes: + if isinstance(node, (ast.Import, ast.ImportFrom)): + bound.update( + alias.asname + or (alias.name.partition(".")[0] if isinstance(node, ast.Import) else alias.name) + for alias in node.names + ) + elif isinstance(node, ast.ExceptHandler) and node.name: + bound.add(node.name) + return loaded, bound + + @pytest.mark.parametrize("filename_pattern", _VULNERABLE_FILES) def test_write_instruction_uses_dynamic_output_dir(filename_pattern: str) -> None: """Write instructions use the resolved output directory pasted as a literal path. @@ -91,11 +131,37 @@ def test_publication_uses_same_directory_temporary_and_atomic_rename() -> None: assert "same-directory temporary file" in step8 assert "atomic rename" in step8 assert "local_findings_{pr_number}.json last" in step8 + assert "`github_review_binding_{pr_number}.json` last" in step8 assert "review_generation_id" in step8 assert "annotation_generation_id" in step8 assert step8.index("**Write Raw Findings JSON (first):**") < step8.index( "**Write Diff-Scoped Context Handoff" ) + assert step8.index("2. diff context;") < step8.index( + "`github_review_binding_{pr_number}.json` last" + ) + + +def test_step8_python_names_are_bound() -> None: + text = _SKILL_PATH.read_text() + step8 = text[text.index("### Step 8") :] + fences = re.findall(r"```python\s*\n(.*?)\n```", step8, flags=re.DOTALL) + assert fences, "Step 8 must contain executable Python fences" + + loaded: set[str] = set() + for fence_index, source in enumerate(fences, start=1): + loaded_in_fence, stored_or_imported = _python_names_in_fence(source) + loaded.update(loaded_in_fence) + unresolved = loaded_in_fence - stored_or_imported - set(dir(builtins)) + undocumented = unresolved - set(_UPSTREAM_BINDINGS) + assert not undocumented, ( + f"Step 8 Python fence {fence_index} uses undocumented names: {sorted(undocumented)}" + ) + assert set(_UPSTREAM_BINDINGS) <= loaded, ( + f"Step 8 no longer consumes documented inputs: {sorted(set(_UPSTREAM_BINDINGS) - loaded)}" + ) + for name, binding in _UPSTREAM_BINDINGS.items(): + assert binding in text, f"Step 8 input {name!r} has no matching skill binding" def test_runtime_threads_validation_aggregation_and_publication_results() -> None: @@ -107,7 +173,9 @@ def test_runtime_threads_validation_aggregation_and_publication_results() -> Non assert "FINAL_REVIEW_FINDINGS = FILTERED_FINDINGS = survivors" in step4 assert "PUBLICATION = prepare_experimental_review_publication(" in step8 assert "survivors=FINAL_REVIEW_FINDINGS" in step8 - assert "receipt=" in step8 + assert "receipt=" not in step8 + assert "github_review_binding_{pr_number}.json" in step8 + assert "Every publisher-written artifact" in step8 assert "PUBLICATION_RESULT = publish_experimental_review_artifacts(" in step8 assert "if not SNAPSHOT_IS_FRESH:" in step8 assert 'verdict = AUDIT_FINALIZATION["verdict"]' in step8 @@ -123,6 +191,7 @@ def test_fixed_destinations_reject_direct_redirects() -> None: for filename in ( "diff_context_{pr_number}.json", "raw_findings_{pr_number}.json", + "github_review_binding_{pr_number}.json", "local_findings_{pr_number}.json", ): assert f'> "{{review_output_dir}}{filename}"' not in text diff --git a/tests/smoke_utils/_review_publication_fixtures.py b/tests/smoke_utils/_review_publication_fixtures.py new file mode 100644 index 0000000000..722b7b8773 --- /dev/null +++ b/tests/smoke_utils/_review_publication_fixtures.py @@ -0,0 +1,78 @@ +"""Shared fixture builders for experimental review publication tests.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from autoskillit.core import GitHubReviewReceipt +from autoskillit.smoke_utils import ( + prepare_experimental_review_publication, + publish_experimental_review_artifacts, +) +from tests.execution.github_review.fakes import authoritative_receipt + +_REVIEW_HEAD = "a" * 40 +_REVIEW_BASE = "b" * 40 +_GENERATION_IDENTITY_KEYS = ( + "_head_sha", + "_base_sha", + "_merge_base_sha", + "annotation_generation_id", + "review_generation_id", +) + + +def _write_server_receipt( + output_dir: Path, + *, + pr_number: int = 42, + head_sha: str = _REVIEW_HEAD, + **overrides: object, +) -> tuple[GitHubReviewReceipt, Path]: + receipt = authoritative_receipt(pr_number=pr_number, head_sha=head_sha, **overrides) + receipt_path = output_dir / f"batch_review_response_{pr_number}.json" + receipt_path.parent.mkdir(parents=True, exist_ok=True) + receipt_path.write_text(json.dumps(receipt.to_dict(), sort_keys=True, separators=(",", ":"))) + receipt_path.chmod(0o600) + return receipt, receipt_path + + +def _prepared_publication( + mode: str, + *, + written_at: str = "t0", + fresh: bool = True, + head_sha: str = _REVIEW_HEAD, +) -> dict[str, Any]: + return prepare_experimental_review_publication( + raw_ledger={"candidate_records": []}, + survivors=[], + snapshot={"head_sha": head_sha, "base_sha": _REVIEW_BASE}, + annotation_generation_id="g" * 64, + mode=mode, + snapshot_is_fresh=fresh, + handoff_metadata={"written_at": written_at, "schema_version": 2}, + ) + + +def _publish( + output_dir: Path, + mode: str, + *, + written_at: str = "t0", + fresh: bool = True, + pr_number: str = "42", + head_sha: str = _REVIEW_HEAD, +) -> dict[str, Any]: + return publish_experimental_review_artifacts( + publication=_prepared_publication( + mode, + written_at=written_at, + fresh=fresh, + head_sha=head_sha, + ), + output_dir=str(output_dir), + pr_number=pr_number, + ) diff --git a/tests/smoke_utils/test_experimental_review_publication.py b/tests/smoke_utils/test_experimental_review_publication.py index 144f6b52d2..48f4c4da59 100644 --- a/tests/smoke_utils/test_experimental_review_publication.py +++ b/tests/smoke_utils/test_experimental_review_publication.py @@ -2,8 +2,8 @@ import json import os +import re from pathlib import Path -from typing import Any import pytest @@ -17,34 +17,28 @@ render_unpostable_review_section, validate_experimental_auditor_outputs, ) +from tests.execution.github_review.fakes import authoritative_receipt from tests.smoke_utils._experimental_helpers import ( _experimental_candidate, ) +from tests.smoke_utils._review_publication_fixtures import ( + _GENERATION_IDENTITY_KEYS, + _REVIEW_BASE, + _REVIEW_HEAD, + _prepared_publication, + _publish, + _write_server_receipt, +) pytestmark = [pytest.mark.medium] -def _prepared_local_experimental_publication() -> dict[str, Any]: - return prepare_experimental_review_publication( - raw_ledger={"candidate_records": [{"candidate_id": "candidate-1"}]}, - survivors=[ - { - "candidate_id": "candidate-1", - "disposition_id": "disposition-1", - "file": "src/app.py", - "line": 42, - } - ], - snapshot={ - "head_sha": "head", - "base_sha": "base", - "merge_base_sha": "merge", - "diff_sha256": "diff", - }, - annotation_generation_id="annotation-generation", - mode="local", - snapshot_is_fresh=True, - ) +def _write_receipt_payload(output_dir: Path, payload: bytes, *, pr_number: str = "42") -> Path: + receipt_path = output_dir / f"batch_review_response_{pr_number}.json" + receipt_path.parent.mkdir(parents=True, exist_ok=True) + receipt_path.write_bytes(payload) + receipt_path.chmod(0o600) + return receipt_path def _authority(right_side_lines: dict[str, list[int]]) -> DiffAnchorAuthority: @@ -196,72 +190,54 @@ def test_experimental_publication_preserves_provenance_and_suppresses_stale_effe def test_experimental_publication_retires_obsolete_artifacts(tmp_path: Path) -> None: output_dir = tmp_path / "review-output" - base_arguments = { - "raw_ledger": {"candidate_records": [{"candidate_id": "candidate-1"}]}, - "survivors": [{"file": "src/app.py", "line": 42}], - "snapshot": {"head_sha": "head", "base_sha": "base"}, - "annotation_generation_id": "annotation-generation", - } + _, receipt_path = _write_server_receipt(output_dir, pr_number=18) + receipt_bytes = receipt_path.read_bytes() + receipt_inode = receipt_path.stat().st_ino + assert receipt_path.stat().st_mode & 0o777 == 0o600 - local = prepare_experimental_review_publication( - **base_arguments, - mode="local", - snapshot_is_fresh=True, - ) - publish_experimental_review_artifacts( - publication=local, - output_dir=str(output_dir), - pr_number="18", - ) + local = _publish(output_dir, "local", pr_number="18") assert {path.name for path in output_dir.iterdir()} == { "raw_findings_18.json", "diff_context_18.json", "local_findings_18.json", + receipt_path.name, } + assert receipt_path.read_bytes() == receipt_bytes + assert receipt_path.stat().st_ino == receipt_inode - github_with_receipt = prepare_experimental_review_publication( - **base_arguments, - mode="github", - snapshot_is_fresh=True, - receipt={"posted": True, "http_status": 200}, - ) - publish_experimental_review_artifacts( - publication=github_with_receipt, - output_dir=str(output_dir), - pr_number="18", - ) + github = _publish(output_dir, "github", pr_number="18") + binding_path = output_dir / "github_review_binding_18.json" assert {path.name for path in output_dir.iterdir()} == { "raw_findings_18.json", "diff_context_18.json", - "batch_review_response_18.json", + binding_path.name, + receipt_path.name, } + binding_bytes = binding_path.read_bytes() + assert github["publication_records"][-1]["artifact"] == "github_review_binding" + assert receipt_path.read_bytes() == receipt_bytes + assert receipt_path.stat().st_ino == receipt_inode - github_without_receipt = prepare_experimental_review_publication( - **base_arguments, - mode="github", - snapshot_is_fresh=True, - ) - publish_experimental_review_artifacts( - publication=github_without_receipt, - output_dir=str(output_dir), - pr_number="18", - ) + local = _publish(output_dir, "local", pr_number="18", written_at="t1") assert {path.name for path in output_dir.iterdir()} == { "raw_findings_18.json", "diff_context_18.json", + "local_findings_18.json", + receipt_path.name, } + assert local["publication_records"][-1]["artifact"] == "local_findings" + assert receipt_path.read_bytes() == receipt_bytes + assert receipt_path.stat().st_ino == receipt_inode - stale = prepare_experimental_review_publication( - **base_arguments, - mode="github", - snapshot_is_fresh=False, - ) - publish_experimental_review_artifacts( - publication=stale, - output_dir=str(output_dir), - pr_number="18", - ) - assert {path.name for path in output_dir.iterdir()} == {"raw_findings_18.json"} + binding_path.write_bytes(binding_bytes) + stale = _publish(output_dir, "github", pr_number="18", fresh=False, written_at="t2") + assert {path.name for path in output_dir.iterdir()} == { + "raw_findings_18.json", + receipt_path.name, + } + assert stale["publication_records"][-1]["artifact"] == "raw_findings" + assert receipt_path.read_bytes() == receipt_bytes + assert receipt_path.stat().st_ino == receipt_inode def _combined_review_survivors(gate_state: str) -> list[dict[str, object]]: @@ -702,7 +678,7 @@ def test_combined_findings_survive_local_publication_for_every_gate_state( assert standard_finding["snapshot"] == {"head_sha": "head", "base_sha": "base"} -def test_github_receipt_shares_prederived_generation_and_is_published_last( +def test_github_binding_marker_is_published_last_and_binds_server_receipt( tmp_path: Path, ) -> None: survivors = _combined_review_survivors("valid_true") @@ -713,18 +689,20 @@ def test_github_receipt_shares_prederived_generation_and_is_published_last( for item in survivors ], } + output_dir = tmp_path / "github" + receipt, receipt_path = _write_server_receipt( + output_dir, + pr_number=23, + head_sha=_REVIEW_HEAD, + ) + receipt_bytes = receipt_path.read_bytes() + receipt_inode = receipt_path.stat().st_ino metadata = {"pr_number": 23, "schema_version": 1} - snapshot = {"head_sha": "head", "base_sha": "base", "merge_base_sha": "merge"} - seed = prepare_experimental_review_publication( - raw_ledger=ledger, - survivors=survivors, - snapshot=snapshot, - annotation_generation_id="annotation", - mode="github", - snapshot_is_fresh=True, - handoff_metadata=metadata, - ) - generation_id = seed["artifacts"]["raw_findings"]["review_generation_id"] + snapshot = { + "head_sha": _REVIEW_HEAD, + "base_sha": _REVIEW_BASE, + "merge_base_sha": "c" * 40, + } publication = prepare_experimental_review_publication( raw_ledger=ledger, survivors=survivors, @@ -733,35 +711,29 @@ def test_github_receipt_shares_prederived_generation_and_is_published_last( mode="github", snapshot_is_fresh=True, handoff_metadata=metadata, - receipt={ - "posted": True, - "http_status": 200, - "commit_id": "head", - "review_generation_id": "must-be-overridden", - }, ) + generation_id = publication["artifacts"]["raw_findings"]["review_generation_id"] assert publication["artifact_order"] == [ "raw_findings", "diff_context", - "review_receipt", ] - assert publication["artifacts"]["review_receipt"]["review_generation_id"] == generation_id result = publish_experimental_review_artifacts( publication=publication, - output_dir=str(tmp_path / "github"), + output_dir=str(output_dir), pr_number="23", ) assert [Path(record["path"]).name for record in result["publication_records"]] == [ "raw_findings_23.json", "diff_context_23.json", - "batch_review_response_23.json", + "github_review_binding_23.json", ] documents = { name: json.loads(Path(path).read_text()) for name, path in result["published_paths"].items() } + assert set(documents) == {"raw_findings", "diff_context", "github_review_binding"} assert {document["review_generation_id"] for document in documents.values()} == {generation_id} consumer_index = { (entry["path"], entry["line"]): entry @@ -774,205 +746,133 @@ def test_github_receipt_shares_prederived_generation_and_is_published_last( "overengineering_abstraction_surface", } assert consumer_index[("src/app.py", 42)]["disposition_id"] == ("disposition-reachability") - assert documents["review_receipt"]["commit_id"] == "head" + binding = documents["github_review_binding"] + assert binding == { + **{ + key: documents["diff_context"][key] + for key in _GENERATION_IDENTITY_KEYS + if key in documents["diff_context"] + }, + "receipt_operation_key": receipt.operation_key, + } + assert receipt_path.read_bytes() == receipt_bytes + assert receipt_path.stat().st_ino == receipt_inode -def test_write_temp_bytes_closes_descriptor_when_fdopen_fails( - monkeypatch: pytest.MonkeyPatch, +@pytest.mark.parametrize( + ("receipt_case", "expected_error"), + [ + ("missing", "receipt unreadable"), + ("invalid_json", "receipt malformed"), + ("empty", "receipt malformed"), + ("overflow_created_at", "receipt malformed"), + ("dry_run", "receipt is not a final successful review"), + ("missing_review_id", "receipt is not a final successful review"), + ("different_pr", "receipt is not a final successful review"), + ("different_head", "receipt head does not match the review generation"), + ("noncanonical_pr", "pr_number must be canonical decimal"), + ], +) +def test_github_publication_rejects_unbindable_receipt( tmp_path: Path, + receipt_case: str, + expected_error: str, ) -> None: - import autoskillit.smoke_utils.review._publication as publication_module - - real_close = os.close - opened_fd = -1 - closed_fds: list[int] = [] - - def failing_fdopen(fd: int, mode: str) -> None: - nonlocal opened_fd - opened_fd = fd - raise OSError("injected fdopen failure") - - def recording_close(fd: int) -> None: - closed_fds.append(fd) - real_close(fd) - - monkeypatch.setattr(publication_module.os, "fdopen", failing_fdopen) - monkeypatch.setattr(publication_module.os, "close", recording_close) - - with pytest.raises(OSError, match="injected fdopen failure"): - publication_module._write_temp_bytes(tmp_path, "artifact.json", b"payload") + output_dir = tmp_path / "github" + pr_number = "042" if receipt_case == "noncanonical_pr" else "42" + if receipt_case == "invalid_json": + _write_receipt_payload(output_dir, b"not json") + elif receipt_case == "empty": + _write_receipt_payload(output_dir, b"{}") + elif receipt_case not in {"missing", "noncanonical_pr"}: + receipt_pr = 43 if receipt_case == "different_pr" else 42 + receipt_head = "c" * 40 if receipt_case == "different_head" else _REVIEW_HEAD + payload = authoritative_receipt( + pr_number=receipt_pr, + head_sha=receipt_head, + ).to_dict() + if receipt_case == "overflow_created_at": + payload["created_at"] = int("9" * 400) + elif receipt_case == "dry_run": + payload["state"] = "DRY_RUN" + elif receipt_case == "missing_review_id": + payload["review_id"] = None + _write_receipt_payload( + output_dir, + json.dumps(payload, sort_keys=True, separators=(",", ":")).encode(), + ) - assert closed_fds == [opened_fd] - assert not list(tmp_path.iterdir()) + with pytest.raises(ValueError, match=f"^{re.escape(expected_error)}"): + publish_experimental_review_artifacts( + publication=_prepared_publication("github"), + output_dir=str(output_dir), + pr_number=pr_number, + ) -def test_experimental_publication_executes_same_directory_marker_last_renames( +@pytest.mark.parametrize("mode_state", ["missing", "unsupported"]) +def test_github_publication_rejects_invalid_mode_before_writes( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, + mode_state: str, ) -> None: import autoskillit.smoke_utils.review._publication as publication_module - publication = _prepared_local_experimental_publication() - output_dir = tmp_path / "review-output" - real_replace = os.replace - rename_calls: list[tuple[Path, Path]] = [] + publication = _prepared_publication("github") + if mode_state == "missing": + publication.pop("mode") + else: + publication["mode"] = "unsupported" + writes: list[str] = [] + replacements: list[str] = [] + original_write = publication_module._write_temp_bytes + original_replace = os.replace + + def recording_write(directory: Path, final_name: str, content: bytes) -> Path: + writes.append(final_name) + return original_write(directory, final_name, content) def recording_replace(source: str | os.PathLike[str], destination: str | os.PathLike[str]): - rename_calls.append((Path(source), Path(destination))) - real_replace(source, destination) + replacements.append(Path(destination).name) + original_replace(source, destination) + monkeypatch.setattr(publication_module, "_write_temp_bytes", recording_write) monkeypatch.setattr(publication_module.os, "replace", recording_replace) - result = publish_experimental_review_artifacts( - publication=publication, - output_dir=str(output_dir), - pr_number="17", - ) - - expected_names = [ - "raw_findings_17.json", - "diff_context_17.json", - "local_findings_17.json", - ] - assert [destination.name for _, destination in rename_calls] == expected_names - assert all( - source.parent == destination.parent == output_dir for source, destination in rename_calls - ) - assert all(source.name.endswith(".tmp") for source, _ in rename_calls) - assert list(result["published_paths"]) == [ - "raw_findings", - "diff_context", - "local_findings", - ] - for artifact_name, path in result["published_paths"].items(): - assert json.loads(Path(path).read_text()) == publication["artifacts"][artifact_name] - assert result["publication_records"][-1]["artifact"] == "local_findings" - assert not list(output_dir.glob(".*.tmp")) - - -@pytest.mark.parametrize("failure_index", [0, 1, 2]) -def test_experimental_publication_rolls_back_each_write_boundary( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, - failure_index: int, -) -> None: - import autoskillit.smoke_utils.review._publication as publication_module + output_dir = tmp_path / "github" - publication = _prepared_local_experimental_publication() - output_dir = tmp_path / "review-output" - output_dir.mkdir() - final_paths = [ - output_dir / "raw_findings_18.json", - output_dir / "diff_context_18.json", - output_dir / "local_findings_18.json", - ] - for index, path in enumerate(final_paths): - path.write_text(f"old-{index}") - original_write = publication_module._write_temp_bytes - write_index = 0 - - def failing_write(directory: Path, final_name: str, content: bytes) -> Path: - nonlocal write_index - current_index = write_index - write_index += 1 - if current_index == failure_index: - raise OSError("injected write failure") - return original_write(directory, final_name, content) - - monkeypatch.setattr(publication_module, "_write_temp_bytes", failing_write) - with pytest.raises(RuntimeError, match="publication failed"): + with pytest.raises(ValueError, match="mode"): publish_experimental_review_artifacts( publication=publication, output_dir=str(output_dir), - pr_number="18", + pr_number="42", ) - assert [path.read_text() for path in final_paths] == ["old-0", "old-1", "old-2"] - assert not list(output_dir.glob(".*.tmp")) + assert writes == [] + assert replacements == [] + assert not output_dir.exists() -@pytest.mark.parametrize("failure_index", [0, 1, 2]) -def test_experimental_publication_rolls_back_each_rename_boundary( - monkeypatch: pytest.MonkeyPatch, +@pytest.mark.parametrize( + ("artifact", "corruption"), + [("diff_context", "missing"), ("github_review_binding", "malformed")], +) +def test_github_handoff_reports_unreadable_artifact( tmp_path: Path, - failure_index: int, + artifact: str, + corruption: str, ) -> None: - import autoskillit.smoke_utils.review._publication as publication_module + from autoskillit.smoke_utils import github_review_handoff_error - publication = _prepared_local_experimental_publication() - output_dir = tmp_path / "review-output" - output_dir.mkdir() - final_paths = [ - output_dir / "raw_findings_19.json", - output_dir / "diff_context_19.json", - output_dir / "local_findings_19.json", - ] - for index, path in enumerate(final_paths): - path.write_text(f"old-{index}") - real_replace = os.replace - rename_index = 0 - failure_injected = False - - def failing_replace(source: str | os.PathLike[str], destination: str | os.PathLike[str]): - nonlocal failure_injected, rename_index - if not failure_injected: - current_index = rename_index - rename_index += 1 - if current_index == failure_index: - failure_injected = True - raise OSError("injected rename failure") - real_replace(source, destination) - - monkeypatch.setattr(publication_module.os, "replace", failing_replace) - with pytest.raises(RuntimeError, match="publication failed"): - publish_experimental_review_artifacts( - publication=publication, - output_dir=str(output_dir), - pr_number="19", - ) + output_dir = tmp_path / "github" + _write_server_receipt(output_dir) + result = _publish(output_dir, "github") + artifact_path = Path(result["published_paths"][artifact]) + if corruption == "missing": + artifact_path.unlink() + else: + artifact_path.write_text("not json") - assert [path.read_text() for path in final_paths] == ["old-0", "old-1", "old-2"] - assert not list(output_dir.glob(".*.tmp")) + error = github_review_handoff_error(output_dir=str(output_dir), pr_number="42") - -def test_experimental_publication_preserves_publication_and_rollback_failures( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - import autoskillit.smoke_utils.review._publication as publication_module - - publication = _prepared_local_experimental_publication() - output_dir = tmp_path / "review-output" - output_dir.mkdir() - final_paths = [ - output_dir / "raw_findings_20.json", - output_dir / "diff_context_20.json", - output_dir / "local_findings_20.json", - ] - for index, path in enumerate(final_paths): - path.write_text(f"old-{index}") - real_replace = os.replace - replace_count = 0 - - def failing_replace(source: str | os.PathLike[str], destination: str | os.PathLike[str]): - nonlocal replace_count - replace_count += 1 - if replace_count == 2: - raise OSError("injected publication failure") - if replace_count == 3: - raise OSError("injected rollback failure") - real_replace(source, destination) - - monkeypatch.setattr(publication_module.os, "replace", failing_replace) - with pytest.raises(ExceptionGroup) as exc_info: - publish_experimental_review_artifacts( - publication=publication, - output_dir=str(output_dir), - pr_number="20", - ) - - errors = exc_info.value.exceptions - assert [str(error) for error in errors] == [ - "injected publication failure", - "injected rollback failure", - ] - assert [path.read_text() for path in final_paths[1:]] == ["old-1", "old-2"] + assert error is not None + assert error.startswith(f"{artifact} unreadable:") diff --git a/tests/smoke_utils/test_facade.py b/tests/smoke_utils/test_facade.py index 80ee5b6bb5..573882863b 100644 --- a/tests/smoke_utils/test_facade.py +++ b/tests/smoke_utils/test_facade.py @@ -73,6 +73,7 @@ def test_smoke_utils_all_exports_complete() -> None: "extract_investigation", "fetch_merge_queue_data", "finalize_review_audit", + "github_review_handoff_error", "init_counter", "load_review_audit_manifest", "LOCAL_ROUND_EXEMPT_VERDICTS", @@ -133,6 +134,7 @@ def test_smoke_utils_all_exports_complete() -> None: "extract_investigation", "fetch_merge_queue_data", "finalize_review_audit", + "github_review_handoff_error", "init_counter", "load_review_audit_manifest", "normalize_local_review_finding", diff --git a/tests/smoke_utils/test_review_contracts.py b/tests/smoke_utils/test_review_contracts.py index e4434b435a..08d68dce13 100644 --- a/tests/smoke_utils/test_review_contracts.py +++ b/tests/smoke_utils/test_review_contracts.py @@ -24,6 +24,24 @@ def test_review_handoff_pair_requires_one_ordered_identity_contract() -> None: assert review_handoff_pair_error(identity, dict(identity)) is None +def test_github_binding_pairs_with_diff_context_by_generation_identity() -> None: + diff_context = { + "_head_sha": "head", + "_base_sha": "base", + "_merge_base_sha": "merge-base", + "annotation_generation_id": "annotation", + "review_generation_id": "review", + } + binding = {**diff_context, "receipt_operation_key": "f" * 64} + + assert review_handoff_pair_error(diff_context, binding) is None + + binding["review_generation_id"] = "another-review" + assert "mismatched review_generation_id" in str( + review_handoff_pair_error(diff_context, binding) + ) + + @pytest.mark.parametrize( ("field", "second_value", "expected_error"), [ diff --git a/tests/smoke_utils/test_review_publication_durability.py b/tests/smoke_utils/test_review_publication_durability.py new file mode 100644 index 0000000000..addbcc0692 --- /dev/null +++ b/tests/smoke_utils/test_review_publication_durability.py @@ -0,0 +1,606 @@ +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any + +import pytest + +from autoskillit.smoke_utils import ( + prepare_experimental_review_publication, + publish_experimental_review_artifacts, +) +from tests.smoke_utils._review_publication_fixtures import ( + _prepared_publication, + _publish, + _write_server_receipt, +) + +pytestmark = [pytest.mark.medium] + + +def _prepared_local_experimental_publication() -> dict[str, Any]: + return prepare_experimental_review_publication( + raw_ledger={"candidate_records": [{"candidate_id": "candidate-1"}]}, + survivors=[ + { + "candidate_id": "candidate-1", + "disposition_id": "disposition-1", + "file": "src/app.py", + "line": 42, + } + ], + snapshot={ + "head_sha": "head", + "base_sha": "base", + "merge_base_sha": "merge", + "diff_sha256": "diff", + }, + annotation_generation_id="annotation-generation", + mode="local", + snapshot_is_fresh=True, + ) + + +def test_write_temp_bytes_closes_descriptor_when_fdopen_fails( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + import autoskillit.smoke_utils.review._publication as publication_module + + real_close = os.close + opened_fd = -1 + closed_fds: list[int] = [] + + def failing_fdopen(fd: int, mode: str) -> None: + nonlocal opened_fd + opened_fd = fd + raise OSError("injected fdopen failure") + + def recording_close(fd: int) -> None: + closed_fds.append(fd) + real_close(fd) + + monkeypatch.setattr(publication_module.os, "fdopen", failing_fdopen) + monkeypatch.setattr(publication_module.os, "close", recording_close) + + with pytest.raises(OSError, match="injected fdopen failure"): + publication_module._write_temp_bytes(tmp_path, "artifact.json", b"payload") + + assert closed_fds == [opened_fd] + assert not list(tmp_path.iterdir()) + + +@pytest.mark.parametrize( + ("mode", "marker_name"), + [("local", "local_findings"), ("github", "github_review_binding")], +) +def test_experimental_publication_executes_same_directory_marker_last_renames( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + mode: str, + marker_name: str, +) -> None: + import autoskillit.smoke_utils.review._publication as publication_module + + if mode == "github": + _write_server_receipt(tmp_path / "review-output") + publication = _prepared_publication("github") + else: + publication = _prepared_local_experimental_publication() + output_dir = tmp_path / "review-output" + real_replace = os.replace + rename_calls: list[tuple[Path, Path]] = [] + events: list[tuple[str, Path]] = [] + real_file_sync = publication_module.fsync_file + real_directory_sync = publication_module.fsync_directory + + def recording_file_sync(path: Path) -> None: + events.append(("file_sync", path)) + real_file_sync(path) + + def recording_directory_sync(path: Path) -> None: + events.append(("directory_sync", path)) + real_directory_sync(path) + + def recording_replace(source: str | os.PathLike[str], destination: str | os.PathLike[str]): + source_path = Path(source) + destination_path = Path(destination) + rename_calls.append((source_path, destination_path)) + events.append(("replace", destination_path)) + real_replace(source, destination) + + monkeypatch.setattr(publication_module, "fsync_file", recording_file_sync) + monkeypatch.setattr(publication_module, "fsync_directory", recording_directory_sync) + monkeypatch.setattr(publication_module.os, "replace", recording_replace) + result = publish_experimental_review_artifacts( + publication=publication, + output_dir=str(output_dir), + pr_number="42" if mode == "github" else "17", + ) + + identifier = "42" if mode == "github" else "17" + expected_names = [ + f"raw_findings_{identifier}.json", + f"diff_context_{identifier}.json", + f"{marker_name}_{identifier}.json", + ] + assert [destination.name for _, destination in rename_calls] == expected_names + assert all( + source.parent == destination.parent == output_dir for source, destination in rename_calls + ) + assert all(source.name.endswith(".tmp") for source, _ in rename_calls) + first_final_change = next(index for index, (kind, _) in enumerate(events) if kind == "replace") + assert all( + index < first_final_change for index, (kind, _) in enumerate(events) if kind == "file_sync" + ) + for _, destination in rename_calls: + replacement_index = next( + index + for index, (kind, path) in enumerate(events) + if kind == "replace" and path == destination + ) + assert events[replacement_index + 1] == ("directory_sync", output_dir) + assert all(path.parent == output_dir for kind, path in events if kind == "file_sync") + assert list(result["published_paths"]) == [ + "raw_findings", + "diff_context", + marker_name, + ] + for artifact_name, path in result["published_paths"].items(): + if artifact_name in publication["artifacts"]: + assert json.loads(Path(path).read_text()) == publication["artifacts"][artifact_name] + assert result["publication_records"][-1]["artifact"] == marker_name + assert not list(output_dir.glob(".*.tmp")) + + +@pytest.mark.parametrize("state", ["complete", "stale"]) +def test_github_publication_invalidates_old_markers_before_payload_changes( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + state: str, +) -> None: + import autoskillit.smoke_utils.review._publication as publication_module + + output_dir = tmp_path / "review-output" + _, receipt_path = _write_server_receipt(output_dir) + receipt_bytes = receipt_path.read_bytes() + receipt_inode = receipt_path.stat().st_ino + _publish(output_dir, "github", written_at="t0") + marker_paths = { + output_dir / "local_findings_42.json", + output_dir / "github_review_binding_42.json", + } + if state == "stale": + (output_dir / "local_findings_42.json").write_text("prior local marker") + + events: list[tuple[str, Path]] = [] + marker_unlinks = 0 + invalidation_synced = False + real_unlink = Path.unlink + real_replace = os.replace + real_directory_sync = publication_module.fsync_directory + payload_names = {"raw_findings_42.json", "diff_context_42.json"} + + def recording_unlink(path: Path, missing_ok: bool = False) -> None: + nonlocal marker_unlinks + if path in marker_paths: + marker_unlinks += 1 + events.append(("unlink_marker", path)) + elif path.name in payload_names: + assert all(not marker.exists() for marker in marker_paths) + assert invalidation_synced + events.append(("unlink_payload", path)) + real_unlink(path, missing_ok=missing_ok) + + def recording_directory_sync(path: Path) -> None: + nonlocal invalidation_synced + real_directory_sync(path) + events.append(("directory_sync", path)) + if marker_unlinks and all(not marker.exists() for marker in marker_paths): + invalidation_synced = True + + def recording_replace(source: str | os.PathLike[str], destination: str | os.PathLike[str]): + destination_path = Path(destination) + if destination_path.name in payload_names: + assert all(not marker.exists() for marker in marker_paths) + assert invalidation_synced + events.append(("replace_payload", destination_path)) + else: + events.append(("replace_marker", destination_path)) + real_replace(source, destination) + + monkeypatch.setattr(Path, "unlink", recording_unlink) + monkeypatch.setattr(publication_module, "fsync_directory", recording_directory_sync) + monkeypatch.setattr(publication_module.os, "replace", recording_replace) + result = publish_experimental_review_artifacts( + publication=_prepared_publication( + "github", + written_at="t1", + fresh=state == "complete", + ), + output_dir=str(output_dir), + pr_number="42", + ) + + first_payload_change = next( + index + for index, (kind, _) in enumerate(events) + if kind in {"replace_payload", "unlink_payload"} + ) + assert any(kind == "directory_sync" for kind, _ in events[:first_payload_change]) + assert all( + index < first_payload_change + for index, (kind, _) in enumerate(events) + if kind == "unlink_marker" + ) + if state == "complete": + marker_change = next( + index for index, (kind, _) in enumerate(events) if kind == "replace_marker" + ) + assert all( + index < marker_change + for index, (kind, _) in enumerate(events) + if kind in {"replace_payload", "unlink_payload"} + ) + assert events[marker_change + 1] == ("directory_sync", output_dir) + assert result["publication_records"][-1]["artifact"] == "github_review_binding" + assert (output_dir / "github_review_binding_42.json").exists() + else: + assert not any(kind == "replace_marker" for kind, _ in events) + assert result["publication_records"][-1]["artifact"] == "raw_findings" + assert not any(marker.exists() for marker in marker_paths) + assert receipt_path.read_bytes() == receipt_bytes + assert receipt_path.stat().st_ino == receipt_inode + + +@pytest.mark.parametrize( + ("failure_point", "directory_sync_index"), + [ + ("staged_file", None), + ("marker_invalidation", 0), + ("payload_rename", 1), + ("final_marker", 3), + ], +) +def test_publication_sync_failure_restores_previous_generation( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + failure_point: str, + directory_sync_index: int | None, +) -> None: + import autoskillit.smoke_utils.review._publication as publication_module + + output_dir = tmp_path / "review-output" + _, receipt_path = _write_server_receipt(output_dir) + receipt_bytes = receipt_path.read_bytes() + receipt_inode = receipt_path.stat().st_ino + _publish(output_dir, "github", written_at="t0") + artifact_paths = { + output_dir / "raw_findings_42.json", + output_dir / "diff_context_42.json", + output_dir / "github_review_binding_42.json", + } + previous = {path: path.read_bytes() for path in artifact_paths} + binding_path = output_dir / "github_review_binding_42.json" + payload_paths = { + output_dir / "raw_findings_42.json", + output_dir / "diff_context_42.json", + } + events: list[tuple[str, Path, bytes | None]] = [] + file_sync_failed = False + directory_sync_failed = False + directory_sync_calls = 0 + real_file_sync = publication_module.fsync_file + real_directory_sync = publication_module.fsync_directory + real_replace = os.replace + real_unlink = Path.unlink + + def recording_file_sync(path: Path) -> None: + nonlocal file_sync_failed + events.append(("file_sync", path, None)) + if failure_point == "staged_file" and not file_sync_failed: + file_sync_failed = True + raise OSError("injected staged file sync failure") + real_file_sync(path) + + def recording_directory_sync(path: Path) -> None: + nonlocal directory_sync_calls, directory_sync_failed + current_index = directory_sync_calls + directory_sync_calls += 1 + if current_index == directory_sync_index and not directory_sync_failed: + directory_sync_failed = True + events.append(("directory_sync_failure", path, None)) + raise OSError("injected directory sync failure") + real_directory_sync(path) + events.append(("directory_sync", path, None)) + + def recording_replace(source: str | os.PathLike[str], destination: str | os.PathLike[str]): + source_path = Path(source) + destination_path = Path(destination) + events.append(("replace", destination_path, source_path.read_bytes())) + real_replace(source, destination) + + def recording_unlink(path: Path, missing_ok: bool = False) -> None: + if path in artifact_paths: + events.append(("unlink", path, None)) + real_unlink(path, missing_ok=missing_ok) + + monkeypatch.setattr(publication_module, "fsync_file", recording_file_sync) + monkeypatch.setattr(publication_module, "fsync_directory", recording_directory_sync) + monkeypatch.setattr(publication_module.os, "replace", recording_replace) + monkeypatch.setattr(Path, "unlink", recording_unlink) + + with pytest.raises(RuntimeError, match="publication failed"): + publish_experimental_review_artifacts( + publication=_prepared_publication("github", written_at="t1"), + output_dir=str(output_dir), + pr_number="42", + ) + + assert {path: path.read_bytes() for path in artifact_paths} == previous + marker_restore_indices = [ + index + for index, (kind, path, content) in enumerate(events) + if kind == "replace" and path == binding_path and content == previous[binding_path] + ] + if failure_point == "staged_file": + assert marker_restore_indices == [] + assert not any(kind in {"replace", "unlink"} for kind, _, _ in events) + else: + marker_restore_index = marker_restore_indices[-1] + changed_payloads = { + path + for index, (kind, path, content) in enumerate(events) + if kind == "replace" + and path in payload_paths + and content != previous[path] + and index < marker_restore_index + } + for path in changed_payloads: + changed_index = next( + index + for index, (kind, event_path, content) in enumerate(events) + if kind == "replace" and event_path == path and content != previous[path] + ) + restore_index = next( + index + for index, (kind, event_path, content) in enumerate(events) + if kind == "replace" + and event_path == path + and content == previous[path] + and index > changed_index + and index < marker_restore_index + ) + assert any( + kind == "directory_sync" + and event_path == output_dir + and restore_index < index < marker_restore_index + for index, (kind, event_path, _) in enumerate(events) + ) + assert not list(output_dir.glob(".*.tmp")) + assert receipt_path.read_bytes() == receipt_bytes + assert receipt_path.stat().st_ino == receipt_inode + + +@pytest.mark.parametrize("restore_failure", ["file", "directory"]) +def test_sync_failure_during_restoration_leaves_commit_markers_absent( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + restore_failure: str, +) -> None: + import autoskillit.smoke_utils.review._publication as publication_module + + output_dir = tmp_path / "review-output" + _, receipt_path = _write_server_receipt(output_dir) + receipt_bytes = receipt_path.read_bytes() + receipt_inode = receipt_path.stat().st_ino + _publish(output_dir, "github", written_at="t0") + raw_path = output_dir / "raw_findings_42.json" + diff_path = output_dir / "diff_context_42.json" + binding_path = output_dir / "github_review_binding_42.json" + previous_raw = raw_path.read_bytes() + previous_diff = diff_path.read_bytes() + publication_sync_failed = False + restoration_payload_replaced = False + restoration_sync_failed = False + directory_sync_calls = 0 + real_file_sync = publication_module.fsync_file + real_directory_sync = publication_module.fsync_directory + real_replace = os.replace + + def failing_file_sync(path: Path) -> None: + nonlocal restoration_sync_failed + if restore_failure == "file" and publication_sync_failed and not restoration_sync_failed: + restoration_sync_failed = True + raise OSError("injected restoration file sync failure") + real_file_sync(path) + + def failing_directory_sync(path: Path) -> None: + nonlocal directory_sync_calls, publication_sync_failed, restoration_sync_failed + current_index = directory_sync_calls + directory_sync_calls += 1 + if current_index == 1 and not publication_sync_failed: + publication_sync_failed = True + raise OSError("injected publication sync failure") + if ( + restore_failure == "directory" + and restoration_payload_replaced + and not restoration_sync_failed + ): + restoration_sync_failed = True + raise OSError("injected restoration directory sync failure") + real_directory_sync(path) + + def recording_replace(source: str | os.PathLike[str], destination: str | os.PathLike[str]): + nonlocal restoration_payload_replaced + source_path = Path(source) + destination_path = Path(destination) + content = source_path.read_bytes() + if destination_path == raw_path and content == previous_raw and publication_sync_failed: + restoration_payload_replaced = True + real_replace(source, destination) + + monkeypatch.setattr(publication_module, "fsync_file", failing_file_sync) + monkeypatch.setattr(publication_module, "fsync_directory", failing_directory_sync) + monkeypatch.setattr(publication_module.os, "replace", recording_replace) + + with pytest.raises(ExceptionGroup) as exc_info: + publish_experimental_review_artifacts( + publication=_prepared_publication("github", written_at="t1"), + output_dir=str(output_dir), + pr_number="42", + ) + + expected_restoration_error = ( + "injected restoration file sync failure" + if restore_failure == "file" + else "injected restoration directory sync failure" + ) + assert [str(error) for error in exc_info.value.exceptions] == [ + "injected publication sync failure", + expected_restoration_error, + ] + assert publication_sync_failed + assert restoration_sync_failed + assert not binding_path.exists() + assert not (output_dir / "local_findings_42.json").exists() + assert diff_path.read_bytes() == previous_diff + assert not list(output_dir.glob(".*.tmp")) + assert receipt_path.read_bytes() == receipt_bytes + assert receipt_path.stat().st_ino == receipt_inode + + +@pytest.mark.parametrize("failure_index", [0, 1, 2]) +def test_experimental_publication_rolls_back_each_write_boundary( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + failure_index: int, +) -> None: + import autoskillit.smoke_utils.review._publication as publication_module + + publication = _prepared_local_experimental_publication() + output_dir = tmp_path / "review-output" + output_dir.mkdir() + final_paths = [ + output_dir / "raw_findings_18.json", + output_dir / "diff_context_18.json", + output_dir / "local_findings_18.json", + output_dir / "github_review_binding_18.json", + ] + for index, path in enumerate(final_paths): + path.write_text(f"old-{index}") + original_write = publication_module._write_temp_bytes + write_index = 0 + + def failing_write(directory: Path, final_name: str, content: bytes) -> Path: + nonlocal write_index + current_index = write_index + write_index += 1 + if current_index == failure_index: + raise OSError("injected write failure") + return original_write(directory, final_name, content) + + monkeypatch.setattr(publication_module, "_write_temp_bytes", failing_write) + with pytest.raises(RuntimeError, match="publication failed"): + publish_experimental_review_artifacts( + publication=publication, + output_dir=str(output_dir), + pr_number="18", + ) + + assert [path.read_text() for path in final_paths] == ["old-0", "old-1", "old-2", "old-3"] + assert not list(output_dir.glob(".*.tmp")) + + +@pytest.mark.parametrize("failure_index", [0, 1, 2]) +def test_experimental_publication_rolls_back_each_rename_boundary( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + failure_index: int, +) -> None: + import autoskillit.smoke_utils.review._publication as publication_module + + publication = _prepared_local_experimental_publication() + output_dir = tmp_path / "review-output" + output_dir.mkdir() + final_paths = [ + output_dir / "raw_findings_19.json", + output_dir / "diff_context_19.json", + output_dir / "local_findings_19.json", + output_dir / "github_review_binding_19.json", + ] + for index, path in enumerate(final_paths): + path.write_text(f"old-{index}") + real_replace = os.replace + rename_index = 0 + failure_injected = False + + def failing_replace(source: str | os.PathLike[str], destination: str | os.PathLike[str]): + nonlocal failure_injected, rename_index + if not failure_injected: + current_index = rename_index + rename_index += 1 + if current_index == failure_index: + failure_injected = True + raise OSError("injected rename failure") + real_replace(source, destination) + + monkeypatch.setattr(publication_module.os, "replace", failing_replace) + with pytest.raises(RuntimeError, match="publication failed"): + publish_experimental_review_artifacts( + publication=publication, + output_dir=str(output_dir), + pr_number="19", + ) + + assert [path.read_text() for path in final_paths] == ["old-0", "old-1", "old-2", "old-3"] + assert not list(output_dir.glob(".*.tmp")) + + +def test_experimental_publication_preserves_publication_and_rollback_failures( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + import autoskillit.smoke_utils.review._publication as publication_module + + publication = _prepared_local_experimental_publication() + output_dir = tmp_path / "review-output" + output_dir.mkdir() + final_paths = [ + output_dir / "raw_findings_20.json", + output_dir / "diff_context_20.json", + output_dir / "local_findings_20.json", + output_dir / "github_review_binding_20.json", + ] + for index, path in enumerate(final_paths): + path.write_text(f"old-{index}") + real_replace = os.replace + replace_count = 0 + + def failing_replace(source: str | os.PathLike[str], destination: str | os.PathLike[str]): + nonlocal replace_count + replace_count += 1 + if replace_count == 2: + raise OSError("injected publication failure") + if replace_count == 3: + raise OSError("injected rollback failure") + real_replace(source, destination) + + monkeypatch.setattr(publication_module.os, "replace", failing_replace) + with pytest.raises(ExceptionGroup) as exc_info: + publish_experimental_review_artifacts( + publication=publication, + output_dir=str(output_dir), + pr_number="20", + ) + + errors = exc_info.value.exceptions + assert [str(error) for error in errors] == [ + "injected publication failure", + "injected rollback failure", + ] + assert final_paths[1].read_text() == "old-1" + assert not final_paths[2].exists() + assert not final_paths[3].exists() From bea7a6a8d3f29fd09a08ea433b2010acf74a861c Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 12:57:15 -0700 Subject: [PATCH 03/21] fix: align review binding integration with repository guards --- src/autoskillit/skills_extended/review-pr/SKILL.md | 5 +++-- src/autoskillit/smoke_utils/review/_publication.py | 6 ++++-- tests/_test_filter.py | 2 +- .../skills/test_resolve_review_diff_context_consumption.py | 3 ++- tests/skills/test_review_pr_artifact_idempotency.py | 1 - 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/autoskillit/skills_extended/review-pr/SKILL.md b/src/autoskillit/skills_extended/review-pr/SKILL.md index 42243007cf..e8b727879c 100644 --- a/src/autoskillit/skills_extended/review-pr/SKILL.md +++ b/src/autoskillit/skills_extended/review-pr/SKILL.md @@ -812,8 +812,9 @@ If an initially valid retained snapshot is now unavailable or differs, finalizat survivor sets from effect-producing consumers, permits only diagnostic raw/summary envelopes with empty survivors, and emits `stale_snapshot`. In that movement branch revalidation sets `FINAL_SNAPSHOT_STATE=stale` before any consumer is selected. Initial gate degradation remains -`authority_degraded` and emits `needs_human`. Do not publish diff context, local -findings, the GitHub binding marker, comments, reviews, or approvals from either state. +`authority_degraded` and emits `needs_human`. +Do not publish diff context, local findings, the GitHub binding marker, comments, +reviews, or approvals from either state. On freshness, set `COMMIT_ID="$METRICS_HEAD_SHA"` once and never query a later head to replace it. diff --git a/src/autoskillit/smoke_utils/review/_publication.py b/src/autoskillit/smoke_utils/review/_publication.py index 61ba9ec174..93207b7608 100644 --- a/src/autoskillit/smoke_utils/review/_publication.py +++ b/src/autoskillit/smoke_utils/review/_publication.py @@ -7,20 +7,22 @@ import hashlib import json import os -import re import tempfile from collections.abc import Mapping, Sequence from pathlib import Path from typing import cast +import regex as re + from autoskillit.core import ( ContainmentError, GitHubReviewReceipt, + fsync_directory, + fsync_file, github_review_receipt_filename, is_final_github_review_state, read_stable_contained_bytes, ) -from autoskillit.core.runtime.private_file import fsync_directory, fsync_file from autoskillit.smoke_utils._review_contracts import ( _is_non_empty_string, review_handoff_pair_error, diff --git a/tests/_test_filter.py b/tests/_test_filter.py index f8e23a5295..e247acd1cf 100644 --- a/tests/_test_filter.py +++ b/tests/_test_filter.py @@ -387,7 +387,7 @@ class SourceMapRejection(enum.StrEnum): "artifact_lease": frozenset({"core", "workspace"}), "worktree_gate_lease": frozenset({"core", "execution", "server", "workspace"}), "executable_binding": frozenset({"cli", "core", "execution"}), - "private_file": frozenset({"core", "execution", "pipeline"}), + "private_file": frozenset({"core", "execution", "pipeline", "smoke_utils"}), "feature_flags": frozenset( {"core", "cli", "config", "execution", "recipe", "server", "workspace"} ), diff --git a/tests/skills/test_resolve_review_diff_context_consumption.py b/tests/skills/test_resolve_review_diff_context_consumption.py index ae01d06b30..820ef38578 100644 --- a/tests/skills/test_resolve_review_diff_context_consumption.py +++ b/tests/skills/test_resolve_review_diff_context_consumption.py @@ -192,13 +192,14 @@ def test_v2_review_level_findings_enter_the_resolver_without_thread_identity() - assert 0 <= pair_check < schema_check < review_level_load rejection = step2.find("On a non-null result", pair_check) assert pair_check < rejection < schema_check + normalized = " ".join(step2.split()) for instruction in ( "diff_context_map = {}", "Pre-built context rejected:", "skip schema processing and pre-built review-level ingestion", "use the existing fallback path", ): - assert instruction in step2 + assert instruction in normalized for field in ("candidate_id", "path", "line", "message", "severity", "dimension"): assert field in step2 diff --git a/tests/skills/test_review_pr_artifact_idempotency.py b/tests/skills/test_review_pr_artifact_idempotency.py index f3e6a7f356..9fe03a0e3a 100644 --- a/tests/skills/test_review_pr_artifact_idempotency.py +++ b/tests/skills/test_review_pr_artifact_idempotency.py @@ -35,7 +35,6 @@ _RECEIPT_PREFIX = "batch_review_response_" _UPSTREAM_BINDINGS: dict[str, str] = { "FINAL_SNAPSHOT_STATE": "Bind `FINAL_SNAPSHOT_STATE` from the one printed word", - "FINALIZED_SNAPSHOT_STATE": "Update `FINALIZED_SNAPSHOT_STATE` to the state passed", "AUDIT_FINALIZATION": "AUDIT_FINALIZATION = finalize_review_audit(", "FINAL_REVIEW_FINDINGS": "`FINAL_REVIEW_FINDINGS = FILTERED_FINDINGS = survivors`", "UNPOSTABLE_FINDINGS": "`UNPOSTABLE_FINDINGS = unpostable`", From 355086e3f4ff188ab416c19d3c02fbe59d8311d8 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 13:00:56 -0700 Subject: [PATCH 04/21] test: include review publication in private file cascade expectations --- tests/test_test_filter_core_cascade.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_test_filter_core_cascade.py b/tests/test_test_filter_core_cascade.py index 077f99f3e6..696fcbd073 100644 --- a/tests/test_test_filter_core_cascade.py +++ b/tests/test_test_filter_core_cascade.py @@ -78,7 +78,9 @@ def test_feature_flags_cascade(self) -> None: def test_artifact_lease_cascade(self) -> None: assert MODULE_CASCADE_CORE["artifact_lease"] == frozenset({"core", "workspace"}) - assert MODULE_CASCADE_CORE["private_file"] == frozenset({"core", "execution", "pipeline"}) + assert MODULE_CASCADE_CORE["private_file"] == frozenset( + {"core", "execution", "pipeline", "smoke_utils"} + ) def test_branch_guard_cascade(self) -> None: assert MODULE_CASCADE_CORE["branch_guard"] == frozenset( From 15654dda3bed9c00b2577e0b63b5a8b9bf54f067 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 14:11:29 -0700 Subject: [PATCH 05/21] test: close review receipt audit coverage and import gaps --- .../core/types/test_github_review_contract.py | 5 ++-- tests/server/test_tools_pr_ops.py | 23 +++++++++++++++++++ ...resolve_review_diff_context_consumption.py | 3 ++- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/tests/core/types/test_github_review_contract.py b/tests/core/types/test_github_review_contract.py index 0e2b295b8f..d63a3e4d21 100644 --- a/tests/core/types/test_github_review_contract.py +++ b/tests/core/types/test_github_review_contract.py @@ -26,7 +26,6 @@ ReviewOperationState, ReviewReconciliationResult, ReviewResponseClass, - github_review_receipt_filename, is_final_github_review_state, is_valid_github_review_head_sha, is_valid_github_review_logical_iteration, @@ -156,7 +155,9 @@ def test_contract_module_exports_exact_public_surface() -> None: def test_github_review_receipt_filename_formats_the_server_artifact_name() -> None: - assert github_review_receipt_filename(42) == "batch_review_response_42.json" + from autoskillit.core import github_review_receipt_filename + + assert github_review_receipt_filename(7) == "batch_review_response_7.json" def test_contract_module_imports_only_stdlib_and_anchor_sibling() -> None: diff --git a/tests/server/test_tools_pr_ops.py b/tests/server/test_tools_pr_ops.py index aaa0f19299..bb5861e420 100644 --- a/tests/server/test_tools_pr_ops.py +++ b/tests/server/test_tools_pr_ops.py @@ -206,6 +206,29 @@ async def test_ledger_receipt_is_authoritative( assert poster.receipt_lookups == [receipt.operation_key] +@pytest.mark.anyio +async def test_verify_review_receipt_reports_unexpected_error( + tool_ctx_kitchen_open, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + receipt = authoritative_receipt() + poster = _FakeReviewPoster() + verify = Mock(side_effect=RuntimeError("ledger verification failed")) + monkeypatch.setattr(poster, "verify_receipt", verify) + monkeypatch.setattr(tool_ctx_kitchen_open, "github_review_poster", poster) + monkeypatch.setattr(tools_pr_ops, "_get_ctx", lambda: tool_ctx_kitchen_open) + receipt_path = _write_receipt(tmp_path, receipt.to_dict()) + + result = await _verify_receipt(tmp_path, receipt_path) + + assert result == { + "reviews_posted": "false", + "rejection_reason": "unexpected_error", + } + verify.assert_called_once_with(receipt.operation_key) + + @pytest.mark.anyio async def test_handcrafted_receipt_without_ledger_row_is_rejected( tool_ctx_kitchen_open, diff --git a/tests/skills/test_resolve_review_diff_context_consumption.py b/tests/skills/test_resolve_review_diff_context_consumption.py index 820ef38578..a9c5b9f2c0 100644 --- a/tests/skills/test_resolve_review_diff_context_consumption.py +++ b/tests/skills/test_resolve_review_diff_context_consumption.py @@ -6,7 +6,6 @@ import pytest from autoskillit.smoke_utils import ( - github_review_handoff_error, prepare_experimental_review_publication, publish_experimental_review_artifacts, ) @@ -267,6 +266,8 @@ def test_actual_publisher_output_matches_resolve_review_path_line_boundary( tmp_path: Path, mode: str, ) -> None: + from autoskillit.smoke_utils import github_review_handoff_error + findings = [ { "file": "src/reach.py", From d729cc07f14f9ee99866ae37be53979bb665980c Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 15:14:03 -0700 Subject: [PATCH 06/21] fix(review): reject invalid receipt filename identifiers --- src/autoskillit/core/types/github/_type_github_review.py | 2 ++ tests/core/types/test_github_review_contract.py | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/src/autoskillit/core/types/github/_type_github_review.py b/src/autoskillit/core/types/github/_type_github_review.py index d8679217b3..19a57531f6 100644 --- a/src/autoskillit/core/types/github/_type_github_review.py +++ b/src/autoskillit/core/types/github/_type_github_review.py @@ -63,6 +63,8 @@ def is_valid_github_review_repository(value: object) -> bool: def github_review_receipt_filename(pr_number: int) -> str: """Return the server receipt filename for a pull request.""" + if not _is_positive_int(pr_number): + raise ValueError("pr_number must be a positive integer") return f"batch_review_response_{pr_number}.json" diff --git a/tests/core/types/test_github_review_contract.py b/tests/core/types/test_github_review_contract.py index d63a3e4d21..32d2be3a16 100644 --- a/tests/core/types/test_github_review_contract.py +++ b/tests/core/types/test_github_review_contract.py @@ -160,6 +160,14 @@ def test_github_review_receipt_filename_formats_the_server_artifact_name() -> No assert github_review_receipt_filename(7) == "batch_review_response_7.json" +@pytest.mark.parametrize("pr_number", [0, -1, True, "7", 7.0]) +def test_github_review_receipt_filename_rejects_invalid_pr_number(pr_number: object) -> None: + from autoskillit.core import github_review_receipt_filename + + with pytest.raises(ValueError, match="pr_number must be a positive integer"): + github_review_receipt_filename(pr_number) # type: ignore[arg-type] + + def test_contract_module_imports_only_stdlib_and_anchor_sibling() -> None: """The review contracts depend only on stdlib and their IL-0 anchor type.""" source_path = Path(inspect.getsourcefile(_type_github_review) or "") From 34510d5fb30b71c08164db44a439eec273c5c06a Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 15:17:43 -0700 Subject: [PATCH 07/21] fix(review): share strict review JSON decoding --- src/autoskillit/core/__init__.pyi | 1 + .../core/types/github/_type_github_review.py | 19 +++++++++++++++++ src/autoskillit/server/tools/tools_pr_ops.py | 21 +++---------------- .../smoke_utils/review/_publication.py | 3 ++- .../core/types/test_github_review_contract.py | 1 + .../test_experimental_review_publication.py | 9 ++++---- 6 files changed, 31 insertions(+), 23 deletions(-) diff --git a/src/autoskillit/core/__init__.pyi b/src/autoskillit/core/__init__.pyi index 321a886e0c..b238e72e27 100644 --- a/src/autoskillit/core/__init__.pyi +++ b/src/autoskillit/core/__init__.pyi @@ -1430,6 +1430,7 @@ from .types import ( from .types import compute_remaining as compute_remaining from .types import compute_runtime_binding_digest as compute_runtime_binding_digest from .types import context_admission_envelope_header as context_admission_envelope_header +from .types import decode_github_review_json_object as decode_github_review_json_object from .types import ( decode_stored_context_admission_envelope as decode_stored_context_admission_envelope, ) diff --git a/src/autoskillit/core/types/github/_type_github_review.py b/src/autoskillit/core/types/github/_type_github_review.py index 19a57531f6..b849b0d49a 100644 --- a/src/autoskillit/core/types/github/_type_github_review.py +++ b/src/autoskillit/core/types/github/_type_github_review.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import re from collections.abc import Mapping from dataclasses import asdict, dataclass @@ -21,6 +22,7 @@ "ReviewOperationState", "ReviewReconciliationResult", "ReviewResponseClass", + "decode_github_review_json_object", "github_review_receipt_filename", "is_final_github_review_state", "is_valid_github_review_head_sha", @@ -68,6 +70,23 @@ def github_review_receipt_filename(pr_number: int) -> str: return f"batch_review_response_{pr_number}.json" +def decode_github_review_json_object(data: bytes) -> dict[str, object]: + """Decode a review JSON object, rejecting duplicate keys and preserving all fields.""" + + def unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON object key: {key!r}") + result[key] = value + return result + + value = json.loads(data, object_pairs_hook=unique_object) + if not isinstance(value, dict): + raise TypeError("review receipt must be a JSON object") + return value + + class ReviewOperationState(StrEnum): DRY_RUN = "DRY_RUN" PREPARED = "PREPARED" diff --git a/src/autoskillit/server/tools/tools_pr_ops.py b/src/autoskillit/server/tools/tools_pr_ops.py index e001d73b31..1743244357 100644 --- a/src/autoskillit/server/tools/tools_pr_ops.py +++ b/src/autoskillit/server/tools/tools_pr_ops.py @@ -8,7 +8,6 @@ import time from dataclasses import replace from pathlib import Path -from typing import Any import structlog from fastmcp import Context @@ -23,6 +22,7 @@ ReviewOperationState, ReviewResponseClass, atomic_write, + decode_github_review_json_object, destination_location, get_logger, github_review_receipt_filename, @@ -91,21 +91,6 @@ def _map_pr_view_reviews(data: dict) -> list: ] -def _load_unique_json_object(data: bytes) -> dict[str, object]: - def _unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: - result: dict[str, Any] = {} - for key, value in pairs: - if key in result: - raise ValueError(f"duplicate JSON object key: {key!r}") - result[key] = value - return result - - value = json.loads(data, object_pairs_hook=_unique_object) - if not isinstance(value, dict): - raise TypeError("review receipt must be a JSON object") - return value - - def _review_verification_result(rejection: str | None) -> str: if rejection is None: return json.dumps({"reviews_posted": "true"}) @@ -122,7 +107,7 @@ def _load_receipt_payload(receipt: Path, root: Path) -> dict[str, object] | str: except (ContainmentError, OSError): return "receipt_unreadable" try: - return _load_unique_json_object(receipt_bytes) + return decode_github_review_json_object(receipt_bytes) except (TypeError, ValueError): return "receipt_malformed" @@ -481,7 +466,7 @@ async def post_pr_review( max_size_bytes=1_000_000, ) authority = DiffAnchorAuthority.from_wire( - _load_unique_json_object(authority_bytes) + decode_github_review_json_object(authority_bytes) ) except (ContainmentError, OSError) as exc: return _review_post_error( diff --git a/src/autoskillit/smoke_utils/review/_publication.py b/src/autoskillit/smoke_utils/review/_publication.py index 93207b7608..bbba786b3e 100644 --- a/src/autoskillit/smoke_utils/review/_publication.py +++ b/src/autoskillit/smoke_utils/review/_publication.py @@ -17,6 +17,7 @@ from autoskillit.core import ( ContainmentError, GitHubReviewReceipt, + decode_github_review_json_object, fsync_directory, fsync_file, github_review_receipt_filename, @@ -273,7 +274,7 @@ def _load_github_review_receipt( except (ContainmentError, OSError) as exc: raise ValueError(f"receipt unreadable: {exc}") from exc try: - receipt = GitHubReviewReceipt.from_wire(json.loads(data)) + receipt = GitHubReviewReceipt.from_wire(decode_github_review_json_object(data)) except ( KeyError, TypeError, diff --git a/tests/core/types/test_github_review_contract.py b/tests/core/types/test_github_review_contract.py index 32d2be3a16..e89b7e0436 100644 --- a/tests/core/types/test_github_review_contract.py +++ b/tests/core/types/test_github_review_contract.py @@ -57,6 +57,7 @@ "ReviewReconciliationResult", "ReviewFindingDispositionKind", "github_review_receipt_filename", + "decode_github_review_json_object", "is_final_github_review_state", "is_valid_github_review_head_sha", "is_valid_github_review_logical_iteration", diff --git a/tests/smoke_utils/test_experimental_review_publication.py b/tests/smoke_utils/test_experimental_review_publication.py index 48f4c4da59..49da16ada6 100644 --- a/tests/smoke_utils/test_experimental_review_publication.py +++ b/tests/smoke_utils/test_experimental_review_publication.py @@ -764,6 +764,7 @@ def test_github_binding_marker_is_published_last_and_binds_server_receipt( [ ("missing", "receipt unreadable"), ("invalid_json", "receipt malformed"), + ("duplicate_keys", "receipt malformed"), ("empty", "receipt malformed"), ("overflow_created_at", "receipt malformed"), ("dry_run", "receipt is not a final successful review"), @@ -797,10 +798,10 @@ def test_github_publication_rejects_unbindable_receipt( payload["state"] = "DRY_RUN" elif receipt_case == "missing_review_id": payload["review_id"] = None - _write_receipt_payload( - output_dir, - json.dumps(payload, sort_keys=True, separators=(",", ":")).encode(), - ) + data = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + if receipt_case == "duplicate_keys": + data = b'{"pr_number":42,' + data[1:] + _write_receipt_payload(output_dir, data) with pytest.raises(ValueError, match=f"^{re.escape(expected_error)}"): publish_experimental_review_artifacts( From f3d70a1a0cf997fcba3e612d99265b247082a6e8 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 15:18:43 -0700 Subject: [PATCH 08/21] fix(review): check review skill receipt names against the formatter --- tests/arch/test_review_receipt_single_writer.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/arch/test_review_receipt_single_writer.py b/tests/arch/test_review_receipt_single_writer.py index 22bda16920..2356775851 100644 --- a/tests/arch/test_review_receipt_single_writer.py +++ b/tests/arch/test_review_receipt_single_writer.py @@ -2,10 +2,13 @@ from __future__ import annotations +import re from pathlib import Path import pytest +from autoskillit.core import github_review_receipt_filename + pytestmark = [pytest.mark.layer("arch"), pytest.mark.small] _SOURCE_ROOT = Path(__file__).resolve().parents[2] / "src" / "autoskillit" @@ -25,6 +28,16 @@ def test_receipt_filename_literal_is_owned_by_its_core_definition() -> None: } +def test_review_skill_receipt_names_match_the_core_formatter() -> None: + skill = (_SOURCE_ROOT / "skills_extended" / "review-pr" / "SKILL.md").read_text() + templates = re.findall(r"`(batch_review_response_[^`]*\.json)`", skill) + assert templates + for template in templates: + assert template.replace("${pr_number}", "42").replace( + "{pr_number}", "42" + ) == github_review_receipt_filename(42) + + def test_receipt_filename_formatter_has_only_the_three_declared_call_sites() -> None: assert _files_containing("github_review_receipt_filename") == { "core/types/github/_type_github_review.py", From 356167cfa7a0345b97cb752a604bd06914f7134e Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 15:20:00 -0700 Subject: [PATCH 09/21] fix(review): preserve every publication rollback failure --- .../smoke_utils/review/_publication.py | 22 +++++++++---------- .../test_review_publication_durability.py | 13 ++++++----- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/src/autoskillit/smoke_utils/review/_publication.py b/src/autoskillit/smoke_utils/review/_publication.py index bbba786b3e..eb7f0b4e0f 100644 --- a/src/autoskillit/smoke_utils/review/_publication.py +++ b/src/autoskillit/smoke_utils/review/_publication.py @@ -366,16 +366,16 @@ def _restore_publication_files( prior_bytes: Mapping[str, bytes | None], changed: Sequence[str], marker_names: tuple[str, ...], -) -> OSError | None: +) -> list[OSError]: try: for name in marker_names: if final_paths[name].exists(): final_paths[name].unlink() fsync_directory(root) except OSError as error: - return error + return [error] - rollback_error: OSError | None = None + rollback_errors: list[OSError] = [] payload_names = [name for name in dict.fromkeys(changed) if name not in marker_names] for name in payload_names: try: @@ -390,11 +390,11 @@ def _restore_publication_files( rollback_path.unlink(missing_ok=True) fsync_directory(root) except OSError as error: - rollback_error = error + rollback_errors.append(error) marker_names_to_restore = ( [name for name in marker_names if prior_bytes[name] is not None] - if rollback_error is None + if not rollback_errors else [] ) for name in marker_names_to_restore: @@ -408,8 +408,8 @@ def _restore_publication_files( rollback_path.unlink(missing_ok=True) fsync_directory(root) except OSError as error: - rollback_error = error - return rollback_error + rollback_errors.append(error) + return rollback_errors def publish_experimental_review_artifacts( @@ -483,7 +483,7 @@ def publish_experimental_review_artifacts( except Exception as publication_error: for temporary_path in staged.values(): temporary_path.unlink(missing_ok=True) - rollback_error = ( + rollback_errors = ( _restore_publication_files( root, final_paths, @@ -492,12 +492,12 @@ def publish_experimental_review_artifacts( marker_names, ) if changed - else None + else [] ) - if rollback_error is not None: + if rollback_errors: raise ExceptionGroup( "publication failed and rollback was incomplete", - [publication_error, rollback_error], + [publication_error, *rollback_errors], ) raise RuntimeError( "experimental review artifact publication failed" diff --git a/tests/smoke_utils/test_review_publication_durability.py b/tests/smoke_utils/test_review_publication_durability.py index addbcc0692..8c248fded6 100644 --- a/tests/smoke_utils/test_review_publication_durability.py +++ b/tests/smoke_utils/test_review_publication_durability.py @@ -559,9 +559,11 @@ def failing_replace(source: str | os.PathLike[str], destination: str | os.PathLi assert not list(output_dir.glob(".*.tmp")) +@pytest.mark.parametrize("publication_failure_at", [2, 3]) def test_experimental_publication_preserves_publication_and_rollback_failures( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, + publication_failure_at: int, ) -> None: import autoskillit.smoke_utils.review._publication as publication_module @@ -582,10 +584,10 @@ def test_experimental_publication_preserves_publication_and_rollback_failures( def failing_replace(source: str | os.PathLike[str], destination: str | os.PathLike[str]): nonlocal replace_count replace_count += 1 - if replace_count == 2: + if replace_count == publication_failure_at: raise OSError("injected publication failure") - if replace_count == 3: - raise OSError("injected rollback failure") + if replace_count > publication_failure_at: + raise OSError(f"injected rollback failure {replace_count - publication_failure_at}") real_replace(source, destination) monkeypatch.setattr(publication_module.os, "replace", failing_replace) @@ -599,8 +601,9 @@ def failing_replace(source: str | os.PathLike[str], destination: str | os.PathLi errors = exc_info.value.exceptions assert [str(error) for error in errors] == [ "injected publication failure", - "injected rollback failure", + *[f"injected rollback failure {index}" for index in range(1, publication_failure_at)], ] - assert final_paths[1].read_text() == "old-1" + if publication_failure_at == 2: + assert final_paths[1].read_text() == "old-1" assert not final_paths[2].exists() assert not final_paths[3].exists() From a791faa8a374b17216980bb4df51bb6be0025776 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 15:21:34 -0700 Subject: [PATCH 10/21] refactor(review): remove redundant rollback payload deduplication --- src/autoskillit/smoke_utils/review/_publication.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/autoskillit/smoke_utils/review/_publication.py b/src/autoskillit/smoke_utils/review/_publication.py index eb7f0b4e0f..d58aef5765 100644 --- a/src/autoskillit/smoke_utils/review/_publication.py +++ b/src/autoskillit/smoke_utils/review/_publication.py @@ -376,7 +376,7 @@ def _restore_publication_files( return [error] rollback_errors: list[OSError] = [] - payload_names = [name for name in dict.fromkeys(changed) if name not in marker_names] + payload_names = [name for name in changed if name not in marker_names] for name in payload_names: try: previous = prior_bytes[name] From be95d898f3b5668eaf69b26e09cc50d70da88cf4 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 15:28:30 -0700 Subject: [PATCH 11/21] fix(review): validate publication diff context before binding receipts --- .../smoke_utils/review/_publication.py | 31 +++++++++++++------ .../test_experimental_review_publication.py | 19 ++++++++++++ 2 files changed, 41 insertions(+), 9 deletions(-) diff --git a/src/autoskillit/smoke_utils/review/_publication.py b/src/autoskillit/smoke_utils/review/_publication.py index d58aef5765..9be9c81c2d 100644 --- a/src/autoskillit/smoke_utils/review/_publication.py +++ b/src/autoskillit/smoke_utils/review/_publication.py @@ -295,6 +295,26 @@ def _load_github_review_receipt( return receipt +def _github_review_binding( + root: Path, + identifier: str, + diff_context: object, +) -> dict[str, object]: + if not isinstance(diff_context, Mapping) or not _is_non_empty_string( + diff_context.get("_head_sha") + ): + raise ValueError("diff_context must contain a non-empty _head_sha") + receipt = _load_github_review_receipt( + root, + identifier, + head_sha=cast(str, diff_context["_head_sha"]), + ) + return { + **{key: diff_context[key] for key in _GENERATION_IDENTITY_KEYS if key in diff_context}, + "receipt_operation_key": receipt.operation_key, + } + + def _publication_order( publication: Mapping[str, object], root: Path, @@ -323,16 +343,9 @@ def _publication_order( if publication.get("state") == "complete" and mode == "github": if order != ["raw_findings", "diff_context"]: raise ValueError("complete github publication requires raw findings and diff context") - diff_context = cast(Mapping[str, object], artifacts["diff_context"]) - receipt = _load_github_review_receipt( - root, - identifier, - head_sha=cast(str, diff_context["_head_sha"]), + artifacts["github_review_binding"] = _github_review_binding( + root, identifier, artifacts["diff_context"] ) - artifacts["github_review_binding"] = { - **{key: diff_context[key] for key in _GENERATION_IDENTITY_KEYS if key in diff_context}, - "receipt_operation_key": receipt.operation_key, - } order.append("github_review_binding") return order, artifacts diff --git a/tests/smoke_utils/test_experimental_review_publication.py b/tests/smoke_utils/test_experimental_review_publication.py index 49da16ada6..c6d4e460d3 100644 --- a/tests/smoke_utils/test_experimental_review_publication.py +++ b/tests/smoke_utils/test_experimental_review_publication.py @@ -41,6 +41,25 @@ def _write_receipt_payload(output_dir: Path, payload: bytes, *, pr_number: str = return receipt_path +@pytest.mark.parametrize("diff_context", [[], {}, {"_head_sha": 42}]) +def test_github_publication_rejects_malformed_diff_context_before_writes( + tmp_path: Path, + diff_context: object, +) -> None: + publication = _prepared_publication("github") + publication["artifacts"]["diff_context"] = diff_context + output_dir = tmp_path / "review-output" + + with pytest.raises(ValueError, match="diff_context must contain a non-empty _head_sha"): + publish_experimental_review_artifacts( + publication=publication, + output_dir=str(output_dir), + pr_number="42", + ) + + assert not output_dir.exists() + + def _authority(right_side_lines: dict[str, list[int]]) -> DiffAnchorAuthority: return DiffAnchorAuthority.authoritative( repository="openai/autoskillit", From e1697d8689167bea4ed27483407a49c51b29d618 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 15:32:18 -0700 Subject: [PATCH 12/21] fix(review): enforce mode-specific publication markers --- .../smoke_utils/review/_publication.py | 13 ++++++----- .../test_experimental_review_publication.py | 22 +++++++++++++++++++ 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/src/autoskillit/smoke_utils/review/_publication.py b/src/autoskillit/smoke_utils/review/_publication.py index 9be9c81c2d..69eb23e83c 100644 --- a/src/autoskillit/smoke_utils/review/_publication.py +++ b/src/autoskillit/smoke_utils/review/_publication.py @@ -42,6 +42,7 @@ "local_findings": "local_findings_{pr_number}.json", "github_review_binding": "github_review_binding_{pr_number}.json", } +_MARKER_ARTIFACT_NAMES = ("local_findings", "github_review_binding") _GENERATION_IDENTITY_KEYS = ( "_head_sha", "_base_sha", @@ -331,13 +332,15 @@ def _publication_order( raise ValueError("publication contains an unknown artifact") if not order or order[0] != "raw_findings": raise ValueError("raw_findings must be the first publication") - if "local_findings" in order and order[-1] != "local_findings": - raise ValueError("local_findings must be the final publication marker") - if "github_review_binding" in order and order[-1] != "github_review_binding": - raise ValueError("github_review_binding must be the final publication marker") + misplaced_markers = set(order[:-1]).intersection(_MARKER_ARTIFACT_NAMES) + if misplaced_markers: + raise ValueError(f"{min(misplaced_markers)} must be the final publication marker") mode = publication.get("mode") if mode not in {"github", "local"}: raise ValueError("publication mode must be 'github' or 'local'") + incompatible_marker = "github_review_binding" if mode == "local" else "local_findings" + if incompatible_marker in order: + raise ValueError(f"{incompatible_marker} is incompatible with {mode} mode") artifacts: dict[str, object] = dict(raw_artifacts) if publication.get("state") == "complete" and mode == "github": @@ -449,7 +452,7 @@ def publish_experimental_review_artifacts( name: root / filename.format(pr_number=identifier) for name, filename in _PUBLICATION_FILENAMES.items() } - marker_names = ("local_findings", "github_review_binding") + marker_names = _MARKER_ARTIFACT_NAMES retired_names = [ name for name in final_paths if name not in order and name not in marker_names ] diff --git a/tests/smoke_utils/test_experimental_review_publication.py b/tests/smoke_utils/test_experimental_review_publication.py index c6d4e460d3..8f0e7c42da 100644 --- a/tests/smoke_utils/test_experimental_review_publication.py +++ b/tests/smoke_utils/test_experimental_review_publication.py @@ -60,6 +60,28 @@ def test_github_publication_rejects_malformed_diff_context_before_writes( assert not output_dir.exists() +@pytest.mark.parametrize("mode", ["local", "github"]) +def test_publication_rejects_mode_incompatible_marker_before_writes( + tmp_path: Path, + mode: str, +) -> None: + publication = _prepared_publication(mode) + marker = "github_review_binding" if mode == "local" else "local_findings" + publication["artifacts"].pop("local_findings", None) + publication["artifacts"][marker] = {} + publication["artifact_order"] = ["raw_findings", "diff_context", marker] + output_dir = tmp_path / "review-output" + + with pytest.raises(ValueError, match=f"{marker} is incompatible with {mode} mode"): + publish_experimental_review_artifacts( + publication=publication, + output_dir=str(output_dir), + pr_number="42", + ) + + assert not output_dir.exists() + + def _authority(right_side_lines: dict[str, list[int]]) -> DiffAnchorAuthority: return DiffAnchorAuthority.authoritative( repository="openai/autoskillit", From 560c43c9a805b16c59f987f9c96b673b060c8cd5 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 15:33:26 -0700 Subject: [PATCH 13/21] test(review): allow additional receipt formatter callers --- tests/arch/test_review_receipt_single_writer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/arch/test_review_receipt_single_writer.py b/tests/arch/test_review_receipt_single_writer.py index 2356775851..2c0ea5572e 100644 --- a/tests/arch/test_review_receipt_single_writer.py +++ b/tests/arch/test_review_receipt_single_writer.py @@ -38,9 +38,9 @@ def test_review_skill_receipt_names_match_the_core_formatter() -> None: ) == github_review_receipt_filename(42) -def test_receipt_filename_formatter_has_only_the_three_declared_call_sites() -> None: - assert _files_containing("github_review_receipt_filename") == { +def test_receipt_filename_formatter_is_used_by_its_current_owners() -> None: + assert { "core/types/github/_type_github_review.py", "server/tools/tools_pr_ops.py", "smoke_utils/review/_publication.py", - } + } <= _files_containing("github_review_receipt_filename") From 28336527e5fbbd3b00ebe51006b9ef8d83d03a53 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 15:36:23 -0700 Subject: [PATCH 14/21] refactor(review): move receipt fixtures out of execution test fakes --- tests/_github_review_fixtures.py | 39 +++++++++++++++++++ tests/execution/github_review/fakes.py | 33 +--------------- tests/server/test_tools_pr_ops.py | 2 +- ...resolve_review_diff_context_consumption.py | 2 +- .../_review_publication_fixtures.py | 2 +- .../test_experimental_review_publication.py | 2 +- 6 files changed, 44 insertions(+), 36 deletions(-) create mode 100644 tests/_github_review_fixtures.py diff --git a/tests/_github_review_fixtures.py b/tests/_github_review_fixtures.py new file mode 100644 index 0000000000..0000da2e19 --- /dev/null +++ b/tests/_github_review_fixtures.py @@ -0,0 +1,39 @@ +"""Core-only receipt fixtures shared across review test layers.""" + +from dataclasses import replace +from typing import Any + +from autoskillit.core import ( + GitHubReviewReceipt, + ReviewOperationState, + ReviewReconciliationResult, + ReviewResponseClass, +) + + +def authoritative_receipt(**overrides: Any) -> GitHubReviewReceipt: + """Return the canonical server receipt used by verifier and publisher tests.""" + receipt = GitHubReviewReceipt( + schema_version=1, + operation_key="f" * 64, + repository="owner/repo", + pr_number=7, + head_sha="a" * 40, + logical_iteration="review-pr:1", + requested_event="COMMENT", + effective_event="COMMENT", + requested_body_digest="b" * 64, + effective_body_digest="b" * 64, + canonical_finding_digest="c" * 64, + state=ReviewOperationState.SUCCEEDED, + response_class=ReviewResponseClass.SUCCESS, + review_id=901, + comment_ids=(), + canonical_finding_count=0, + reconciliation_result=ReviewReconciliationResult.NOT_NEEDED, + finding_dispositions=(), + created_at=1.0, + updated_at=2.0, + final_attempt_digest="d" * 64, + ) + return replace(receipt, **overrides) diff --git a/tests/execution/github_review/fakes.py b/tests/execution/github_review/fakes.py index 7e35e4250e..1549ab78ee 100644 --- a/tests/execution/github_review/fakes.py +++ b/tests/execution/github_review/fakes.py @@ -3,17 +3,14 @@ from __future__ import annotations import asyncio -from dataclasses import dataclass, replace +from dataclasses import dataclass from pathlib import Path from typing import Any from autoskillit.core import ( DiffAnchorAuthority, GitHubReviewComment, - GitHubReviewReceipt, GitHubReviewRequest, - ReviewOperationState, - ReviewReconciliationResult, ReviewResponseClass, ) from autoskillit.execution import ( @@ -301,31 +298,3 @@ def _poster( review_comment_cap=review_comment_cap, wall_clock=clock, ) - - -def authoritative_receipt(**overrides: Any) -> GitHubReviewReceipt: - """Return the canonical server receipt used by verifier and publisher tests.""" - receipt = GitHubReviewReceipt( - schema_version=1, - operation_key="f" * 64, - repository="owner/repo", - pr_number=7, - head_sha="a" * 40, - logical_iteration="review-pr:1", - requested_event="COMMENT", - effective_event="COMMENT", - requested_body_digest="b" * 64, - effective_body_digest="b" * 64, - canonical_finding_digest="c" * 64, - state=ReviewOperationState.SUCCEEDED, - response_class=ReviewResponseClass.SUCCESS, - review_id=901, - comment_ids=(), - canonical_finding_count=0, - reconciliation_result=ReviewReconciliationResult.NOT_NEEDED, - finding_dispositions=(), - created_at=1.0, - updated_at=2.0, - final_attempt_digest="d" * 64, - ) - return replace(receipt, **overrides) diff --git a/tests/server/test_tools_pr_ops.py b/tests/server/test_tools_pr_ops.py index bb5861e420..76beb04d36 100644 --- a/tests/server/test_tools_pr_ops.py +++ b/tests/server/test_tools_pr_ops.py @@ -28,7 +28,7 @@ post_pr_review, verify_review_receipt, ) -from tests.execution.github_review.fakes import authoritative_receipt +from tests._github_review_fixtures import authoritative_receipt from tests.server._recipe_fixture_parameters import ( IMPLEMENTATION_READY_PARAMETERS, ) diff --git a/tests/skills/test_resolve_review_diff_context_consumption.py b/tests/skills/test_resolve_review_diff_context_consumption.py index a9c5b9f2c0..0d544aa647 100644 --- a/tests/skills/test_resolve_review_diff_context_consumption.py +++ b/tests/skills/test_resolve_review_diff_context_consumption.py @@ -9,7 +9,7 @@ prepare_experimental_review_publication, publish_experimental_review_artifacts, ) -from tests.execution.github_review.fakes import authoritative_receipt +from tests._github_review_fixtures import authoritative_receipt pytestmark = [pytest.mark.layer("skills"), pytest.mark.medium] diff --git a/tests/smoke_utils/_review_publication_fixtures.py b/tests/smoke_utils/_review_publication_fixtures.py index 722b7b8773..1a1142c3f4 100644 --- a/tests/smoke_utils/_review_publication_fixtures.py +++ b/tests/smoke_utils/_review_publication_fixtures.py @@ -11,7 +11,7 @@ prepare_experimental_review_publication, publish_experimental_review_artifacts, ) -from tests.execution.github_review.fakes import authoritative_receipt +from tests._github_review_fixtures import authoritative_receipt _REVIEW_HEAD = "a" * 40 _REVIEW_BASE = "b" * 40 diff --git a/tests/smoke_utils/test_experimental_review_publication.py b/tests/smoke_utils/test_experimental_review_publication.py index 8f0e7c42da..75a81ab23c 100644 --- a/tests/smoke_utils/test_experimental_review_publication.py +++ b/tests/smoke_utils/test_experimental_review_publication.py @@ -17,7 +17,7 @@ render_unpostable_review_section, validate_experimental_auditor_outputs, ) -from tests.execution.github_review.fakes import authoritative_receipt +from tests._github_review_fixtures import authoritative_receipt from tests.smoke_utils._experimental_helpers import ( _experimental_candidate, ) From eaf4c19acc5a21a1c625960ab866c8d06438d16e Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 15:37:32 -0700 Subject: [PATCH 15/21] test(review): cover malformed receipt verification requests --- .../test_review_receipt_production_seam.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/server/test_review_receipt_production_seam.py b/tests/server/test_review_receipt_production_seam.py index 9f284beeff..037cffa3be 100644 --- a/tests/server/test_review_receipt_production_seam.py +++ b/tests/server/test_review_receipt_production_seam.py @@ -206,6 +206,31 @@ async def test_new_post_without_republication_is_rejected_by_consumer( ) +@pytest.mark.parametrize( + "overrides", + [ + {"cwd": "relative"}, + {"repository": "invalid"}, + {"pr_number": True}, + {"pr_number": 0}, + {"head_sha": "invalid"}, + {"logical_iteration": "review-pr"}, + {"post_state": "PREPARED"}, + ], +) +async def test_verifier_rejects_malformed_request_fields( + overrides: dict[str, object], + tool_ctx_kitchen_open: Any, + tmp_path: Path, +) -> None: + _, receipt_path = _review_paths(tmp_path) + + assert await _verify_review(tmp_path, receipt_path, **overrides) == { + "reviews_posted": "false", + "rejection_reason": "invalid_request", + } + + def _configure_verifier_rejection( case: str, tmp_path: Path, From 0a9849f64d8c833a18b3bbfc7651be286749fa99 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 15:38:15 -0700 Subject: [PATCH 16/21] test(review): detect in-place server receipt rewrites --- tests/server/test_review_receipt_production_seam.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/server/test_review_receipt_production_seam.py b/tests/server/test_review_receipt_production_seam.py index 037cffa3be..0581db98a5 100644 --- a/tests/server/test_review_receipt_production_seam.py +++ b/tests/server/test_review_receipt_production_seam.py @@ -151,11 +151,12 @@ async def test_server_receipt_survives_github_publication_and_verifies( assert receipt_path.is_file() assert receipt_path.stat().st_mode & 0o777 == 0o600 receipt_bytes = receipt_path.read_bytes() - receipt_inode = receipt_path.stat().st_ino + receipt_stat = receipt_path.stat() publication = _publish(output_dir, "github", written_at="t0") assert receipt_path.read_bytes() == receipt_bytes - assert receipt_path.stat().st_ino == receipt_inode + assert receipt_path.stat().st_ino == receipt_stat.st_ino + assert receipt_path.stat().st_mtime_ns == receipt_stat.st_mtime_ns assert await _verify_review(tmp_path, receipt_path, post_state=expected_state) == { "reviews_posted": "true" } From 24ff7d851cf1a082f2ca97937802171a16dc0d6f Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 15:38:54 -0700 Subject: [PATCH 17/21] refactor(review): keep handoff helper imports at module scope --- tests/server/test_review_receipt_production_seam.py | 3 +-- tests/skills/test_resolve_review_diff_context_consumption.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/server/test_review_receipt_production_seam.py b/tests/server/test_review_receipt_production_seam.py index 0581db98a5..8d993c25c9 100644 --- a/tests/server/test_review_receipt_production_seam.py +++ b/tests/server/test_review_receipt_production_seam.py @@ -13,6 +13,7 @@ import autoskillit.server.tools.tools_pr_ops as tools_pr_ops from autoskillit.server.tools.tools_pr_ops import post_pr_review, verify_review_receipt from autoskillit.smoke_utils import ( + github_review_handoff_error, prepare_experimental_review_publication, publish_experimental_review_artifacts, ) @@ -119,8 +120,6 @@ def _publish( def _handoff_error(output_dir: Path) -> str | None: - from autoskillit.smoke_utils import github_review_handoff_error - return github_review_handoff_error(output_dir=str(output_dir), pr_number="42") diff --git a/tests/skills/test_resolve_review_diff_context_consumption.py b/tests/skills/test_resolve_review_diff_context_consumption.py index 0d544aa647..075dd37087 100644 --- a/tests/skills/test_resolve_review_diff_context_consumption.py +++ b/tests/skills/test_resolve_review_diff_context_consumption.py @@ -6,6 +6,7 @@ import pytest from autoskillit.smoke_utils import ( + github_review_handoff_error, prepare_experimental_review_publication, publish_experimental_review_artifacts, ) @@ -266,8 +267,6 @@ def test_actual_publisher_output_matches_resolve_review_path_line_boundary( tmp_path: Path, mode: str, ) -> None: - from autoskillit.smoke_utils import github_review_handoff_error - findings = [ { "file": "src/reach.py", From cf2663849984fbb6102af732046b04d71e3683e7 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 15:39:36 -0700 Subject: [PATCH 18/21] refactor(review): reuse shared publication fixture setup --- .../test_review_receipt_production_seam.py | 31 ++----------------- 1 file changed, 2 insertions(+), 29 deletions(-) diff --git a/tests/server/test_review_receipt_production_seam.py b/tests/server/test_review_receipt_production_seam.py index 8d993c25c9..b41f155231 100644 --- a/tests/server/test_review_receipt_production_seam.py +++ b/tests/server/test_review_receipt_production_seam.py @@ -12,22 +12,18 @@ import autoskillit.server.tools.tools_pr_ops as tools_pr_ops from autoskillit.server.tools.tools_pr_ops import post_pr_review, verify_review_receipt -from autoskillit.smoke_utils import ( - github_review_handoff_error, - prepare_experimental_review_publication, - publish_experimental_review_artifacts, -) +from autoskillit.smoke_utils import github_review_handoff_error from tests.execution.github_review.fakes import ( CreateOutcome, ManualClock, StatefulReviewGateway, _poster, ) +from tests.smoke_utils._review_publication_fixtures import _publish pytestmark = [pytest.mark.layer("server"), pytest.mark.anyio, pytest.mark.medium] _HEAD_SHA = "a" * 40 -_BASE_SHA = "b" * 40 _LOGICAL_ITERATION = "review-pr:2" @@ -96,29 +92,6 @@ async def _verify_review( return json.loads(await verify_review_receipt(**arguments)) # type: ignore[arg-type] -def _publish( - output_dir: Path, - mode: str, - *, - written_at: str, - fresh: bool = True, -) -> dict[str, Any]: - publication = prepare_experimental_review_publication( - raw_ledger={"candidate_records": []}, - survivors=[], - snapshot={"head_sha": _HEAD_SHA, "base_sha": _BASE_SHA}, - annotation_generation_id="g" * 64, - mode=mode, - snapshot_is_fresh=fresh, - handoff_metadata={"written_at": written_at, "schema_version": 2}, - ) - return publish_experimental_review_artifacts( - publication=publication, - output_dir=str(output_dir), - pr_number="42", - ) - - def _handoff_error(output_dir: Path) -> str | None: return github_review_handoff_error(output_dir=str(output_dir), pr_number="42") From 20081d231ae955e5e9cca0bf53157dc6fb4a8e5d Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 15:40:19 -0700 Subject: [PATCH 19/21] test(review): scope handoff path assertions to their owning step --- .../skills/test_resolve_review_diff_context_consumption.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/skills/test_resolve_review_diff_context_consumption.py b/tests/skills/test_resolve_review_diff_context_consumption.py index 075dd37087..4b858f22d0 100644 --- a/tests/skills/test_resolve_review_diff_context_consumption.py +++ b/tests/skills/test_resolve_review_diff_context_consumption.py @@ -138,9 +138,9 @@ def test_step4_still_reads_file_for_editing(): def test_diff_context_path_matches_review_pr_output_path(): """resolve-review's diff_context path must reference the review-pr handoff file.""" - full = _skill_text() - assert "diff_context_" in full and "REVIEW_PR_OUTPUT" in full - assert "${AUTOSKILLIT_TEMP}" not in full + section = _step2_section() + assert "diff_context_" in section and "REVIEW_PR_OUTPUT" in section + assert "${AUTOSKILLIT_TEMP}" not in section def test_step2_map_type_is_dict_of_dicts(): From e5027807c06c3291ab97f4491c8dbe3d77f59492 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 15:42:20 -0700 Subject: [PATCH 20/21] refactor(review): use the receipt formatter in test setup --- .../test_review_receipt_production_seam.py | 3 ++- tests/server/test_tools_pr_ops.py | 21 ++++++++++++------- ...resolve_review_diff_context_consumption.py | 3 ++- .../_review_publication_fixtures.py | 4 ++-- .../test_experimental_review_publication.py | 6 +++--- 5 files changed, 23 insertions(+), 14 deletions(-) diff --git a/tests/server/test_review_receipt_production_seam.py b/tests/server/test_review_receipt_production_seam.py index b41f155231..e915741edc 100644 --- a/tests/server/test_review_receipt_production_seam.py +++ b/tests/server/test_review_receipt_production_seam.py @@ -11,6 +11,7 @@ import pytest import autoskillit.server.tools.tools_pr_ops as tools_pr_ops +from autoskillit.core import github_review_receipt_filename from autoskillit.server.tools.tools_pr_ops import post_pr_review, verify_review_receipt from autoskillit.smoke_utils import github_review_handoff_error from tests.execution.github_review.fakes import ( @@ -29,7 +30,7 @@ def _review_paths(tmp_path: Path) -> tuple[Path, Path]: output_dir = tmp_path / ".autoskillit" / "temp" / "review-pr" / "iter_2" - return output_dir, output_dir / "batch_review_response_42.json" + return output_dir, output_dir / github_review_receipt_filename(42) def _prepare_review_context( diff --git a/tests/server/test_tools_pr_ops.py b/tests/server/test_tools_pr_ops.py index 76beb04d36..efbb31f8c6 100644 --- a/tests/server/test_tools_pr_ops.py +++ b/tests/server/test_tools_pr_ops.py @@ -9,6 +9,7 @@ import pytest import autoskillit.server.tools.tools_pr_ops as tools_pr_ops +from autoskillit.core import github_review_receipt_filename from autoskillit.core.types import ( DiffAnchorAuthority, GitHubReviewComment, @@ -165,7 +166,7 @@ async def test_close_issues_sequentially_delays_between_calls() -> None: def _write_receipt(cwd: Path, payload: dict[str, object]) -> Path: - path = cwd / ".autoskillit" / "temp" / "batch_review_response_7.json" + path = cwd / ".autoskillit" / "temp" / github_review_receipt_filename(7) path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(payload, sort_keys=True, separators=(",", ":"))) return path @@ -533,7 +534,7 @@ async def test_post_pr_review_uses_injected_poster_with_exact_typed_request( tmp_path, ) -> None: """The headless tool delegates through _get_ctx without opening the kitchen gate.""" - receipt_path = str(tmp_path / ".autoskillit" / "temp" / "batch_review_response_7.json") + receipt_path = str(tmp_path / ".autoskillit" / "temp" / github_review_receipt_filename(7)) authority_path = tmp_path / ".autoskillit" / "temp" / "anchor_authority_7.json" authority_path.parent.mkdir(parents=True, exist_ok=True) authority = DiffAnchorAuthority.authoritative( @@ -625,7 +626,7 @@ async def test_post_pr_review_dry_run_is_delegated_without_gate( monkeypatch: pytest.MonkeyPatch, tmp_path, ) -> None: - receipt_path = str(tmp_path / ".autoskillit" / "temp" / "batch_review_response_7.json") + receipt_path = str(tmp_path / ".autoskillit" / "temp" / github_review_receipt_filename(7)) expected = GitHubReviewPostResult( operation_key="f" * 64, head_sha="a" * 40, @@ -691,7 +692,9 @@ async def test_post_pr_review_missing_poster_returns_structured_error( result = json.loads( await post_pr_review( cwd=str(tmp_path), - receipt_path=str(tmp_path / ".autoskillit" / "temp" / "batch_review_response_7.json"), + receipt_path=str( + tmp_path / ".autoskillit" / "temp" / github_review_receipt_filename(7) + ), anchor_authority_path=str( tmp_path / ".autoskillit" / "temp" / "anchor_authority_7.json" ), @@ -736,7 +739,9 @@ async def test_post_pr_review_never_raises_when_poster_fails( result = json.loads( await post_pr_review( cwd=str(tmp_path), - receipt_path=str(tmp_path / ".autoskillit" / "temp" / "batch_review_response_7.json"), + receipt_path=str( + tmp_path / ".autoskillit" / "temp" / github_review_receipt_filename(7) + ), anchor_authority_path=str( tmp_path / ".autoskillit" / "temp" / "anchor_authority_7.json" ), @@ -766,7 +771,7 @@ async def test_post_pr_review_never_raises_on_invalid_comment_shape( ) -> None: poster = _FakeReviewPoster( _successful_review_result( - str(tmp_path / ".autoskillit" / "temp" / "batch_review_response_7.json") + str(tmp_path / ".autoskillit" / "temp" / github_review_receipt_filename(7)) ) ) monkeypatch.setattr(tool_ctx, "github_review_poster", poster) @@ -784,7 +789,9 @@ async def test_post_pr_review_never_raises_on_invalid_comment_shape( result = json.loads( await post_pr_review( cwd=str(tmp_path), - receipt_path=str(tmp_path / ".autoskillit" / "temp" / "batch_review_response_7.json"), + receipt_path=str( + tmp_path / ".autoskillit" / "temp" / github_review_receipt_filename(7) + ), anchor_authority_path=str( tmp_path / ".autoskillit" / "temp" / "anchor_authority_7.json" ), diff --git a/tests/skills/test_resolve_review_diff_context_consumption.py b/tests/skills/test_resolve_review_diff_context_consumption.py index 4b858f22d0..8d1d558eb8 100644 --- a/tests/skills/test_resolve_review_diff_context_consumption.py +++ b/tests/skills/test_resolve_review_diff_context_consumption.py @@ -5,6 +5,7 @@ import pytest +from autoskillit.core import github_review_receipt_filename from autoskillit.smoke_utils import ( github_review_handoff_error, prepare_experimental_review_publication, @@ -315,7 +316,7 @@ def test_actual_publisher_output_matches_resolve_review_path_line_boundary( pr_number = str(receipt.pr_number) if receipt is not None else "51" if receipt is not None: output_dir.mkdir(parents=True) - receipt_path = output_dir / f"batch_review_response_{pr_number}.json" + receipt_path = output_dir / github_review_receipt_filename(receipt.pr_number) receipt_path.write_text( json.dumps(receipt.to_dict(), sort_keys=True, separators=(",", ":")) ) diff --git a/tests/smoke_utils/_review_publication_fixtures.py b/tests/smoke_utils/_review_publication_fixtures.py index 1a1142c3f4..79dd429f9b 100644 --- a/tests/smoke_utils/_review_publication_fixtures.py +++ b/tests/smoke_utils/_review_publication_fixtures.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import Any -from autoskillit.core import GitHubReviewReceipt +from autoskillit.core import GitHubReviewReceipt, github_review_receipt_filename from autoskillit.smoke_utils import ( prepare_experimental_review_publication, publish_experimental_review_artifacts, @@ -32,7 +32,7 @@ def _write_server_receipt( **overrides: object, ) -> tuple[GitHubReviewReceipt, Path]: receipt = authoritative_receipt(pr_number=pr_number, head_sha=head_sha, **overrides) - receipt_path = output_dir / f"batch_review_response_{pr_number}.json" + receipt_path = output_dir / github_review_receipt_filename(pr_number) receipt_path.parent.mkdir(parents=True, exist_ok=True) receipt_path.write_text(json.dumps(receipt.to_dict(), sort_keys=True, separators=(",", ":"))) receipt_path.chmod(0o600) diff --git a/tests/smoke_utils/test_experimental_review_publication.py b/tests/smoke_utils/test_experimental_review_publication.py index 75a81ab23c..fdb52e3d88 100644 --- a/tests/smoke_utils/test_experimental_review_publication.py +++ b/tests/smoke_utils/test_experimental_review_publication.py @@ -7,7 +7,7 @@ import pytest -from autoskillit.core import DiffAnchorAuthority +from autoskillit.core import DiffAnchorAuthority, github_review_receipt_filename from autoskillit.smoke_utils import ( EXPERIMENTAL_REVIEW_AUDITORS, aggregate_combined_review_candidates, @@ -33,8 +33,8 @@ pytestmark = [pytest.mark.medium] -def _write_receipt_payload(output_dir: Path, payload: bytes, *, pr_number: str = "42") -> Path: - receipt_path = output_dir / f"batch_review_response_{pr_number}.json" +def _write_receipt_payload(output_dir: Path, payload: bytes, *, pr_number: int = 42) -> Path: + receipt_path = output_dir / github_review_receipt_filename(pr_number) receipt_path.parent.mkdir(parents=True, exist_ok=True) receipt_path.write_bytes(payload) receipt_path.chmod(0o600) From 79b84c4bc4706047ca59b913da0cdbd55d6cf3d5 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 15:44:28 -0700 Subject: [PATCH 21/21] fix(review): preserve client errors for invalid receipt identifiers --- src/autoskillit/server/tools/tools_pr_ops.py | 4 +- tests/server/test_tools_pr_ops.py | 39 ++++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/src/autoskillit/server/tools/tools_pr_ops.py b/src/autoskillit/server/tools/tools_pr_ops.py index 1743244357..fcfc71caeb 100644 --- a/src/autoskillit/server/tools/tools_pr_ops.py +++ b/src/autoskillit/server/tools/tools_pr_ops.py @@ -395,10 +395,10 @@ async def post_pr_review( state=ReviewOperationState.AMBIGUOUS, response_class=ReviewResponseClass.SERVER_ERROR, ) - if not os.path.isdir(cwd): + if not os.path.isdir(cwd) or type(pr_number) is not int or pr_number <= 0: return _review_post_error( head_sha=head_sha, - error="cwd must be an existing directory", + error="cwd must be an existing directory and pr_number a positive integer", state=ReviewOperationState.TERMINAL, response_class=ReviewResponseClass.CLIENT_ERROR, ) diff --git a/tests/server/test_tools_pr_ops.py b/tests/server/test_tools_pr_ops.py index efbb31f8c6..e6c8cfe6bf 100644 --- a/tests/server/test_tools_pr_ops.py +++ b/tests/server/test_tools_pr_ops.py @@ -527,6 +527,45 @@ def _successful_review_result(receipt_path: str) -> GitHubReviewPostResult: ) +@pytest.mark.anyio +@pytest.mark.parametrize("pr_number", [True, 0]) +async def test_post_pr_review_rejects_invalid_pr_before_receipt_path_formatting( + pr_number: int, + tool_ctx, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + poster = _FakeReviewPoster() + monkeypatch.setattr(tool_ctx, "github_review_poster", poster) + monkeypatch.setattr(tools_pr_ops, "_get_ctx", lambda: tool_ctx) + + result = json.loads( + await post_pr_review( + cwd=str(tmp_path), + receipt_path=str( + tmp_path / ".autoskillit" / "temp" / github_review_receipt_filename(7) + ), + anchor_authority_path=str( + tmp_path / ".autoskillit" / "temp" / "anchor_authority_7.json" + ), + repository="owner/repo", + pr_number=pr_number, + head_sha="a" * 40, + logical_iteration="review-pr:1", + event="COMMENT", + body="Review", + comments=[], + dry_run=False, + ctx=Mock(), + ) + ) + + assert result["state"] == "TERMINAL" + assert result["response_class"] == "CLIENT_ERROR" + assert "positive integer" in result["error"] + assert poster.requests == [] + + @pytest.mark.anyio async def test_post_pr_review_uses_injected_poster_with_exact_typed_request( tool_ctx,