diff --git a/.autoskillit/test-filter-manifest.yaml b/.autoskillit/test-filter-manifest.yaml index 5f4ee22d77..79e8c1c729 100644 --- a/.autoskillit/test-filter-manifest.yaml +++ b/.autoskillit/test-filter-manifest.yaml @@ -246,6 +246,9 @@ tests/execution/fixtures/*.json: tests/execution/fixtures/*.jsonl: - execution/ +tests/infra/fixtures/e2e_sessions/**: + - infra/ + tests/config/golden_configs/*.yaml: - config/ diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index a35b998ce8..aedb28d507 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -104,12 +104,13 @@ jobs: env: MINIMAX_API_KEY: ${{ matrix.kind != 'clean-install' && secrets.MINIMAX_API_KEY || '' }} E2E_SANDBOX_TOKEN: ${{ matrix.kind != 'clean-install' && secrets.E2E_SANDBOX_TOKEN || '' }} + E2E_IMPLEMENTATION_ISSUE_URL: ${{ vars.E2E_IMPLEMENTATION_ISSUE_URL }} TEST: ${{ matrix.test }} run: | set -euo pipefail docker_args=(--shm-size=3g) if [[ '${{ matrix.kind }}' != 'clean-install' ]]; then - docker_args+=(--env MINIMAX_API_KEY --env E2E_SANDBOX_TOKEN) + docker_args+=(--env MINIMAX_API_KEY --env E2E_SANDBOX_TOKEN --env E2E_IMPLEMENTATION_ISSUE_URL) docker_args+=(--volume "$RUNNER_TEMP/e2e/data:/home/autoskillit/.local/share/autoskillit") fi if [[ "$TEST" == 'headless-smoke' ]]; then diff --git a/scripts/e2e/README.md b/scripts/e2e/README.md index 79873bf682..54bbe7b568 100644 --- a/scripts/e2e/README.md +++ b/scripts/e2e/README.md @@ -11,6 +11,7 @@ This directory holds everything the workflow runs: | `e2e_catalog.py` | Stdlib loader and validator for the catalog | | `e2e_select.py` | Picks the tests an event runs (`select`) and judges the required check (`gate`) | | `e2e_harness.py` | Runs one test (`run`), performs exact sandbox teardown (`cleanup`) and redacts artifacts (`redact`) | +| `e2e_sessions.py` | Captures synchronous lifecycle events, measures concurrency and pairs ownership evidence | | `autoskillit-config.yaml` | AutoSkillit user-layer config the harness installs | | `post-e2e-failure.sh` | Appends a failure to the test's tracking issue | @@ -40,6 +41,11 @@ model tests: because MiniMax serves the Anthropic Messages API. 3. Create the `e2e` label in this repository. 4. In branch protection, mark only `e2e-gate` as required — never `select` or `run (…)`. +5. Before publishing a branch with the selectable `implementation` entry, obtain a passing + #5232 sandbox smoke CI run. A maintainer must seed a fixed OPEN sandbox issue describing + a small feature and its test, then set repository variable `E2E_IMPLEMENTATION_ISSUE_URL` + to its canonical `https://github.com/OWNER/REPO/issues/NUMBER` URL. Reuse that issue on + every run. Missing configuration fails preflight; it is not an expected product defect. ## Sandbox repository requirements @@ -196,7 +202,11 @@ matrix is always parseable (`{"include":[]}` when nothing is selected). bypasses sampling because a labelled pull request whose sample fell out would otherwise never run for that commit. - **Picking:** tests whose `trigger_paths` match a changed path, up to `MAX_TESTS` (2); when - none match, exactly one test. Ties are ordered by the SHA-256 hex digest of + none match, one eligible ordinary test. A pipeline requires actual size at or above a + threshold and a matching trigger path, even with the `e2e` label. This filter also applies + to fallback; a pipeline-only ineligible pool selects nothing. At most one pipeline is + selected within the total cap. Named workflow dispatch bypasses PR eligibility. + Ties are ordered by the SHA-256 hex digest of `"{seed}:{name}"`, so the decision is a pure function of the payload and the changed paths. Patterns use `fnmatch.fnmatchcase`, where `*` also matches `/` (`scripts/e2e/*` matches every file below `scripts/e2e/`). @@ -212,10 +222,53 @@ succeeded or nothing was selected and `run` was skipped. Every other combination The `run` job uses one repository-wide concurrency group (`group: e2e`, `queue: max`, `cancel-in-progress: false`) and `max-parallel: 1`, so every selected test of every run waits its turn and never cancels another. `select` and `e2e-gate` are not queued. A -catalog entry's `peak_sessions` is capped at `MAX_PEAK_SESSIONS` (8): one step's fan-out of -up to six subagents plus the orchestrator and step session. With one test running at a -time, CI stays at or below eight of the roughly twelve concurrent MiniMax sessions. -`clean-install` requires exactly zero `peak_sessions`. +catalog entry's `peak_sessions` is capped at `MAX_PEAK_SESSIONS` (8). For an uncommissioned +pipeline this declares capacity, not a measured fan-out. A step can fan out to up to six +subagents alongside its orchestrator and step session. With runs serialized, CI stays at or +below eight of roughly twelve concurrent MiniMax sessions. `clean-install` requires exactly +zero `peak_sessions`. + +The implementation pipeline installs synchronous command hooks for `SessionStart`, +`SessionEnd`, `SubagentStart` and `SubagentStop`, preserving existing hooks and MiniMax +settings. These hooks follow the [Claude lifecycle schema](https://code.claude.com/docs/en/hooks). +The agent hooks use `.+` to require a non-empty agent type: under the default main +agent, Claude Code also fires `SubagentStop` for internal agents with an empty type. +An omitted or empty matcher admits those callbacks; `.+` accepts every non-empty +built-in, custom or namespaced plugin agent name without an allowlist or product-registry +imports in these stdlib-only scripts. Session hooks have no matcher. +Sessions must run with the default main agent. The harness fails preflight if the +user-level settings file has an `agent` key; the fleet command builders pass neither +`--agent` nor `--settings`, and the bundled plugin sets no main agent. Project/local +settings in the sandbox, managed settings and other enabled plugins' main-agent defaults +are not checked. A main agent from those sources makes internal stops carry its name: +they fail coverage as a named stop without a start, without the blank-type `capture:` code. +Claude Code 2.1.275 fixed specific matchers admitting blank-type stops; the image's +2.1.280 pin includes that fix. +Each identity spans its first start through its final end, so compaction and resume yield +a conservative upper bound. Half-open intervals end before simultaneous starts. The +summary reconciles the dispatched root, managed lineage and enumerable native transcripts. +AutoSkillit writes `logs/sessions.jsonl` and each session's `summary.json`; their dispatch +and parent/caller fields are AutoSkillit provenance, separate from Claude Code transcript +and lifecycle hook fields. Empty links do not imply a Claude Code schema guarantee. +Managed sessions are adopted through declared parent fields or receipted `run_skill`, +`complete_run_skill_result` or `recover_run_skill_result` replies in an owned transcript. +Each must have exactly one index row with the run's `dispatch_id`. Parentless sessions +without receipts, missing or ambiguous index rows, and receipt-named sessions absent +from both trace and index fail coverage. Shared dispatch IDs or overlapping lifetimes +never adopt a session. Missing endpoints, identities or owned transcripts fail coverage; +transcript activity never infers a lifetime. The captured pre-fix trace contains a blank-type +stop immediately before a compact-source session start; its producing feature is unproven. +An owned agent record without a named type fails coverage with a `capture:` violation. +A named stop without its start or transcript still fails coverage, as do traces captured +before the matcher was installed. The collector retains delivered callbacks and adds +violations; it does not filter them out. Individual codes are in `out/session-concurrency.json` +under `violations`. A peak exceeding capacity fails validation. + +Commission with a successful MiniMax CI run and repeat against the same seed. Inspect +planning, dry-walkthrough, implementation, test, integration, audit GO and open-PR evidence, +plus cleanup and the sandbox CI association. Replace provisional `peak_sessions` with +the observed conservative peak and set `peak_sessions_evidence` to the CI run URL, tested +commit and first-start/final-end method. Expected failures do not satisfy commissioning. Every harness subprocess has a timeout that fires before the job's: `fleet run` gets the test's `timeout_sec` plus `HARNESS_GRACE_SEC`, and the job gets that budget in minutes plus @@ -259,6 +312,13 @@ digest, id, `sealed` state and coverage status remain in that session's files survive `pre_review_cleanup`. A container killed before or during collection may leave no evidence or only partial evidence. +Pipeline evidence includes `session-events.jsonl`, `session-concurrency.json`, compact +`ownership.json`, the seed, effective configuration, fleet stdout/stderr/envelope, captured +PR metadata/files, detached-head test commands, associated CI runs/jobs, and sandbox head-ref +listings before the run and after cleanup (`head-refs-before.*`, `head-refs-after.*`). Collection +occurs inside the container before transcript paths disappear. Incomplete evidence is +retained and fails the scenario; redaction and upload still run after failure. + `clean-install` mounts no home directories and retains evidence under `out/`. Its redactor receives the same required `--secret-env` names with variables unset and skips the absent home-log tree. Redaction and upload still run after test failures. @@ -323,6 +383,57 @@ row and close the bug. Fleet translates the L3 result's `reason` into the failed command envelope's `error` and `user_visible_message` fields; the harness matches that envelope rather than the recipe payload's `failed_step`/`reason`. +For the implementation pipeline, the project-owned +[`DispatchCompleted.to_envelope`](../../src/autoskillit/fleet/campaign_state/state_outcomes.py) +must report success, `dispatch_status="success"` and root/dispatch identities. The nested +[`implementation` terminal](../../src/autoskillit/recipes/implementation.yaml) must have +`reason="implementation_complete"`. Successful no-change and already-done terminals remain +valid recipe behavior but fail this scenario. No PR URL is required in the envelope. + +The harness resolves the configured seed once and dispatches its exact URL and title/body +task once. Effective project-aware config comes from `autoskillit config show`, whose +implementation calls `load_config(Path.cwd())`; both configured and sandbox default branches +must be `main`. No caller `base_branch` ingredient is sent. Sandbox setup and baseline tests +must succeed before launch. + +Paired root transcript tool calls/results prove the created branch; paired root/managed +claim results prove any added claim label. The collector accepts the direct +`mcp__autoskillit__` and plugin `mcp__plugin_autoskillit_autoskillit__` namespaces; +a tracked tool under any other namespace fails the proof. Claude Code persisted +transcripts carry replies in row-level `mcpMeta.structuredContent` when `mcpMeta` +is present, otherwise in JSON block content. Malformed metadata or metadata on a +row with several results fails the paired resource call. Hook display text never +replaces unusable metadata. The pretty-output hook emits `updatedMCPToolOutput`; +`_result_object` remains the shared envelope decoder, and metadata presence alone +does not identify whether the hook ran. These shapes are pinned by +`tests/infra/fixtures/e2e_sessions/` from Claude Code 2.1.280; re-capture them when +an otherwise successful call reports a failed branch or claim result. + +Discovery paginates through the PR baseline +and leaves unrelated PRs untouched. Require exactly one owned open, unmerged sandbox PR +targeting `main`, without auto-merge, with the canonical seed URL and non-removed changes +under both `sandbox/` and `tests/`. Fetch and detach at the captured head before setup and +tests. Require a successful associated sandbox `CI` run and `test` job whose nested PR +association has the exact branch revision; an outer merge SHA is not the branch SHA. +Finally re-read the head and state to reject movement during validation. + +Teardown runs after validation, failure or exception. It independently collects metrics, +closes owned open PRs, deletes evidenced branches (including branches without a PR), and +removes only the effective claim label proven added by this run. The seed remains OPEN +and other labels are preserved. Missing ownership proof preserves uncertain resources and +fails cleanup. Cleanup errors remain fatal alongside the original failure. + +Branch reconciliation lists the sandbox's branches before the run and after cleanup. +A branch created during the run that `ownership.json` does not explain is preserved +and fails cleanup; baseline branches are excluded from unexplained-branch reports. +Branches named by the ownership proof retain the existing cleanup rule. The original +API stdout saved as `head-refs-before.stdout.txt` and `head-refs-after.stdout.txt` supplies +the in-memory snapshots; accompanying stderr and command records diagnose failed requests. +This check only detects unexplained branches: the creating run records the failure once, +and the branch and any PR on it must be removed by hand. The next run includes it in +its baseline. A manual run overlapping CI can trigger the check because CI serialization +does not cover local runs. + ## Catalog schema ```json @@ -351,15 +462,39 @@ the harness matches that envelope rather than the recipe payload's `failed_step` ``` `recipe`, `ingredients` and `expected_pull_request_state` are required for `recipe` tests -and forbidden for `canary` and `clean-install` tests. `expected_failures` is optional for -`clean-install` and recipe tests. Clean-install rows use the exact doctor `severity`, `check` -and `message`; recipe rows require `severity: error`, `check` and `message` matching the failed -envelope. Every row includes an open bug issue URL -(`https://github.com/TalonT-Org/AutoSkillit/issues/`). Duplicate diagnostic identities -and unknown keys are rejected. `peak_sessions` must be +and forbidden for `canary` and `clean-install` tests. Optional recipe-only fields are +`pipeline` (boolean, default false), `issue_url_env` (seed variable name), +`required_changed_paths` (nonempty relative prefixes), `test_command` (nonempty argv), and +`peak_sessions_evidence` (nonempty provenance string). Environment-owned seeds cannot also +specify literal `task` or `issue_url` ingredients. Path prefixes cannot be absolute or +traverse parents. For example, the implementation entry uses: + +```json +{"pipeline": true, "issue_url_env": "E2E_IMPLEMENTATION_ISSUE_URL", + "required_changed_paths": ["sandbox/", "tests/"], "test_command": ["task", "test-check"]} +``` + +The recipe_fixture field names a staged YAML fixture for the headless smoke scenario. + +The expected_failures field is optional for recipes and clean-install. Pipeline rows +contain exactly check, message, issue; ordinary recipe and clean-install rows also +contain severity. Ordinary recipes require severity: error and match the failed +envelope's error and user_visible_message; clean-install allows error or warning +and matches doctor diagnostics. Pipeline rows match structured implementation findings. +Every row links to an open AutoSkillit bug issue. Duplicate diagnostic identities and +unknown keys are rejected. The peak_sessions value must be 1–8 for model tests and exactly zero for clean-install. `timeout_sec` must keep the derived job timeout within 360 minutes. +Recipe expectations match exact stable `(check, message)` product findings linked to real +bug issues. Setup, discovery, malformed evidence and cleanup failures cannot be admitted. +Unknown findings and absent expected findings fail. Each recipe run records `outcome` in +`result.json` as `passed`, `expected_failure` (with `passed=false`), or `failed`. A linked +known defect may +use the accepted CLI exit convention, but only `passed` with no expected findings counts +as successful commissioning. File or deduplicate observed product bugs separately; do not +predeclare a provider defect or replace the MiniMax settings-file route. + ### Adding a test 1. Add the entry to `catalog.json` with a realistic `peak_sessions` and `timeout_sec`. diff --git a/scripts/e2e/autoskillit-config.yaml b/scripts/e2e/autoskillit-config.yaml index 577e8df598..e4a8c3de29 100644 --- a/scripts/e2e/autoskillit-config.yaml +++ b/scripts/e2e/autoskillit-config.yaml @@ -3,6 +3,9 @@ quota_guard: enabled: false +branching: + default_base_branch: main + features: fleet: true fleet_headless_run: true diff --git a/scripts/e2e/catalog.json b/scripts/e2e/catalog.json index 7dfdd54199..177fbf50a5 100644 --- a/scripts/e2e/catalog.json +++ b/scripts/e2e/catalog.json @@ -64,6 +64,35 @@ "recipe_fixture": "sandbox-smoke.yaml", "ingredients": {}, "expected_pull_request_state": "closed" + }, + { + "name": "implementation", + "kind": "recipe", + "pipeline": true, + "peak_sessions": 8, + "timeout_sec": 7200, + "trigger_paths": [ + "src/autoskillit/recipes/*", + "src/autoskillit/skills/*", + "src/autoskillit/skills_extended/*", + "src/autoskillit/server/*", + "src/autoskillit/execution/*" + ], + "recipe": "implementation", + "ingredients": { + "source_dir": "/workspace/sandbox", + "open_pr": "true", + "auto_merge": "false", + "arch_lenses": "false", + "audit_impl": "true" + }, + "expected_pull_request_state": "open", + "issue_url_env": "E2E_IMPLEMENTATION_ISSUE_URL", + "required_changed_paths": [ + "sandbox/", + "tests/" + ], + "test_command": ["task", "test-check"] } ] } diff --git a/scripts/e2e/e2e_catalog.py b/scripts/e2e/e2e_catalog.py index 46354d43f4..40b255bfca 100644 --- a/scripts/e2e/e2e_catalog.py +++ b/scripts/e2e/e2e_catalog.py @@ -7,7 +7,7 @@ import re from collections.abc import Mapping from dataclasses import dataclass -from pathlib import Path +from pathlib import Path, PurePosixPath, PureWindowsPath from types import MappingProxyType CATALOG_PATH = Path(__file__).with_name("catalog.json") @@ -28,7 +28,21 @@ _CATALOG_KEYS = frozenset({"sandbox_repository", "tests"}) _COMMON_KEYS = frozenset({"name", "kind", "peak_sessions", "timeout_sec", "trigger_paths"}) _RECIPE_KEYS = frozenset({"recipe", "ingredients", "expected_pull_request_state"}) +_PIPELINE_METADATA_KEYS = frozenset( + { + "issue_url_env", + "required_changed_paths", + "test_command", + "peak_sessions_evidence", + } +) +_RECIPE_OPTIONAL_KEYS = _PIPELINE_METADATA_KEYS | { + "pipeline", + "expected_failures", + "recipe_fixture", +} _EXPECTED_FAILURE_KEYS = frozenset({"severity", "check", "message", "issue"}) +_PIPELINE_EXPECTED_FAILURE_KEYS = frozenset({"check", "message", "issue"}) _BUG_URL_PATTERN = re.compile(r"https://github\.com/TalonT-Org/AutoSkillit/issues/[1-9][0-9]*") @@ -50,6 +64,11 @@ class CatalogTest: expected_pull_request_state: str | None expected_failures: tuple[Mapping[str, str], ...] = () recipe_fixture: str | None = None + pipeline: bool = False + issue_url_env: str | None = None + required_changed_paths: tuple[str, ...] = () + test_command: tuple[str, ...] = () + peak_sessions_evidence: str | None = None @dataclass(frozen=True, slots=True) @@ -83,6 +102,13 @@ def _require_str(value: object, where: str) -> str: return value +def _require_nonblank_str(value: object, where: str) -> str: + result = _require_str(value, where) + if not result.strip(): + raise CatalogError(f"{where} must be non-empty") + return result + + def _require_int(value: object, where: str, *, low: int, high: int | None = None) -> int: if isinstance(value, bool) or not isinstance(value, int): raise CatalogError(f"{where} must be an integer") @@ -111,7 +137,7 @@ def _parse_ingredients(value: object, where: str) -> tuple[tuple[str, str], ...] def _parse_recipe_fields( raw: Mapping[str, object], where: str -) -> tuple[str, tuple[tuple[str, str], ...], str]: +) -> tuple[str, tuple[tuple[str, str], ...], str, str | None]: recipe = _require_str(raw["recipe"], f"{where}.recipe") ingredients = _parse_ingredients(raw["ingredients"], f"{where}.ingredients") state = _require_str( @@ -121,28 +147,40 @@ def _parse_recipe_fields( raise CatalogError( f"{where}.expected_pull_request_state must be one of {PULL_REQUEST_STATES}" ) - return recipe, ingredients, state + fixture = None + if "recipe_fixture" in raw: + fixture = _require_str(raw["recipe_fixture"], f"{where}.recipe_fixture") + if not _RECIPE_FIXTURE_PATTERN.fullmatch(fixture): + raise CatalogError(f"{where}.recipe_fixture must be a .yaml basename") + return recipe, ingredients, state, fixture -def _parse_expected_failures(value: object, where: str) -> tuple[Mapping[str, str], ...]: +def _parse_expected_failures( + value: object, where: str, *, pipeline: bool = False +) -> tuple[Mapping[str, str], ...]: where = f"{where}.expected_failures" if not isinstance(value, list): raise CatalogError(f"{where} must be a list") + keys = _PIPELINE_EXPECTED_FAILURE_KEYS if pipeline else _EXPECTED_FAILURE_KEYS rows: list[Mapping[str, str]] = [] - seen: set[tuple[str, str, str]] = set() + seen: set[tuple[str, ...]] = set() for index, raw in enumerate(value): location = f"{where}[{index}]" if not isinstance(raw, dict): raise CatalogError(f"{location} must be an object") - _require_keys(raw, _EXPECTED_FAILURE_KEYS, location) - row = {key: _require_str(raw[key], f"{location}.{key}") for key in _EXPECTED_FAILURE_KEYS} + _require_keys(raw, keys, location) + row = {key: _require_str(raw[key], f"{location}.{key}") for key in keys} if any(not item.strip() for item in row.values()): raise CatalogError(f"{location} fields must be non-empty") - if row["severity"] not in ("error", "warning"): + if not pipeline and row["severity"] not in ("error", "warning"): raise CatalogError(f"{location}.severity must be error or warning") if not _BUG_URL_PATTERN.fullmatch(row["issue"]): raise CatalogError(f"{location}.issue must link to an AutoSkillit bug issue") - identity = (row["severity"], row["check"], row["message"]) + identity = ( + (row["check"], row["message"]) + if pipeline + else (row["severity"], row["check"], row["message"]) + ) if identity in seen: raise CatalogError(f"{location}: duplicate diagnostic {identity}") seen.add(identity) @@ -150,6 +188,99 @@ def _parse_expected_failures(value: object, where: str) -> tuple[Mapping[str, st return tuple(rows) +def _parse_recipe_options( + raw: Mapping[str, object], where: str, ingredients: tuple[tuple[str, str], ...] +) -> tuple[str | None, tuple[str, ...], tuple[str, ...], str | None]: + issue_url_env = ( + _require_nonblank_str(raw["issue_url_env"], f"{where}.issue_url_env") + if "issue_url_env" in raw + else None + ) + if issue_url_env and {"issue_url", "task"}.intersection(key for key, _ in ingredients): + raise CatalogError( + f"{where}.ingredients cannot contain issue_url or task when issue_url_env is set" + ) + changed_paths = ( + _parse_required_changed_paths( + raw["required_changed_paths"], f"{where}.required_changed_paths" + ) + if "required_changed_paths" in raw + else () + ) + test_command = ( + _parse_test_command(raw["test_command"], f"{where}.test_command") + if "test_command" in raw + else () + ) + evidence = ( + _require_nonblank_str(raw["peak_sessions_evidence"], f"{where}.peak_sessions_evidence") + if "peak_sessions_evidence" in raw + else None + ) + return issue_url_env, changed_paths, test_command, evidence + + +def _parse_required_changed_paths(value: object, where: str) -> tuple[str, ...]: + if not isinstance(value, list) or not value: + raise CatalogError(f"{where} must be a non-empty list") + changed_paths: list[str] = [] + for index, item in enumerate(value): + location = f"{where}[{index}]" + prefix = _require_nonblank_str(item, location) + posix_path = PurePosixPath(prefix) + windows_path = PureWindowsPath(prefix) + invalid_path = any( + ( + posix_path.is_absolute(), + windows_path.is_absolute(), + bool(windows_path.drive), + ".." in posix_path.parts, + ".." in windows_path.parts, + ) + ) + if invalid_path: + raise CatalogError(f"{location} must be a relative path prefix without '..'") + changed_paths.append(prefix) + return tuple(changed_paths) + + +def _parse_test_command(command: object, where: str) -> tuple[str, ...]: + if not isinstance(command, list) or not command: + raise CatalogError(f"{where} must be a non-empty argv list") + return tuple( + _require_nonblank_str(item, f"{where}[{index}]") for index, item in enumerate(command) + ) + + +def _validate_test_keys(raw: Mapping[str, object], kind: str, where: str) -> bool: + keys = _COMMON_KEYS | _RECIPE_KEYS if kind == "recipe" else _COMMON_KEYS + if kind == "clean-install" and "expected_failures" in raw: + keys |= {"expected_failures"} + if kind != "recipe": + _require_keys(raw, keys, where) + return False + + missing = sorted(keys - raw.keys()) + unknown = sorted(raw.keys() - keys - _RECIPE_OPTIONAL_KEYS) + if missing or unknown: + raise CatalogError(f"{where}: missing keys {missing}, unknown keys {unknown}") + pipeline = raw.get("pipeline", False) + if not isinstance(pipeline, bool): + raise CatalogError(f"{where}.pipeline must be a boolean") + pipeline_only = {"required_changed_paths", "test_command", "peak_sessions_evidence"} + if pipeline: + missing_pipeline = sorted({"required_changed_paths", "test_command"} - raw.keys()) + if missing_pipeline: + raise CatalogError(f"{where}: missing pipeline keys {missing_pipeline}") + else: + unexpected_pipeline = sorted(pipeline_only & raw.keys()) + if unexpected_pipeline: + raise CatalogError( + f"{where}: pipeline-only keys {unexpected_pipeline} require pipeline=true" + ) + return pipeline + + def _parse_test(raw: object, index: int) -> CatalogTest: where = f"tests[{index}]" if not isinstance(raw, dict): @@ -157,26 +288,36 @@ def _parse_test(raw: object, index: int) -> CatalogTest: kind = raw.get("kind") if kind not in KINDS: raise CatalogError(f"{where}.kind must be one of {KINDS}") - keys = _COMMON_KEYS | _RECIPE_KEYS if kind == "recipe" else _COMMON_KEYS - if kind == "recipe" and "recipe_fixture" in raw: - keys |= {"recipe_fixture"} - if kind in ("recipe", "clean-install") and "expected_failures" in raw: - keys |= {"expected_failures"} - _require_keys(raw, keys, where) + pipeline = _validate_test_keys(raw, kind, where) name = _require_str(raw["name"], f"{where}.name") if not _NAME_PATTERN.fullmatch(name): raise CatalogError(f"{where}.name must match {_NAME_PATTERN.pattern}") - recipe, ingredients, state = ( - _parse_recipe_fields(raw, where) if kind == "recipe" else (None, (), None) + recipe, ingredients, state, recipe_fixture = ( + _parse_recipe_fields(raw, where) if kind == "recipe" else (None, (), None, None) ) - recipe_fixture = None - if kind == "recipe" and "recipe_fixture" in raw: - recipe_fixture = _require_str(raw["recipe_fixture"], f"{where}.recipe_fixture") - if not _RECIPE_FIXTURE_PATTERN.fullmatch(recipe_fixture): - raise CatalogError(f"{where}.recipe_fixture must be a .yaml basename") - expected_failures = _parse_expected_failures(raw.get("expected_failures", []), where) - if kind == "recipe" and any(row["severity"] != "error" for row in expected_failures): + issue_url_env: str | None = None + required_changed_paths: tuple[str, ...] = () + test_command: tuple[str, ...] = () + peak_sessions_evidence: str | None = None + if kind in ("recipe", "clean-install"): + expected_failures = _parse_expected_failures( + raw.get("expected_failures", []), where, pipeline=pipeline + ) + else: + expected_failures = () + if ( + kind == "recipe" + and not pipeline + and any(row["severity"] != "error" for row in expected_failures) + ): raise CatalogError(f"{where}.expected_failures severity must be error for recipe tests") + if kind == "recipe": + ( + issue_url_env, + required_changed_paths, + test_command, + peak_sessions_evidence, + ) = _parse_recipe_options(raw, where, ingredients) test = CatalogTest( name=name, kind=kind, @@ -193,6 +334,11 @@ def _parse_test(raw: object, index: int) -> CatalogTest: expected_pull_request_state=state, expected_failures=expected_failures, recipe_fixture=recipe_fixture, + pipeline=pipeline, + issue_url_env=issue_url_env, + required_changed_paths=required_changed_paths, + test_command=test_command, + peak_sessions_evidence=peak_sessions_evidence, ) if job_timeout_minutes(test) > MAX_JOB_MINUTES: raise CatalogError(f"{where}.timeout_sec exceeds the {MAX_JOB_MINUTES}-minute job limit") diff --git a/scripts/e2e/e2e_harness.py b/scripts/e2e/e2e_harness.py index ad5ff3c8f3..e4749f0806 100644 --- a/scripts/e2e/e2e_harness.py +++ b/scripts/e2e/e2e_harness.py @@ -9,6 +9,7 @@ import json import os import re +import shlex import shutil import subprocess import sys @@ -18,9 +19,11 @@ import uuid from collections.abc import Callable, Mapping, Sequence from pathlib import Path -from typing import Any +from typing import Any, TypedDict, cast +from urllib.parse import quote import e2e_catalog +import e2e_sessions MINIMAX_BASE_URL = "https://api.minimax.io/anthropic" MINIMAX_MODEL = "MiniMax-M3[1m]" @@ -69,6 +72,9 @@ _SCRUBBED_ENV = frozenset({"CLAUDE_CODE_OAUTH_TOKEN", *SECRET_ENV}) _STDERR_TAIL_CHARS = 2000 +_SESSION_TRACE_FILENAME = "session-events.jsonl" +_CI_POLL_TIMEOUT_SEC = 600 +_CI_POLL_INTERVAL_SEC = 10 CLEAN_INSTALL_ALLOWED_WARNINGS: dict[tuple[str, str], str] = { ( "pytest_temp_capacity", @@ -84,6 +90,18 @@ CallResult = tuple[subprocess.CompletedProcess[str] | None, str | None] +class PullRequestListRow(TypedDict): + number: int + state: str + additions: int + deletions: int + + +class PipelinePullRequestListRow(TypedDict): + number: int + headRefOid: str | None + + def run_command( argv: Sequence[str], *, @@ -304,28 +322,54 @@ def latest_pull_request_number( def new_pull_requests( - repository: str, baseline: int, env: Mapping[str, str], runner: Runner -) -> tuple[list[dict[str, Any]] | None, str | None]: + repository: str, + baseline: int, + env: Mapping[str, str], + runner: Runner, +) -> tuple[list[PullRequestListRow] | None, str | None]: rows, failure = _list_pull_requests( repository, 20, "number,state,additions,deletions", env, runner ) if rows is None: return None, failure - return [row for row in rows if int(row["number"]) > baseline], None + return cast(list[PullRequestListRow], [row for row in rows if row["number"] > baseline]), None + + +def new_pipeline_pull_requests( + repository: str, + baseline: int, + env: Mapping[str, str], + runner: Runner, +) -> tuple[list[PipelinePullRequestListRow] | None, str | None]: + limit = 20 + fields = "number,headRefOid" + while True: + rows, failure = _list_pull_requests(repository, limit, fields, env, runner) + if rows is None: + return None, failure + if len(rows) < limit or any(row["number"] <= baseline for row in rows): + return ( + cast( + list[PipelinePullRequestListRow], + [row for row in rows if row["number"] > baseline], + ), + None, + ) + limit *= 2 def fleet_run_argv( - test: e2e_catalog.CatalogTest, runtime_ingredients: Mapping[str, str] | None = None + test: e2e_catalog.CatalogTest, *, ingredients: Mapping[str, str] | None = None ) -> list[str]: - effective = dict(test.ingredients) - effective.update(runtime_ingredients or {}) - ingredients = [arg for key, value in effective.items() for arg in ("-i", f"{key}={value}")] + values = dict(test.ingredients) + values.update(ingredients or {}) + arguments = [arg for key, value in values.items() for arg in ("-i", f"{key}={value}")] return [ "autoskillit", "fleet", "run", str(test.recipe), - *ingredients, + *arguments, "--disable-quota-guard", "--timeout-sec", str(test.timeout_sec), @@ -352,6 +396,33 @@ def check_envelope(returncode: int, stdout: str) -> tuple[list[str], dict[str, A return failures, envelope +def check_implementation_envelope( + envelope: Mapping[str, Any], +) -> tuple[list[str], list[dict[str, str]]]: + failures: list[str] = [] + findings: list[dict[str, str]] = [] + payload = envelope.get("l3_payload") + reason = payload.get("reason") if isinstance(payload, dict) else None + status = envelope.get("dispatch_status") + if envelope.get("success") is not True or status != "success": + stable_reason = envelope.get("reason") or reason + if isinstance(stable_reason, str) and stable_reason: + findings.append({"check": "fleet_envelope", "message": f"{status}: {stable_reason}"}) + else: + failures.append("fleet run: unsuccessful envelope has no specific failure reason") + elif reason != "implementation_complete": + if isinstance(reason, str) and reason: + findings.append({"check": "implementation_terminal", "message": reason}) + else: + failures.append("fleet run: implementation terminal reason is missing") + if not all( + isinstance(envelope.get(key), str) and envelope[key] + for key in ("dispatch_id", "dispatched_session_id") + ): + failures.append("fleet run: dispatch/session identity is missing") + return failures, findings + + def session_index_dir_names(log_dir: Path) -> frozenset[str]: index = log_dir / "sessions.jsonl" if not index.exists(): @@ -410,29 +481,49 @@ def run_fleet( out: Path, env: Mapping[str, str], runner: Runner, - runtime_ingredients: Mapping[str, str] | None = None, -) -> tuple[list[str], dict[str, Any] | None]: + *, + ingredients: Mapping[str, str] | None = None, +) -> tuple[dict[str, Any] | None, list[str], list[dict[str, str]]]: log_dir = Path(env["AUTOSKILLIT_LOG_DIR"]) baseline = session_index_dir_names(log_dir) failures: list[str] = [] + findings: list[dict[str, str]] = [] envelope: dict[str, Any] | None = None try: result, failure = _call( runner, - fleet_run_argv(test, runtime_ingredients), + fleet_run_argv(test, ingredients=ingredients), what="fleet run", + evidence=out / "fleet-run", cwd=SANDBOX_CLONE, env=env, timeout=test.timeout_sec + e2e_catalog.HARNESS_GRACE_SEC, - evidence=out / "fleet-run", ) + stdout = (out / "fleet-run.stdout.txt").read_text(encoding="utf-8", errors="replace") + line = last_line(stdout) + (out / "envelope.json").write_text(line + "\n", encoding="utf-8") + if not test.pipeline: + if result is None: + failures.append(str(failure)) + else: + envelope_failures, envelope = check_envelope(result.returncode, stdout) + failures.extend(envelope_failures) + return envelope, failures, findings + try: + envelope = json.loads(line) + except json.JSONDecodeError: + envelope = None + if not isinstance(envelope, dict): + envelope = None + failures.append("fleet run: the last stdout line is not a JSON object envelope") if result is None: failures.append(str(failure)) - else: - (out / "fleet-run.stdout.log").write_text(result.stdout, encoding="utf-8") - (out / "fleet-run.stderr.log").write_text(result.stderr, encoding="utf-8") - (out / "envelope.json").write_text(last_line(result.stdout) + "\n", encoding="utf-8") - failures, envelope = check_envelope(result.returncode, result.stdout) + elif envelope is not None: + envelope_failures, findings = check_implementation_envelope(envelope) + failures.extend(envelope_failures) + if failure and not findings: + failures.append(failure) + return envelope, failures, findings finally: failures.extend( check_session_windows( @@ -441,7 +532,6 @@ def run_fleet( sessions_required=envelope is not None and envelope.get("success") is True, ) ) - return failures, envelope def match_recipe_failure( @@ -571,7 +661,7 @@ def run_smoke_recipe( if runtime is None: return failures, [] (out / "smoke-launched").write_text(test.name + "\n", encoding="utf-8") - fleet_failures, envelope = run_fleet(test, out, env, runner, runtime) + envelope, fleet_failures, _ = run_fleet(test, out, env, runner, ingredients=runtime) fleet_failures, matched = match_recipe_failure(test, fleet_failures, envelope) failures += fleet_failures if ( @@ -981,6 +1071,572 @@ def close_open_pull_requests( return failures +def install_session_hooks(home: Path, trace: Path) -> None: + path = home / ".claude" / "settings.json" + settings = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(settings, dict): + raise ValueError("session hooks: settings.json must be an object") + if "agent" in settings: + # Internal-agent callbacks carry the main agent's name, which the matcher admits. + raise ValueError("session hooks: settings.json sets a main agent") + if "hooks" not in settings: + hooks: dict[str, Any] = {} + else: + hooks = settings["hooks"] + if not isinstance(hooks, dict): + raise ValueError("session hooks: settings.json hooks must be an object") + command = f"python3 /opt/e2e/e2e_sessions.py {shlex.quote(str(trace))}" + for event, matcher in e2e_sessions.LIFECYCLE_HOOK_MATCHERS.items(): + entry: dict[str, Any] = {"hooks": [{"type": "command", "command": command, "timeout": 10}]} + if matcher is not None: + entry["matcher"] = matcher + if event not in hooks: + hooks[event] = [] + elif not isinstance(hooks[event], list): + raise ValueError(f"session hooks: settings.json hooks.{event} must be an array") + hooks[event].append(entry) + settings["hooks"] = hooks + path.write_text(json.dumps(settings, indent=2) + "\n", encoding="utf-8") + + +def _json_call( + argv: Sequence[str], + env: Mapping[str, str], + runner: Runner, + *, + evidence: Path | None = None, + cwd: Path | None = None, +) -> Any: + result, failure = _setup_call(runner, argv, env, evidence=evidence, cwd=cwd) + if failure or result is None: + raise ValueError(failure or "missing command result") + try: + return json.loads(result.stdout) + except json.JSONDecodeError as exc: + raise ValueError( + f"{' '.join(argv[:3])}: invalid JSON at line {exc.lineno} " + f"column {exc.colno}: {exc.msg}" + ) from exc + + +def _api_pages(endpoint: str, env: Mapping[str, str], runner: Runner, evidence: Path) -> list[Any]: + pages = _json_call( + ["gh", "api", endpoint, "--paginate", "--slurp"], env, runner, evidence=evidence + ) + if not isinstance(pages, list): + raise ValueError("GitHub API: expected paginated JSON array") + return pages + + +def list_head_refs( + repository: str, out: Path, env: Mapping[str, str], runner: Runner, stage: str +) -> frozenset[str]: + """Return the branch names currently present in the sandbox repository.""" + pages = _api_pages( + f"repos/{repository}/git/matching-refs/heads/", env, runner, out / f"head-refs-{stage}" + ) + if not pages: + raise ValueError("head-ref listing must contain a page") + refs: set[str] = set() + for page in pages: + if not isinstance(page, list): + raise ValueError("head-ref listing page must be an array") + for row in page: + ref = row.get("ref") if isinstance(row, dict) else None + if ( + not isinstance(ref, str) + or not ref.startswith("refs/heads/") + or ref == "refs/heads/" + ): + raise ValueError("head-ref listing must contain nonempty refs/heads/ objects") + refs.add(ref.removeprefix("refs/heads/")) + return frozenset(refs) + + +def _seed_config_fields(config: object) -> tuple[str, str]: + if not isinstance(config, Mapping): + raise ValueError("seed: effective config must be an object") + values: list[str] = [] + for section, field in (("github", "in_progress_label"), ("branching", "default_base_branch")): + settings = config.get(section) + if not isinstance(settings, Mapping): + raise ValueError(f"seed: effective config {section} section must be an object") + value = settings.get(field) + if not isinstance(value, str) or not value.strip(): + raise ValueError( + f"seed: effective config {section}.{field} must be a non-empty string" + ) + values.append(value) + return values[0], values[1] + + +def prepare_seed( + test: e2e_catalog.CatalogTest, + repository: str, + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> tuple[dict[str, Any], dict[str, str], str]: + url = env.get(test.issue_url_env or "", "") + match = re.fullmatch( + r"https://github\.com/" + re.escape(repository) + r"/issues/([1-9][0-9]*)", url + ) + if not match: + raise ValueError("seed: missing or noncanonical sandbox issue URL") + config = _json_call( + ["autoskillit", "config", "show"], + env, + runner, + evidence=out / "effective-config", + cwd=SANDBOX_CLONE, + ) + label, configured_base_branch = _seed_config_fields(config) + seed = _json_call( + ["gh", "issue", "view", url, "--json", "number,url,state,title,body,labels"], + env, + runner, + evidence=out / "seed", + ) + if ( + not isinstance(seed, dict) + or seed.get("url") != url + or type(seed.get("number")) is not int + or seed["number"] != int(match[1]) + or seed.get("state") != "OPEN" + or not isinstance(seed.get("title"), str) + or not seed["title"].strip() + or not isinstance(seed.get("body"), str) + or not isinstance(seed.get("labels"), list) + or any( + not isinstance(row, dict) or not isinstance(row.get("name"), str) + for row in seed["labels"] + ) + ): + raise ValueError("seed: expected the configured OPEN sandbox issue with valid metadata") + if label in {row["name"] for row in seed["labels"]}: + raise ValueError("seed: issue is already claimed") + repo = _json_call( + ["gh", "repo", "view", repository, "--json", "defaultBranchRef"], + env, + runner, + evidence=out / "repository", + ) + default_branch = repo.get("defaultBranchRef") if isinstance(repo, dict) else None + if ( + not isinstance(default_branch, dict) + or default_branch.get("name") != "main" + or configured_base_branch != "main" + ): + raise ValueError("seed: sandbox and effective configured base branch must both be main") + ingredients = dict(test.ingredients) + ingredients.update(issue_url=url, task=f"{seed['title']}\n\n{seed['body']}") + for stage, argv in ( + ("baseline-install", ["task", "install-worktree"]), + ("baseline-test", ["task", "test-check"]), + ): + _, failure = _call( + runner, + argv, + what=stage, + evidence=out / stage, + cwd=SANDBOX_CLONE, + env=env, + timeout=SETUP_COMMAND_TIMEOUT_SEC, + ) + if failure: + raise ValueError(failure) + return seed, ingredients, label + + +def _repo_matches(value: Any, repository: str) -> bool: + return isinstance(value, dict) and ( + value.get("full_name") == repository + or value.get("url") == f"https://api.github.com/repos/{repository}" + ) + + +def owned_pull_requests( + rows: Sequence[PipelinePullRequestListRow], + proof: e2e_sessions.OwnershipProof, + repository: str, + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> list[dict[str, Any]]: + owned = [] + for row in rows: + pr = _json_call( + ["gh", "api", f"repos/{repository}/pulls/{row['number']}"], + env, + runner, + evidence=out / f"pr-{row['number']}", + ) + if not isinstance(pr, dict): + raise ValueError("PR discovery: expected an object") + head = pr.get("head") + if ( + isinstance(head, dict) + and head.get("ref") in proof["branches"] + and _repo_matches(head.get("repo"), repository) + ): + pr["_listed_head"] = row["headRefOid"] + owned.append(pr) + return owned + + +class PipelineValidationError(ValueError): + """A pipeline scenario validation failed without a harness defect.""" + + +def validate_sandbox_ci( + repository: str, + number: int, + head: str, + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> None: + workflows = _api_pages( + f"repos/{repository}/actions/workflows?per_page=100", env, runner, out / "ci-workflows" + ) + candidates = [ + workflow + for page in workflows + for workflow in page.get("workflows", []) + if workflow.get("name") == "CI" and workflow.get("state") == "active" + ] + if len(candidates) != 1: + raise PipelineValidationError("sandbox CI: expected one active CI workflow") + deadline = time.monotonic() + _CI_POLL_TIMEOUT_SEC + matching_run_seen = False + while True: + pages = _api_pages( + f"repos/{repository}/actions/workflows/{candidates[0]['id']}/runs?event=pull_request&per_page=100", + env, + runner, + out / "ci-runs", + ) + runs = [ + run + for page in pages + for run in page.get("workflow_runs", []) + if any( + pr.get("number") == number + and pr.get("head", {}).get("sha") == head + and _repo_matches(pr.get("head", {}).get("repo"), repository) + and _repo_matches(pr.get("base", {}).get("repo"), repository) + for pr in run.get("pull_requests", []) + ) + ] + if runs: + matching_run_seen = True + run = max(runs, key=lambda value: (value["id"], value.get("run_attempt", 1))) + if run.get("status") == "completed": + jobs = _api_pages( + f"repos/{repository}/actions/runs/{run['id']}/jobs?per_page=100", + env, + runner, + out / "ci-jobs", + ) + tests = [ + job + for page in jobs + for job in page.get("jobs", []) + if job.get("name") == "test" + ] + if ( + run.get("conclusion") != "success" + or not tests + or any( + job.get("status") != "completed" or job.get("conclusion") != "success" + for job in tests + ) + ): + raise PipelineValidationError("sandbox CI: CI run and test job must succeed") + return + remaining = deadline - time.monotonic() + if remaining <= 0: + break + time.sleep(min(_CI_POLL_INTERVAL_SEC, remaining)) + if matching_run_seen: + raise PipelineValidationError( + "sandbox CI: matching run did not complete before the deadline" + ) + raise PipelineValidationError( + "sandbox CI: no run associated with the exact PR head before the deadline" + ) + + +def _implementation_pr_preconditions(pr: Mapping[str, Any], repository: str) -> list[str]: + head = pr.get("head", {}) + base = pr.get("base", {}) + oid = head.get("sha") + checks = ( + (pr.get("state") == "open", "state must be open"), + (pr.get("merged") is False, "merged must be false"), + (not pr.get("merged_at"), "merged_at must be empty"), + (pr.get("auto_merge") is None, "auto-merge must be disabled"), + (base.get("ref") == "main", "base branch must be main"), + (head.get("ref") != "main", "head branch must not be main"), + (isinstance(oid, str) and bool(oid), "head SHA must be a non-empty string"), + (_repo_matches(base.get("repo"), repository), "base repository must match the sandbox"), + (_repo_matches(head.get("repo"), repository), "head repository must match the sandbox"), + ) + return [f"implementation PR: {message}" for valid, message in checks if not valid] + + +def validate_implementation_pr( + test: e2e_catalog.CatalogTest, + prs: Sequence[Mapping[str, Any]], + seed: Mapping[str, Any], + repository: str, + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> list[str]: + if len(prs) != 1: + return [f"expected exactly one new owned sandbox pull request, found {len(prs)}"] + pr = prs[0] + number = pr["number"] + head = pr.get("head", {}) + oid = head.get("sha") + precondition_failures = _implementation_pr_preconditions(pr, repository) + if precondition_failures: + return precondition_failures + if oid != pr.get("_listed_head"): + return ["implementation PR: REST head differs from captured headRefOid"] + if not re.search(re.escape(seed["url"]) + r"(?![A-Za-z0-9_/#?=-])", str(pr.get("body", ""))): + return ["implementation PR: body does not reference the canonical seed URL"] + if ( + type(pr.get("additions")) is not int + or type(pr.get("deletions")) is not int + or (pr["additions"] + pr["deletions"] <= 0) + ): + return ["implementation PR: empty or malformed diff"] + pages = _api_pages( + f"repos/{repository}/pulls/{number}/files?per_page=100", env, runner, out / "pr-files" + ) + files = [row for page in pages for row in page] + for prefix in test.required_changed_paths: + if not any( + row.get("status") != "removed" + and isinstance(row.get("filename"), str) + and row["filename"].startswith(prefix) + and row.get("additions", 0) + row.get("deletions", 0) > 0 + for row in files + ): + return [f"implementation PR: missing non-removed changes under {prefix}"] + failures = validate_pr_head(test, number, oid, out, env, runner) + if failures: + return failures + validate_sandbox_ci(repository, number, oid, out, env, runner) + final = _json_call( + ["gh", "api", f"repos/{repository}/pulls/{number}"], env, runner, evidence=out / "pr-final" + ) + if ( + final.get("head", {}).get("sha") != oid + or final.get("state") != "open" + or final.get("merged") is not False + or final.get("merged_at") + ): + return ["implementation PR: head or open/unmerged state changed during validation"] + return [] + + +def validate_pr_head( + test: e2e_catalog.CatalogTest, + number: int, + oid: str, + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> list[str]: + commands = ( + ("pr-fetch", ["git", "fetch", "origin", f"refs/pull/{number}/head"]), + ("pr-oid", ["git", "rev-parse", "FETCH_HEAD"]), + ("pr-checkout", ["git", "checkout", "--detach", oid]), + ("pr-install", ["task", "install-worktree"]), + ("pr-test", list(test.test_command)), + ) + for stage, argv in commands: + result, failure = _call( + runner, + argv, + what=stage, + evidence=out / stage, + cwd=SANDBOX_CLONE, + env=env, + timeout=SETUP_COMMAND_TIMEOUT_SEC, + ) + if failure: + return [failure] + if stage == "pr-oid" and (result is None or result.stdout.strip() != oid): + return ["implementation PR: fetched head differs from captured head"] + return [] + + +def discover_cleanup_prs( + repository: str, + baseline: int, + proof: e2e_sessions.OwnershipProof, + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> tuple[list[dict[str, Any]], list[str]]: + failures: list[str] = [] + owned: list[dict[str, Any]] = [] + for attempt in range(2): + try: + rows, failure = new_pipeline_pull_requests(repository, baseline, env, runner) + if rows is None: + raise ValueError(failure) + owned = owned_pull_requests(rows, proof, repository, out, env, runner) + if owned or attempt == 1: + if rows and not proof["branches"]: + failures.append("cleanup: PR ownership could not be established") + break + except Exception as exc: + if attempt == 1: + failures.append(f"cleanup discovery: {exc}") + if attempt == 0: + time.sleep(1) + return owned, failures + + +def delete_owned_branch( + repository: str, branch: str, env: Mapping[str, str], runner: Runner +) -> None: + if branch == "main" or not branch or branch.startswith("refs/"): + raise ValueError("refusing to delete an unsafe/default branch") + endpoint = f"repos/{repository}/git/refs/heads/{quote(branch, safe='')}" + result, failure = _setup_call(runner, ["gh", "api", endpoint], env) + if ( + failure + and result is not None + and re.search(r"(?m)^gh: .*\bHTTP 404\b", result.stderr or "") + ): + return + if failure: + raise ValueError(failure) + time.sleep(1) + _, failure = _setup_call(runner, ["gh", "api", "--method", "DELETE", endpoint], env) + if failure: + raise ValueError(failure) + + +def release_owned_claim( + proof: e2e_sessions.OwnershipProof, + seed: Mapping[str, Any], + label: str, + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> list[str]: + current = _json_call( + ["gh", "issue", "view", seed["url"], "--json", "state,labels"], + env, + runner, + evidence=out / "seed-final", + ) + if not isinstance(current, Mapping): + raise ValueError("cleanup: final seed issue must be an object") + labels = current.get("labels") + if not isinstance(labels, list) or any( + not isinstance(row, Mapping) or not isinstance(row.get("name"), str) for row in labels + ): + raise ValueError("cleanup: final seed issue labels must be an array of named objects") + failures = [] if current.get("state") == "OPEN" else ["cleanup: seed issue is no longer OPEN"] + if label not in {row["name"] for row in labels}: + return failures + if not proof["claimed"]: + return [*failures, "cleanup: claim label is present without this run's claim proof"] + time.sleep(1) + _, failure = _setup_call( + runner, ["gh", "issue", "edit", seed["url"], "--remove-label", label], env + ) + if failure: + failures.append(f"cleanup: {failure}") + return failures + + +def unexplained_branch_failures( + repository: str, + head_refs: frozenset[str], + proof: e2e_sessions.OwnershipProof, + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> list[str]: + """Report branches created during the run that the ownership proof does not explain.""" + try: + current = list_head_refs(repository, out, env, runner, "after") + except Exception as exc: + return [f"cleanup refs: {exc}"] + return [ + f"cleanup: branch {branch} was created during this run without ownership proof" + for branch in sorted(current - head_refs - set(proof["branches"])) + ] + + +def cleanup_implementation( + repository: str, + baseline: int, + head_refs: frozenset[str], + proof: e2e_sessions.OwnershipProof, + seed: Mapping[str, Any], + label: str, + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> list[str]: + owned, failures = discover_cleanup_prs(repository, baseline, proof, out, env, runner) + for pr in owned: + try: + if pr.get("state") == "open": + time.sleep(1) + _, failure = _setup_call( + runner, ["gh", "pr", "close", str(pr["number"]), "--repo", repository], env + ) + if failure: + failures.append(f"cleanup: {failure}") + except Exception as exc: + failures.append(f"cleanup PR: {exc}") + for branch in proof["branches"]: + try: + delete_owned_branch(repository, branch, env, runner) + except Exception as exc: + failures.append(f"cleanup branch: {exc}") + failures += unexplained_branch_failures(repository, head_refs, proof, out, env, runner) + try: + failures += release_owned_claim(proof, seed, label, out, env, runner) + except Exception as exc: + failures.append(f"cleanup claim: {exc}") + return failures + + +def match_recipe_findings( + failures: list[str], + findings: Sequence[Mapping[str, str]], + expected: Sequence[Mapping[str, str]], +) -> tuple[list[str], list[dict[str, str]]]: + unmatched_failures = list(failures) + annotations = {(row["check"], row["message"]): row for row in expected} + matched: dict[tuple[str, str], dict[str, str]] = {} + for row in findings: + identity = (row["check"], row["message"]) + if identity in annotations: + matched[identity] = dict(annotations[identity]) + else: + unmatched_failures.append(f"{row['check']}: {row['message']}") + for identity, annotation in annotations.items(): + if identity not in matched: + unmatched_failures.append( + f"expected recipe finding absent {identity}; remove stale expected failure " + f"for {annotation['issue']}" + ) + return unmatched_failures, list(matched.values()) + + def run_recipe( test: e2e_catalog.CatalogTest, repository: str, @@ -988,6 +1644,8 @@ def run_recipe( out: Path, env: Mapping[str, str], runner: Runner, + *, + home: Path, ) -> tuple[list[str], list[dict[str, str]]]: failure = ( configure_git(env, runner) @@ -998,15 +1656,42 @@ def run_recipe( return [failure], [] if test.recipe_fixture is not None: return run_smoke_recipe(test, repository, out, env, runner) + ingredients = dict(test.ingredients) + trace = out / _SESSION_TRACE_FILENAME + if test.pipeline or test.issue_url_env: + try: + seed, ingredients, label = prepare_seed(test, repository, out, env, runner) + if test.pipeline: + head_refs = list_head_refs(repository, out, env, runner, "before") + trace.write_text("", encoding="utf-8") + install_session_hooks(home, trace) + except Exception as exc: + return [f"preflight: {exc}"], [] baseline, failure = latest_pull_request_number(repository, env, runner) if baseline is None: return [str(failure)], [] + if not test.pipeline: + return launch_regular_recipe(test, repository, baseline, ingredients, out, env, runner) + return launch_recipe( + test, repository, baseline, head_refs, seed, ingredients, label, out, env, runner + ) + + +def launch_regular_recipe( + test: e2e_catalog.CatalogTest, + repository: str, + baseline: int, + ingredients: Mapping[str, str], + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> tuple[list[str], list[dict[str, str]]]: failures: list[str] = [] - listed: list[dict[str, Any]] | None = None + matched: list[dict[str, str]] = [] + listed: list[PullRequestListRow] | None = None try: - fleet_failures, envelope = run_fleet(test, out, env, runner) - fleet_failures, matched = match_recipe_failure(test, fleet_failures, envelope) - failures += fleet_failures + envelope, failures, _ = run_fleet(test, out, env, runner, ingredients=ingredients) + failures, matched = match_recipe_failure(test, failures, envelope) if not matched: listed, failure = new_pull_requests(repository, baseline, env, runner) if listed is None: @@ -1018,6 +1703,135 @@ def run_recipe( return failures, matched +def save_ownership( + trace: Path, + envelope: dict[str, Any] | None, + log_root: Path, + seed: Mapping[str, Any], + repository: str, + out: Path, + label: str, +) -> e2e_sessions.OwnershipProof: + proof = e2e_sessions.collect_ownership( + trace, envelope, log_root, seed["url"], repository, claim_label=label + ) + (out / "ownership.json").write_text(json.dumps(proof, indent=2) + "\n", encoding="utf-8") + return proof + + +def finish_pipeline( + test: e2e_catalog.CatalogTest, + repository: str, + baseline: int, + head_refs: frozenset[str], + seed: Mapping[str, Any], + label: str, + out: Path, + env: Mapping[str, str], + runner: Runner, + envelope: dict[str, Any] | None, + proof: e2e_sessions.OwnershipProof | None, +) -> list[str]: + failures: list[str] = [] + trace = out / _SESSION_TRACE_FILENAME + log_root = Path(env["AUTOSKILLIT_LOG_DIR"]) + capacity: dict[str, Any] = {"peak_sessions": test.peak_sessions} + if test.peak_sessions_evidence is not None: + capacity["peak_sessions_evidence"] = test.peak_sessions_evidence + try: + (out / "catalog-capacity.json").write_text( + json.dumps(capacity, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + except Exception as exc: + failures.append(f"session capacity evidence: {exc}") + try: + summary = e2e_sessions.summarize(trace, envelope, log_root) + if not summary["coverage"]["complete"]: + failures.append("session measurement: incomplete lifecycle coverage") + if summary["peak_sessions"] > test.peak_sessions: + failures.append("session measurement: observed peak exceeds catalog capacity") + except Exception as exc: + failures.append(f"session measurement: {exc}") + if proof is None: + try: + proof = save_ownership(trace, envelope, log_root, seed, repository, out, label) + failures += [f"ownership: {value}" for value in proof["violations"]] + except Exception as exc: + failures.append(f"cleanup ownership: {exc}") + if proof is None: + proof = { + "branches": [], + "claimed": False, + "transcripts": [], + "violations": [], + } + failures += cleanup_implementation( + repository, baseline, head_refs, proof, seed, label, out, env, runner + ) + return failures + + +def launch_recipe( + test: e2e_catalog.CatalogTest, + repository: str, + baseline: int, + head_refs: frozenset[str], + seed: Mapping[str, Any], + ingredients: Mapping[str, str], + label: str, + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> tuple[list[str], list[dict[str, str]]]: + failures: list[str] = [] + findings: list[dict[str, str]] = [] + envelope: dict[str, Any] | None = None + proof: e2e_sessions.OwnershipProof | None = None + try: + envelope, fleet_failures, findings = run_fleet( + test, out, env, runner, ingredients=ingredients + ) + failures += fleet_failures + proof = save_ownership( + out / _SESSION_TRACE_FILENAME, + envelope, + Path(env["AUTOSKILLIT_LOG_DIR"]), + seed, + repository, + out, + label, + ) + failures += [f"ownership: {value}" for value in proof["violations"]] + listed, failure = new_pipeline_pull_requests(repository, baseline, env, runner) + if listed is None: + failures.append(str(failure)) + else: + owned = owned_pull_requests(listed, proof, repository, out, env, runner) + if not fleet_failures and not findings: + failures += validate_implementation_pr( + test, owned, seed, repository, out, env, runner + ) + except PipelineValidationError as exc: + failures.append(str(exc)) + except Exception as exc: + failures.append(f"harness error: {type(exc).__name__}: {exc}\n{traceback.format_exc()}") + finally: + failures += finish_pipeline( + test, + repository, + baseline, + head_refs, + seed, + label, + out, + env, + runner, + envelope, + proof, + ) + return match_recipe_findings(failures, findings, test.expected_failures) + + def check_clean_install_doctor( stdout: str, expected_failures: Sequence[Mapping[str, str]] ) -> tuple[list[str], list[dict[str, str]]]: @@ -1131,13 +1945,9 @@ def _run_test( token = env["E2E_SANDBOX_TOKEN"] recipe_env = { **scrubbed, - "AUTOSKILLIT_LOG_DIR": str(autoskillit_log_dir(home)), + "AUTOSKILLIT_LOG_DIR": str(e2e_sessions.default_log_root(home)), } - return run_recipe(test, catalog.sandbox_repository, token, out, recipe_env, runner) - - -def autoskillit_log_dir(home: Path) -> Path: - return home / ".local" / "share" / "autoskillit" / "logs" + return run_recipe(test, catalog.sandbox_repository, token, out, recipe_env, runner, home=home) def _copy_evidence_files( @@ -1265,7 +2075,9 @@ def collect_run_evidence(out: Path, home: Path) -> dict[str, Any]: projects, out / "transcripts", home, ("*.jsonl", "*.txt"), errors ) unretained = _unretained_references( - autoskillit_log_dir(home) / "sessions.jsonl", {os.path.realpath(p) for p in copied}, errors + e2e_sessions.default_log_root(home) / "sessions.jsonl", + {os.path.realpath(p) for p in copied}, + errors, ) return { "pipeline_files": len(copied), diff --git a/scripts/e2e/e2e_select.py b/scripts/e2e/e2e_select.py index 906e66a69b..325c779beb 100644 --- a/scripts/e2e/e2e_select.py +++ b/scripts/e2e/e2e_select.py @@ -21,6 +21,7 @@ MIN_CHANGED_LINES = 300 SAMPLE_PROBABILITY = 0.5 MAX_TESTS = 2 +MAX_PIPELINE_TESTS = 1 @dataclass(frozen=True, slots=True) @@ -68,10 +69,12 @@ def selected(self) -> bool: def is_eligible(pr: PullRequestFacts) -> bool: + return E2E_LABEL in pr.labels or _is_large_change(pr) + + +def _is_large_change(pr: PullRequestFacts) -> bool: return ( - E2E_LABEL in pr.labels - or pr.changed_files >= MIN_CHANGED_FILES - or pr.additions + pr.deletions >= MIN_CHANGED_LINES + pr.changed_files >= MIN_CHANGED_FILES or pr.additions + pr.deletions >= MIN_CHANGED_LINES ) @@ -97,13 +100,32 @@ def _touches(test: e2e_catalog.CatalogTest, changed_paths: Sequence[str]) -> boo def pick_tests( - catalog: e2e_catalog.Catalog, changed_paths: Sequence[str], seed: str + catalog: e2e_catalog.Catalog, + changed_paths: Sequence[str], + seed: str, + *, + large_change: bool = False, ) -> tuple[str, ...]: - ranked = sorted(catalog.tests, key=lambda test: _rank(seed, test.name)) - touched = [test.name for test in ranked if _touches(test, changed_paths)] + eligible = [ + test + for test in catalog.tests + if not test.pipeline or (large_change and _touches(test, changed_paths)) + ] + ranked = sorted(eligible, key=lambda test: _rank(seed, test.name)) + touched = [test for test in ranked if _touches(test, changed_paths)] if touched: - return tuple(touched[:MAX_TESTS]) - return (ranked[0].name,) + picked: list[str] = [] + pipeline_count = 0 + for test in touched: + if test.pipeline: + if pipeline_count >= MAX_PIPELINE_TESTS: + continue + pipeline_count += 1 + picked.append(test.name) + if len(picked) == MAX_TESTS: + break + return tuple(picked) + return (ranked[0].name,) if ranked else () def select_for_pull_request( @@ -121,11 +143,14 @@ def select_for_pull_request( f"below thresholds ({pr.changed_files} files, {lines} lines) and no {E2E_LABEL} label", ) seed = f"{pr.number}:{pr.head_sha}" - if E2E_LABEL in pr.labels: - return Selection(pick_tests(catalog, changed_paths, seed), f"{E2E_LABEL} label") - if not is_sampled(pr, SAMPLE_PROBABILITY): + labeled = E2E_LABEL in pr.labels + if not labeled and not is_sampled(pr, SAMPLE_PROBABILITY): return Selection((), "eligible but not sampled for this head commit") - return Selection(pick_tests(catalog, changed_paths, seed), "eligible and sampled") + reason = f"{E2E_LABEL} label" if labeled else "eligible and sampled" + return Selection( + pick_tests(catalog, changed_paths, seed, large_change=_is_large_change(pr)), + reason, + ) def select_for_dispatch(requested: str, catalog: e2e_catalog.Catalog) -> Selection: diff --git a/scripts/e2e/e2e_sessions.py b/scripts/e2e/e2e_sessions.py new file mode 100644 index 0000000000..3059654c2a --- /dev/null +++ b/scripts/e2e/e2e_sessions.py @@ -0,0 +1,1096 @@ +#!/usr/bin/env python3 +"""Capture and summarize Claude session lifecycle evidence for E2E runs.""" + +from __future__ import annotations + +import fcntl +import json +import math +import os +import re +import sys +import time +from collections.abc import Iterator, Mapping +from pathlib import Path +from typing import Any, TypedDict, TypeGuard +from urllib.parse import urlsplit + +_ToolCall = tuple[str, dict[str, Any], dict[str, Any] | None, bool] + + +class _OwnedSession(TypedDict): + row: dict[str, Any] + summary: dict[str, Any] | None + session_dir: Path + transcript: Path | None + calls: list[_ToolCall] + + +class OwnershipProof(TypedDict): + branches: list[str] + claimed: bool + transcripts: list[dict[str, str]] + violations: list[str] + + +_SESSION_EVENTS = frozenset({"SessionStart", "SessionEnd"}) +_AGENT_EVENTS = frozenset({"SubagentStart", "SubagentStop"}) +_LIFECYCLE_EVENTS = _SESSION_EVENTS | _AGENT_EVENTS +# With the default main agent, internal agents have an empty agent_type. +# Session matchers filter source/reason, so keep session endpoints unfiltered. +_NATIVE_AGENT_MATCHER = ".+" +LIFECYCLE_HOOK_MATCHERS: Mapping[str, str | None] = { + **{event: None for event in sorted(_SESSION_EVENTS)}, + **{event: _NATIVE_AGENT_MATCHER for event in sorted(_AGENT_EVENTS)}, +} +# Pinned to product prefixes by tests/arch/test_e2e_sessions_contract.py. +_AUTOSKILLIT_SERVERS = frozenset( + {("mcp", "autoskillit"), ("mcp", "plugin_autoskillit_autoskillit")} +) +_RESOURCE_TOOLS = frozenset({"claim_and_resolve_issue", "create_and_publish_branch"}) +_LAUNCH_TOOLS = frozenset({"run_skill", "complete_run_skill_result", "recover_run_skill_result"}) +_TRACKED_TOOLS = _RESOURCE_TOOLS | _LAUNCH_TOOLS +_MAX_HOOK_INPUT_BYTES = 65_536 +_MAX_TRACE_RECORD_BYTES = 8_192 +_TRACE_LOCK_TIMEOUT_SEC = 5.0 +_TRACE_LOCK_POLL_SEC = 0.01 + + +def capture_hook_event(trace_path: Path, payload: Mapping[str, Any]) -> bool: + """Append one allow-listed lifecycle record; return False for other hooks.""" + record = _hook_record(payload) + if record is None: + return False + encoded = (json.dumps(record, separators=(",", ":")) + "\n").encode("utf-8") + if len(encoded) > _MAX_TRACE_RECORD_BYTES: + raise ValueError("lifecycle hook record exceeds the size limit") + _append_trace_record(trace_path, encoded) + return True + + +def _append_trace_record(trace_path: Path, encoded: bytes) -> None: + """Serialize a complete append and roll back an incomplete record under lock.""" + trace_path.parent.mkdir(parents=True, exist_ok=True) + fd = os.open(trace_path, os.O_APPEND | os.O_CREAT | os.O_WRONLY, 0o600) + lock_acquired = False + try: + deadline = time.monotonic() + _TRACE_LOCK_TIMEOUT_SEC + while not lock_acquired: + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as exc: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("timed out acquiring lifecycle trace lock") from exc + time.sleep(min(_TRACE_LOCK_POLL_SEC, remaining)) + else: + lock_acquired = True + + start_offset = os.fstat(fd).st_size + remaining_bytes = memoryview(encoded) + written_total = 0 + try: + while remaining_bytes: + written = os.write(fd, remaining_bytes) + if written <= 0: + raise OSError("short lifecycle trace append") + written_total += written + remaining_bytes = remaining_bytes[written:] + except OSError: + if written_total: + os.ftruncate(fd, start_offset) + raise + finally: + if lock_acquired: + fcntl.flock(fd, fcntl.LOCK_UN) + os.close(fd) + + +def _hook_record(payload: Mapping[str, Any]) -> dict[str, Any] | None: + event = payload.get("hook_event_name") or payload.get("event") + if event is None: + return None + if not isinstance(event, str): + raise ValueError("lifecycle hook event must be a string") + if event not in _LIFECYCLE_EVENTS: + return None + + session_id = payload.get("session_id") + if not isinstance(session_id, str) or not session_id: + raise ValueError("lifecycle hook has no session_id") + + record: dict[str, Any] = { + "event": event, + "timestamp": time.time(), + "session_id": session_id, + } + for key in ( + "agent_id", + "agent_type", + "source", + "transcript_path", + "agent_transcript_path", + ): + value = payload.get(key) + if value is None: + continue + if not isinstance(value, str): + raise ValueError(f"lifecycle hook {key} must be a string") + record[key] = value + if event in _AGENT_EVENTS and not record.get("agent_id"): + raise ValueError("subagent lifecycle hook has no agent_id") + return record + + +def _iter_jsonl( + path: Path, label: str, violations: list[str] +) -> Iterator[tuple[int, dict[str, Any]]]: + try: + handle = path.open("rb") + except OSError: + violations.append(f"{label}:unavailable") + return + + with handle: + for line_number, raw_line in enumerate(handle, 1): + if not raw_line.strip(): + violations.append(f"{label}:empty_line:{line_number}") + continue + try: + item = json.loads(raw_line.decode("utf-8")) + except (json.JSONDecodeError, UnicodeDecodeError): + violations.append(f"{label}:invalid_json:{line_number}") + continue + if not isinstance(item, dict): + violations.append(f"{label}:invalid_record:{line_number}") + continue + yield line_number, item + + +def _read_jsonl(path: Path, label: str, violations: list[str]) -> list[dict[str, Any]]: + return [item for _, item in _iter_jsonl(path, label, violations)] + + +def _load_trace(trace_path: Path, violations: list[str]) -> list[dict[str, Any]]: + records = _read_jsonl(trace_path, "trace", violations) + valid: list[dict[str, Any]] = [] + for index, record in enumerate(records, 1): + event = record.get("event") + timestamp = record.get("timestamp") + session_id = record.get("session_id") + if not isinstance(event, str) or event not in _LIFECYCLE_EVENTS: + violations.append(f"trace:unknown_event:{index}") + continue + finite_timestamp = False + if isinstance(timestamp, (int, float)) and not isinstance(timestamp, bool): + try: + finite_timestamp = math.isfinite(timestamp) + except OverflowError: + pass + if not finite_timestamp: + violations.append(f"trace:invalid_timestamp:{index}") + continue + if not isinstance(session_id, str) or not session_id: + violations.append(f"trace:invalid_session_id:{index}") + continue + optional_invalid = any( + key in record and record[key] is not None and not isinstance(record[key], str) + for key in ( + "agent_id", + "agent_type", + "source", + "transcript_path", + "agent_transcript_path", + ) + ) + if optional_invalid: + violations.append(f"trace:invalid_optional_field:{index}") + continue + if event in _AGENT_EVENTS and not record.get("agent_id"): + violations.append(f"trace:missing_agent_id:{index}") + continue + valid.append(record) + return valid + + +def _read_envelope(envelope: dict[str, Any] | None, violations: list[str]) -> tuple[str, str]: + if not isinstance(envelope, dict): + violations.append("envelope:missing") + return "", "" + session_id = envelope.get("dispatched_session_id") + dispatch_id = envelope.get("dispatch_id") + if not isinstance(session_id, str) or not session_id: + violations.append("envelope:missing_dispatched_session_id") + session_id = "" + if not isinstance(dispatch_id, str) or not dispatch_id: + violations.append("envelope:missing_dispatch_id") + dispatch_id = "" + return session_id, dispatch_id + + +def _load_index(log_root: Path, violations: list[str]) -> list[dict[str, Any]]: + index_path = log_root / "sessions.jsonl" + records = _read_jsonl(index_path, "sessions_index", violations) + for line_number, row in enumerate(records, 1): + if not isinstance(row.get("session_id"), str) or not row.get("session_id"): + violations.append(f"sessions_index:missing_session_id:{line_number}") + return records + + +def _safe_path_component(value: Any) -> TypeGuard[str]: + return ( + isinstance(value, str) + and bool(value) + and value not in {".", ".."} + and "/" not in value + and "\\" not in value + and "\x00" not in value + and Path(value).name == value + ) + + +def _safe_session_dir(log_root: Path, row: Mapping[str, Any]) -> Path | None: + name = row.get("dir_name") + if not _safe_path_component(name): + return None + return log_root / "sessions" / name + + +def _safe_subagents_dir(transcript: Path, session_id: str, violations: list[str]) -> Path | None: + label = f"native:{session_id}" + if not _safe_path_component(session_id): + violations.append(f"{label}:unsafe_subagent_root") + return None + transcript_root = transcript.parent + try: + resolved_root = transcript_root.resolve() + resolved_subagents = (transcript_root / session_id / "subagents").resolve() + except (OSError, RuntimeError): + violations.append(f"{label}:unsafe_subagent_root") + return None + if not resolved_subagents.is_relative_to(resolved_root): + violations.append(f"{label}:unsafe_subagent_root") + return None + return resolved_subagents + + +def _read_summary( + session_dir: Path, expected_session_id: str, violations: list[str] +) -> dict[str, Any] | None: + summary_path = session_dir / "summary.json" + try: + raw_summary = summary_path.read_text(encoding="utf-8") + except FileNotFoundError: + violations.append(f"summary:{expected_session_id}:missing") + return None + except OSError as exc: + violations.append(f"summary:{expected_session_id}:unavailable:{type(exc).__name__}") + return None + except UnicodeDecodeError as exc: + violations.append(f"summary:{expected_session_id}:invalid_utf8:{exc.start}") + return None + try: + summary = json.loads(raw_summary) + except json.JSONDecodeError as exc: + violations.append(f"summary:{expected_session_id}:invalid_json:{exc.lineno}:{exc.colno}") + return None + if not isinstance(summary, dict): + violations.append(f"summary:{expected_session_id}:invalid_record") + return None + if summary.get("session_id") != expected_session_id: + violations.append(f"summary:{expected_session_id}:session_id_mismatch") + return None + identity = summary.get("execution_identity") + if not isinstance(identity, dict) or not isinstance(identity.get("parent_session_id"), str): + violations.append(f"summary:{expected_session_id}:invalid_execution_identity") + return None + return summary + + +def default_log_root(home: Path) -> Path: + return home / ".local" / "share" / "autoskillit" / "logs" + + +def _discover_owned( + trace_path: Path, + envelope: dict[str, Any] | None, + log_root: Path | None, +) -> dict[str, Any]: + """Resolve only the dispatched session and fresh, linked managed descendants.""" + root = log_root or default_log_root(Path.home()) + violations: list[str] = [] + events = _load_trace(trace_path, violations) + root_id, dispatch_id = _read_envelope(envelope, violations) + trace_session_ids = { + record["session_id"] for record in events if record["event"] in _SESSION_EVENTS + } + rows = _load_index(root, violations) + rows_by_id: dict[str, list[dict[str, Any]]] = {} + for row in rows: + session_id = row.get("session_id") + if isinstance(session_id, str) and session_id: + rows_by_id.setdefault(session_id, []).append(row) + owned: dict[str, _OwnedSession] = {} + root_rows = rows_by_id.get(root_id, []) + if not root_id or len(root_rows) != 1: + violations.append("sessions_index:root_missing_or_ambiguous") + else: + if root_id not in trace_session_ids: + violations.append("trace:root_session_missing") + root_row = root_rows[0] + if not dispatch_id or root_row.get("dispatch_id") != dispatch_id: + violations.append("sessions_index:root_dispatch_mismatch") + _add_owned_session(root, root_row, root_id, violations, owned) + candidates = _fresh_candidates( + root, root_id, dispatch_id, trace_session_ids, rows_by_id, violations + ) + _add_linked_descendants(root, dispatch_id, candidates, owned, violations) + observed = trace_session_ids | set(rows_by_id) + for session_id in sorted(_launched_session_ids(owned) - observed): + violations.append(f"lineage:{session_id}:launched_session_unobserved") + + return { + "root_id": root_id, + "events": events, + "owned": owned, + "violations": violations, + } + + +def _fresh_candidates( + log_root: Path, + root_id: str, + dispatch_id: str, + trace_session_ids: set[str], + rows_by_id: Mapping[str, list[dict[str, Any]]], + violations: list[str], +) -> dict[str, tuple[dict[str, Any], dict[str, Any] | None]]: + candidates: dict[str, tuple[dict[str, Any], dict[str, Any] | None]] = {} + candidate_ids = trace_session_ids | { + session_id + for session_id, rows in rows_by_id.items() + if dispatch_id and any(row.get("dispatch_id") == dispatch_id for row in rows) + } + for session_id in candidate_ids: + if session_id == root_id: + continue + matching_rows = rows_by_id.get(session_id, []) + if len(matching_rows) != 1: + violations.append(f"sessions_index:{session_id}:missing_or_ambiguous") + continue + row = matching_rows[0] + session_dir = _safe_session_dir(log_root, row) + summary = _read_summary(session_dir, session_id, violations) if session_dir else None + candidates[session_id] = row, summary + return candidates + + +def _launched_session_ids(owned: Mapping[str, _OwnedSession]) -> set[str]: + """Return the session ids named by receipted launch replies in owned transcripts.""" + launched: set[str] = set() + for item in owned.values(): + for tool, _tool_input, reply, is_error in item["calls"]: + if tool not in _LAUNCH_TOOLS or is_error or reply is None: + continue + session_id = reply.get("session_id") + if reply.get("receipt_id") and isinstance(session_id, str) and session_id: + launched.add(session_id) + return launched + + +def _report_unattributed( + remaining: Mapping[str, tuple[dict[str, Any], dict[str, Any] | None]], violations: list[str] +) -> None: + """Report sessions of the run that declare no parent and were named by no launch receipt.""" + for session_id, (row, summary) in sorted(remaining.items()): + declared = [row.get("parent_session_id"), row.get("caller_session_id")] + if summary: + declared.append(summary["execution_identity"]["parent_session_id"]) + if not any(isinstance(value, str) and value for value in declared): + violations.append(f"lineage:{session_id}:unattributed") + + +def _add_linked_descendants( + log_root: Path, + dispatch_id: str, + candidates: dict[str, tuple[dict[str, Any], dict[str, Any] | None]], + owned: dict[str, _OwnedSession], + violations: list[str], +) -> None: + remaining = dict(candidates) + while remaining: + changed = False + owned_ids = set(owned) + launched = _launched_session_ids(owned) + for session_id, (row, summary) in list(remaining.items()): + parents = { + value + for key in ("parent_session_id", "caller_session_id") + if isinstance((value := row.get(key)), str) and value + } + summary_parent = summary["execution_identity"]["parent_session_id"] if summary else "" + by_receipt = session_id in launched + linked = by_receipt or parents.intersection(owned_ids) or summary_parent in owned_ids + if not linked: + continue + if not dispatch_id or row.get("dispatch_id") != dispatch_id: + violations.append(f"sessions_index:{session_id}:dispatch_mismatch") + del remaining[session_id] + continue + if summary is None: + violations.append(f"summary:{session_id}:missing_or_malformed") + elif summary_parent not in owned_ids and (summary_parent or not by_receipt): + violations.append(f"summary:{session_id}:parent_mismatch") + if parents and not parents.intersection(owned_ids): + violations.append(f"sessions_index:{session_id}:parent_mismatch") + _add_owned_session(log_root, row, session_id, violations, owned, summary=summary) + del remaining[session_id] + changed = True + if not changed: + break + _report_unattributed(remaining, violations) + + +def _add_owned_session( + log_root: Path, + row: Mapping[str, Any], + session_id: str, + violations: list[str], + owned: dict[str, _OwnedSession], + *, + summary: dict[str, Any] | None = None, +) -> None: + session_dir = _safe_session_dir(log_root, row) + if session_dir is None: + violations.append(f"sessions_index:{session_id}:invalid_dir_name") + return + required_index_fields = ( + "session_id", + "dir_name", + "dispatch_id", + "parent_session_id", + "caller_session_id", + "claude_code_log", + ) + if any(key not in row for key in required_index_fields): + violations.append(f"sessions_index:{session_id}:missing_fields") + if summary is None: + summary = _read_summary(session_dir, session_id, violations) + if summary is None: + violations.append(f"summary:{session_id}:missing_or_malformed") + elif summary.get("dispatch_id", "") != row.get("dispatch_id", ""): + violations.append(f"summary:{session_id}:dispatch_mismatch") + transcript_value = row.get("claude_code_log") + transcript: Path | None = None + if isinstance(transcript_value, str) and transcript_value: + transcript = Path(transcript_value) + if not transcript.is_absolute(): + transcript = log_root / transcript + else: + violations.append(f"sessions_index:{session_id}:missing_transcript") + calls: list[_ToolCall] = [] + if transcript is None or not transcript.is_file(): + violations.append(f"transcript:{session_id}:missing") + else: + calls = _transcript_tool_pairs(transcript, session_id, violations) + owned[session_id] = { + "row": dict(row), + "summary": summary, + "session_dir": session_dir, + "transcript": transcript, + "calls": calls, + } + + +def _lifecycle_intervals( + events: list[dict[str, Any]], + session_ids: set[str], + violations: list[str], +) -> tuple[list[dict[str, Any]], set[str]]: + grouped = _group_lifecycle_events(events, session_ids) + intervals: list[dict[str, Any]] = [] + seen_sessions: set[str] = set() + for identity, endpoints in grouped.items(): + item = _make_interval(identity, endpoints, violations) + if item is None: + continue + if identity[0] == "session": + seen_sessions.add(identity[1]) + intervals.append(item) + return intervals, seen_sessions + + +def _group_lifecycle_events( + events: list[dict[str, Any]], session_ids: set[str] +) -> dict[tuple[str, ...], dict[str, list[dict[str, Any]]]]: + grouped: dict[tuple[str, ...], dict[str, list[dict[str, Any]]]] = {} + for event in events: + name = event["event"] + session_id = event["session_id"] + if session_id not in session_ids: + continue + identity: tuple[str, ...] + if name in _SESSION_EVENTS: + identity = ("session", session_id) + role = "start" if name == "SessionStart" else "end" + else: + identity = ("agent", session_id, event["agent_id"]) + role = "start" if name == "SubagentStart" else "end" + grouped.setdefault(identity, {"start": [], "end": []})[role].append(event) + return grouped + + +def _resolve_transcript_path(path: Path, label: str, violations: list[str]) -> Path | None: + try: + return path.resolve() + except (OSError, RuntimeError): + violations.append(f"{label}:transcript_resolution_failed") + return None + + +def _event_transcript_paths( + events: list[dict[str, Any]], + path_field: str, + label: str, + violations: list[str], +) -> tuple[set[str], bool]: + """Return resolved event paths and whether any path was supplied.""" + paths: set[str] = set() + has_path_value = False + for event in events: + path_value = event.get(path_field) + if not isinstance(path_value, str) or not path_value: + continue + has_path_value = True + resolved_path = _resolve_transcript_path(Path(path_value), label, violations) + if resolved_path is not None: + paths.add(str(resolved_path)) + return paths, has_path_value + + +def _make_interval( + identity: tuple[str, ...], + endpoints: Mapping[str, list[dict[str, Any]]], + violations: list[str], +) -> dict[str, Any] | None: + starts, stops = endpoints["start"], endpoints["end"] + label = ":".join(identity) + if not starts: + violations.append(f"lifecycle:{label}:orphan_end") + return None + if not stops: + violations.append(f"lifecycle:{label}:missing_end") + return None + start = min(float(item["timestamp"]) for item in starts) + end = max(float(item["timestamp"]) for item in stops) + if any(float(item["timestamp"]) < start for item in stops): + violations.append(f"lifecycle:{label}:orphan_end_before_start") + if end < start or any(float(event["timestamp"]) > end for event in starts): + reason = "end_before_start" if end < start else "missing_final_end" + violations.append(f"lifecycle:{label}:{reason}") + return None + item = {"kind": identity[0], "session_id": identity[1], "start": start, "end": end} + if identity[0] == "agent": + item["agent_id"] = identity[2] + path_field = "agent_transcript_path" + else: + path_field = "transcript_path" + paths, _ = _event_transcript_paths( + starts + stops, path_field, f"lifecycle:{label}", violations + ) + if len(paths) > 1: + violations.append(f"lifecycle:{label}:transcript_mismatch") + if paths: + item["transcript_path"] = sorted(paths)[0] + for key in ("agent_type", "source"): + values = {event[key] for event in starts + stops if isinstance(event.get(key), str)} + if len(values) == 1: + item[key] = next(iter(values)) + return item + + +def _check_transcript_coverage( + events: list[dict[str, Any]], + owned: Mapping[str, _OwnedSession], + violations: list[str], +) -> list[dict[str, str]]: + session_ids = set(owned) + session_paths: dict[str, set[str]] = {session_id: set() for session_id in session_ids} + observed: dict[tuple[str, str], list[dict[str, Any]]] = {} + for event in events: + session_id = event["session_id"] + if session_id not in session_ids: + continue + if event["event"] in _SESSION_EVENTS and event.get("transcript_path"): + resolved_paths, _ = _event_transcript_paths( + [event], + "transcript_path", + f"lifecycle:{session_id}", + violations, + ) + session_paths[session_id].update(resolved_paths) + elif event["event"] in _AGENT_EVENTS: + key = (session_id, event["agent_id"]) + observed.setdefault(key, []).append(event) + if not event.get("agent_type"): + violations.append(f"capture:{session_id}:{event['agent_id']}:agent_type_missing") + for session_id, paths in session_paths.items(): + if len(paths) > 1: + violations.append(f"lifecycle:{session_id}:transcript_mismatch") + enumerated = _enumerate_native_transcripts(owned, session_paths, violations) + + native_agents: list[dict[str, str]] = [] + agent_keys = set(observed) + agent_keys.update( + (session_id, agent_id) for session_id, files in enumerated.items() for agent_id in files + ) + for session_id, agent_id in sorted(agent_keys): + transcript = enumerated.get(session_id, {}).get(agent_id) + proof = _verify_native_transcript( + session_id, agent_id, transcript, observed.get((session_id, agent_id), []), violations + ) + if proof is not None: + native_agents.append(proof) + return native_agents + + +def _enumerate_native_transcripts( + owned: Mapping[str, _OwnedSession], + session_paths: Mapping[str, set[str]], + violations: list[str], +) -> dict[str, dict[str, Path]]: + enumerated: dict[str, dict[str, Path]] = {} + for session_id, item in owned.items(): + transcript = item.get("transcript") + if not isinstance(transcript, Path) or not transcript.is_file(): + continue + paths = session_paths.get(session_id, set()) + if not paths: + violations.append(f"lifecycle:{session_id}:transcript_path_missing") + else: + resolved_transcript = _resolve_transcript_path( + transcript, f"lifecycle:{session_id}", violations + ) + if resolved_transcript is not None and paths != {str(resolved_transcript)}: + violations.append(f"lifecycle:{session_id}:transcript_path_mismatch") + subagents_dir = _safe_subagents_dir(transcript, session_id, violations) + files: dict[str, Path] = {} + if subagents_dir is not None: + for path in sorted(subagents_dir.glob("agent-*.jsonl")): + agent_id = path.name.removeprefix("agent-").removesuffix(".jsonl") + if not agent_id: + continue + resolved_path = _resolve_transcript_path( + path, f"native:{session_id}:{agent_id}", violations + ) + if resolved_path is None: + continue + if not resolved_path.is_relative_to(subagents_dir): + violations.append(f"native:{session_id}:{agent_id}:transcript_outside_root") + continue + files[agent_id] = resolved_path + enumerated[session_id] = files + return enumerated + + +def _verify_native_transcript( + session_id: str, + agent_id: str, + transcript: Path | None, + events: list[dict[str, Any]], + violations: list[str], +) -> dict[str, str] | None: + label = f"native:{session_id}:{agent_id}" + names = {event["event"] for event in events} + if not _AGENT_EVENTS <= names: + violations.append(f"{label}:lifecycle_incomplete") + if transcript is None: + violations.append(f"{label}:transcript_unenumerated") + return None + if not transcript.is_file(): + violations.append(f"{label}:transcript_missing") + return None + resolved_transcript = _resolve_transcript_path(transcript, label, violations) + if resolved_transcript is None: + return None + expected_path = str(resolved_transcript) + paths, has_path_value = _event_transcript_paths( + events, "agent_transcript_path", label, violations + ) + if not paths and not has_path_value: + violations.append(f"{label}:transcript_path_missing") + elif paths and paths != {expected_path}: + violations.append(f"{label}:transcript_mismatch") + for _ in _iter_jsonl(transcript, f"transcript:{session_id}:{agent_id}", violations): + pass + return {"session_id": session_id, "agent_id": agent_id, "transcript_path": expected_path} + + +def _peak_sessions(intervals: list[dict[str, Any]]) -> int: + points: list[tuple[float, int]] = [] + for item in intervals: + if item["start"] == item["end"]: + continue + points.append((item["end"], -1)) + points.append((item["start"], 1)) + active = peak = 0 + for _timestamp, delta in sorted(points, key=lambda point: (point[0], point[1])): + active += delta + peak = max(peak, active) + return peak + + +def summarize( + trace_path: Path, + envelope: dict[str, Any] | None, + log_root: Path | None = None, +) -> dict[str, Any]: + """Write a bounded concurrency summary beside the raw lifecycle trace.""" + context = _discover_owned(trace_path, envelope, log_root) + violations = list(context["violations"]) + owned: dict[str, _OwnedSession] = context["owned"] + owned_ids = set(owned) + intervals, seen_sessions = _lifecycle_intervals(context["events"], owned_ids, violations) + for session_id in sorted(owned_ids - seen_sessions): + violations.append(f"lifecycle:{session_id}:missing_endpoints") + native_agents = _check_transcript_coverage(context["events"], owned, violations) + root_id = context["root_id"] + root_seen = root_id in seen_sessions + summary = { + "method": "hook_lifecycle_intervals", + "trace_path": str(trace_path), + "peak_sessions": _peak_sessions(intervals), + "identities": sorted( + intervals, + key=lambda item: ( + item["start"], + item["kind"], + item["session_id"], + item.get("agent_id", ""), + ), + ), + "coverage": { + "complete": not violations, + "root_session_id": root_id or None, + "root_session_seen": root_seen, + "managed_session_ids": sorted(owned_ids - ({root_id} if root_id else set())), + "owned_session_ids": sorted(owned_ids), + "native_agents": native_agents, + }, + "violations": sorted(set(violations)), + } + output_path = trace_path.with_name("session-concurrency.json") + output_path.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return summary + + +def _issue_number(issue_url: str) -> str: + match = re.search(r"/issues/(\d+)/?$", urlsplit(issue_url).path) + if match: + return match.group(1) + match = re.search(r"#(\d+)$", issue_url) + return match.group(1) if match else "" + + +def _issue_matches_repository(issue_url: str, repository: str) -> bool: + shorthand = re.fullmatch(r"([^/]+)/([^#]+)#\d+", issue_url) + if shorthand: + issue_repository = f"{shorthand.group(1)}/{shorthand.group(2)}" + return issue_repository.lower() == repository.lower() + + parsed = urlsplit(issue_url) + return ( + parsed.scheme == "https" + and parsed.netloc.lower() == "github.com" + and (parsed.path.strip("/").split("/issues/", 1)[0].lower() == repository.lower()) + ) + + +def _remote_matches_repository(remote_url: Any, repository: str) -> bool: + if not isinstance(remote_url, str): + return False + https = re.fullmatch(r"https://github\.com/([^/]+/[^/]+?)(?:\.git)?/?", remote_url) + ssh = re.fullmatch(r"git@github\.com:([^/]+/[^/]+)\.git", remote_url) + match = https or ssh + return bool(match and match.group(1).lower() == repository.lower()) + + +def _result_object(value: Any) -> dict[str, Any] | None: + if isinstance(value, str): + try: + value = json.loads(value) + except json.JSONDecodeError: + return None + if isinstance(value, dict): + if any( + key in value for key in ("success", "claimed", "merge_target", "error", "error_type") + ): + return value + nested = value.get("structuredContent", value.get("result", value.get("content"))) + if isinstance(nested, list): + return _result_text_blocks(nested) + elif isinstance(nested, (dict, str)): + return _result_object(nested) + if isinstance(value, list): + return _result_text_blocks(value) + return None + + +def _result_text_blocks(blocks: list[Any]) -> dict[str, Any] | None: + for block in blocks: + if not isinstance(block, dict) or not isinstance(block.get("text"), str): + continue + try: + decoded = json.loads(block["text"]) + except json.JSONDecodeError: + continue + if isinstance(decoded, dict): + return decoded + return None + + +def _transcript_tool_pairs( + transcript: Path, + session_id: str, + violations: list[str], +) -> list[_ToolCall]: + """Return paired target calls as name, input, result, and result-error flag.""" + pending: dict[str, tuple[str, dict[str, Any]]] = {} + paired: list[_ToolCall] = [] + for line_number, row in _iter_jsonl(transcript, f"transcript:{session_id}", violations): + _pair_record(row, line_number, session_id, pending, paired, violations) + for tool_id, (name, tool_input) in pending.items(): + violations.append(f"transcript:{session_id}:unpaired_tool_use:{tool_id}") + paired.append((name, tool_input, None, False)) + return paired + + +def _receipt( + row: Mapping[str, Any], content: list[Any], block: Mapping[str, Any] +) -> dict[str, Any] | None: + """Decode row metadata when present, otherwise decode the JSON block carrier.""" + meta = row.get("mcpMeta") + if "mcpMeta" in row: + results = [ + item + for item in content + if isinstance(item, dict) and item.get("type") == "tool_result" + ] + if len(results) != 1 or not isinstance(meta, dict): + return None + return _result_object(meta.get("structuredContent")) + return _result_object(block.get("content")) + + +def _pair_record( + row: dict[str, Any], + line_number: int, + session_id: str, + pending: dict[str, tuple[str, dict[str, Any]]], + paired: list[_ToolCall], + violations: list[str], +) -> None: + message = row.get("message") + if not isinstance(message, dict): + message = row + content = message.get("content", row.get("content", [])) + if not isinstance(content, list): + return + role = row.get("type", message.get("role")) + for block in content: + if not isinstance(block, dict): + continue + if role == "assistant" and block.get("type") == "tool_use": + _remember_tool_use(block, line_number, session_id, pending, violations) + elif role == "user" and block.get("type") == "tool_result": + tool_id = block.get("tool_use_id") + if isinstance(tool_id, str) and tool_id in pending: + name, tool_input = pending.pop(tool_id) + result = _receipt(row, content, block) + paired.append((name, tool_input, result, block.get("is_error") is True)) + + +def _remember_tool_use( + block: dict[str, Any], + line_number: int, + session_id: str, + pending: dict[str, tuple[str, dict[str, Any]]], + violations: list[str], +) -> None: + name = block.get("name") + if not isinstance(name, str): + return + parts = tuple(name.split("__")) + tool = parts[-1] + if tool not in _TRACKED_TOOLS: + return + if len(parts) != 3 or parts[:2] not in _AUTOSKILLIT_SERVERS: + violations.append(f"transcript:{session_id}:unrecognised_tool_namespace:{line_number}") + return + tool_id, tool_input = block.get("id"), block.get("input") + if not isinstance(tool_id, str) or not tool_id or not isinstance(tool_input, dict): + violations.append(f"transcript:{session_id}:malformed_tool_use:{line_number}") + return + if tool_id in pending: + violations.append(f"transcript:{session_id}:duplicate_tool_use:{line_number}") + pending[tool_id] = (tool, tool_input) + + +def collect_ownership( + trace_path: Path, + envelope: dict[str, Any] | None, + log_root: Path | None = None, + issue_url: str = "", + repository: str = "", + claim_label: str | None = None, +) -> OwnershipProof: + """Collect compact branch and claim proofs from the owned transcript lineage.""" + context = _discover_owned(trace_path, envelope, log_root) + violations = list(context["violations"]) + root_id = context["root_id"] + expected_number = _issue_number(issue_url) + valid_issue_reference = bool( + expected_number and repository and _issue_matches_repository(issue_url, repository) + ) + if not valid_issue_reference: + violations.append("ownership:invalid_issue_reference") + + branches, claim_calls, transcript_evidence = _owned_tool_proofs( + context["owned"], + root_id, + valid_issue_reference, + expected_number, + issue_url, + repository, + violations, + ) + claimed = _claim_succeeded(claim_calls, expected_number, claim_label, violations) + return { + "branches": sorted(set(branches)), + "claimed": claimed, + "transcripts": sorted(transcript_evidence, key=lambda item: item["session_id"]), + "violations": sorted(set(violations)), + } + + +def _owned_tool_proofs( + owned: Mapping[str, _OwnedSession], + root_id: str, + valid_issue_reference: bool, + expected_number: str, + issue_url: str, + repository: str, + violations: list[str], +) -> tuple[ + list[str], list[tuple[dict[str, Any], dict[str, Any] | None, bool]], list[dict[str, str]] +]: + branches: list[str] = [] + claims: list[tuple[dict[str, Any], dict[str, Any] | None, bool]] = [] + transcripts: list[dict[str, str]] = [] + for session_id, item in owned.items(): + transcript = item.get("transcript") + if not isinstance(transcript, Path) or not transcript.is_file(): + continue + transcripts.append({"session_id": session_id, "path": str(transcript)}) + for name, tool_input, result, is_error in item["calls"]: + target = _branch_target( + session_id, + root_id, + name, + tool_input, + result, + is_error, + valid_issue_reference, + expected_number, + repository, + violations, + ) + if target is not None: + branches.append(target) + if name == "claim_and_resolve_issue" and valid_issue_reference: + if tool_input.get("issue_url") == issue_url: + claims.append((tool_input, result, is_error)) + return branches, claims, transcripts + + +def _branch_target( + session_id: str, + root_id: str, + name: str, + tool_input: dict[str, Any], + result: dict[str, Any] | None, + is_error: bool, + valid_issue_reference: bool, + expected_number: str, + repository: str, + violations: list[str], +) -> str | None: + if name != "create_and_publish_branch" or session_id != root_id or not valid_issue_reference: + return None + if tool_input.get("issue_number") != expected_number: + return None + if not _remote_matches_repository(tool_input.get("remote_url"), repository): + return None + if ( + is_error + or result is None + or "error" in result + or "error_type" in result + or not isinstance(result.get("merge_target"), str) + or not result["merge_target"].strip() + ): + violations.append("ownership:branch_result_failed") + return None + return result["merge_target"].strip() + + +def _claim_succeeded( + claims: list[tuple[dict[str, Any], dict[str, Any] | None, bool]], + expected_number: str, + claim_label: str | None, + violations: list[str], +) -> bool: + if len(claims) > 1: + violations.append("ownership:claim_reentry") + return False + if not claims: + return False + tool_input, result, is_error = claims[0] + requested_label = tool_input.get("label") + label_matches_request = claim_label is None or requested_label in (None, "", claim_label) + claimed = bool( + not is_error + and label_matches_request + and tool_input.get("allow_reentry") is not True + and result is not None + and result.get("success") is True + and result.get("claimed") is True + and str(result.get("issue_number", "")) == expected_number + and result.get("reentry") is not True + ) + if not claimed: + violations.append("ownership:claim_result_failed") + return claimed + + +def main(argv: list[str] | None = None) -> int: + args = sys.argv[1:] if argv is None else argv + if len(args) != 1: + print("usage: e2e_sessions.py ", file=sys.stderr) + return 2 + try: + raw = sys.stdin.buffer.read(_MAX_HOOK_INPUT_BYTES + 1) + if len(raw) > _MAX_HOOK_INPUT_BYTES: + raise ValueError("lifecycle hook input exceeds the size limit") + payload = json.loads(raw) + if not isinstance(payload, dict): + raise ValueError("lifecycle hook input must be an object") + capture_hook_event(Path(args[0]), payload) + except (OSError, ValueError) as exc: + print(f"e2e lifecycle capture failed: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/autoskillit/core/types/audit/_type_plan_set_authority.py b/src/autoskillit/core/types/audit/_type_plan_set_authority.py index cbf64aa99a..24b16c79c9 100644 --- a/src/autoskillit/core/types/audit/_type_plan_set_authority.py +++ b/src/autoskillit/core/types/audit/_type_plan_set_authority.py @@ -499,8 +499,17 @@ def _validate_collections(self) -> None: } if set(self.coverage.parts_without_obligations) != empty_parts: raise ValueError( - "coverage.parts_without_obligations must match actual empty parts" + "coverage.parts_without_obligations must match actual empty parts " + f"(inventory_mode={self.inventory_mode.value!r}, " + f"declared_parts={tuple(sorted(self.coverage.parts_without_obligations))!r}, " + f"actual_empty_parts={tuple(sorted(empty_parts))!r})" ) + elif self.coverage.parts_without_obligations: + raise ValueError( + "coverage.parts_without_obligations is invalid for enumerated inventory " + f"(inventory_mode={self.inventory_mode.value!r}, " + f"declared_parts={tuple(sorted(self.coverage.parts_without_obligations))!r})" + ) @classmethod def create( diff --git a/src/autoskillit/recipe/helpers/_io_loading.py b/src/autoskillit/recipe/helpers/_io_loading.py index d799ad8099..9b6f6c72d7 100644 --- a/src/autoskillit/recipe/helpers/_io_loading.py +++ b/src/autoskillit/recipe/helpers/_io_loading.py @@ -58,7 +58,7 @@ def _substitute_recipe_note_placeholders(value: Any) -> Any: - """Substitute recipe note placeholders. Only strings reach this layer.""" + """Expand note placeholders in strings, leaving non-string values unchanged.""" if isinstance(value, str): for placeholder, note in _RECIPE_NOTE_TABLE.items(): value = value.replace(placeholder, note) diff --git a/tests/_hook_protocol_oracle.py b/tests/_hook_protocol_oracle.py index f3fd523aa0..1124ffa1be 100644 --- a/tests/_hook_protocol_oracle.py +++ b/tests/_hook_protocol_oracle.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import re from collections.abc import Callable, Mapping from pathlib import Path from typing import Any, Literal, NamedTuple @@ -444,6 +445,24 @@ def codex_verdict( ) +_EXACT_MATCHER = re.compile(r"[A-Za-z0-9_\- ,|]*") + + +def claude_matcher_admits(matcher: str | None, value: str) -> bool: + """Apply Claude Code's documented hook-matcher rule to one candidate value. + + Source: the "Matcher patterns" section of the page named by + ``CLAUDE_PROTOCOL_SOURCE``, re-read 2026-10-13. Python's ``re.search`` stands in + for ``RegExp.prototype.test``. ``FileChanged`` and ``StopFailure`` use a narrower + exact-match set and are not modelled. + """ + if matcher is None or matcher in ("", "*"): + return True + if _EXACT_MATCHER.fullmatch(matcher): + return value in {item.strip() for item in re.split(r"[|,]", matcher)} + return re.search(matcher, value) is not None + + # These keys and event exceptions come from Claude's JSON output and # per-event decision-control tables in hooks.md. CLAUDE_HOOK_SPECIFIC_OUTPUT_FIELDS: Mapping[str, frozenset[str]] = { diff --git a/tests/_script_loader.py b/tests/_script_loader.py new file mode 100644 index 0000000000..d2cdccd19a --- /dev/null +++ b/tests/_script_loader.py @@ -0,0 +1,41 @@ +"""Load standalone scripts with isolated per-test module registration.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path +from types import ModuleType + + +def load_script(name: str, path: Path) -> ModuleType: + """Load one standalone script fresh via importlib, registering it in sys.modules under *name*. + + Each caller passes its own unique *name* so dataclasses defined in the loaded script + resolve string annotations through sys.modules[cls.__module__] without colliding with a + sibling test file's copy of the same script (see + tests/arch/test_acceptance_policy_relaxation_gate.py). + + The script's parent directory is temporarily added to sys.path so sibling imports + resolve during execution. The caller's sys.path is restored afterward. + """ + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None + assert spec.loader is not None + mod = importlib.util.module_from_spec(spec) + previous_path = list(sys.path) + missing = object() + previous_module = sys.modules.get(name, missing) + sys.modules[name] = mod + sys.path.insert(0, str(path.parent)) + try: + spec.loader.exec_module(mod) + except BaseException: + if previous_module is missing: + sys.modules.pop(name, None) + else: + sys.modules[name] = previous_module + raise + finally: + sys.path[:] = previous_path + return mod diff --git a/tests/_session_log_helpers.py b/tests/_session_log_helpers.py new file mode 100644 index 0000000000..0aaf55d398 --- /dev/null +++ b/tests/_session_log_helpers.py @@ -0,0 +1,153 @@ +"""Shared builders for session log artifacts in tests.""" + +from __future__ import annotations + +from pathlib import Path + + +def _snap( + *, + captured_at: str | None = "2026-03-03T12:00:00+00:00", + state: str = "sleeping", + vm_rss_kb: int = 100000, + oom_score: int = 50, + fd_count: int = 10, + fd_soft_limit: int = 1024, + sig_pnd: str = "0000000000000000", + sig_blk: str = "0000000000000000", + sig_cgt: str = "0000000000000000", + threads: int = 4, + wchan: str = "", + ctx_switches_voluntary: int = 500, + ctx_switches_involuntary: int = 20, + cpu_percent: float = 0.0, +) -> dict[str, object]: + snapshot: dict[str, object] = { + "state": state, + "vm_rss_kb": vm_rss_kb, + "oom_score": oom_score, + "fd_count": fd_count, + "fd_soft_limit": fd_soft_limit, + "sig_pnd": sig_pnd, + "sig_blk": sig_blk, + "sig_cgt": sig_cgt, + "threads": threads, + "wchan": wchan, + "ctx_switches_voluntary": ctx_switches_voluntary, + "ctx_switches_involuntary": ctx_switches_involuntary, + "cpu_percent": cpu_percent, + } + if captured_at is not None: + snapshot["captured_at"] = captured_at + return snapshot + + +def _flush( + tmp_path: Path, *, backend: str = "claude-code", session_locator=None, **overrides +) -> None: + from autoskillit.core import ExecutionIdentity, FaultDomain + from autoskillit.core.types.results._type_results import ( + InfraOutcome, + ModelIdentity, + ProviderOutcome, + ) + from autoskillit.core.types.results._type_results_execution import ( + RecipeIdentity, + SessionTelemetry, + ) + from autoskillit.execution.session_log.session_log import flush_session_log + + defaults: dict = { + "log_dir": str(tmp_path), + "cwd": "/home/test/project", + "evidence_worktree": None, + "session_id": "test-session-001", + "pid": 12345, + "skill_command": "/autoskillit:investigate some error", + "success": True, + "needs_retry": False, + "retry_reason": "none", + "infra": InfraOutcome( + exit_category="completed", + fault_domain=FaultDomain.LOGIC, + ), + "api_error_status": None, + "is_error": False, + "subtype": "completed", + "exit_code": 0, + "start_ts": "2026-03-03T12:00:00+00:00", + "proc_snapshots": [_snap(), _snap(), _snap()], + "dispatch_id": "", + "caller_session_id": "", + "github_api_log": None, + "token_usage": None, + "turn_usage": [], + "timing_seconds": None, + "audit_record": None, + "loc_insertions": 0, + "loc_deletions": 0, + "subagent_model_outcomes": (), + "child_outcomes": (), + "model_identity": ModelIdentity.unknown(), + "execution_identity": ExecutionIdentity.empty(), + } + defaults.update(overrides) + + # Convert legacy model_identifier + profile_name to ModelIdentity if callers pass them + _model_identifier = defaults.pop("model_identifier", None) + _profile_name = defaults.pop("profile_name", None) + if _model_identifier is not None or _profile_name is not None: + _mid = _model_identifier or "" + _pn = _profile_name or "" + if _pn and _pn != "anthropic": + defaults["model_identity"] = ModelIdentity.for_provider( + configured=_mid, effective="", profile=_pn + ) + elif _mid: + defaults["model_identity"] = ModelIdentity.anthropic(_mid) + else: + defaults["model_identity"] = ModelIdentity.unknown() + + # Extract telemetry kwargs and build SessionTelemetry before forwarding + _github_api_log = defaults.pop("github_api_log") + _session_id = defaults.get("session_id", "") + _api_usage = _github_api_log.drain(_session_id) if _github_api_log is not None else None + telemetry = SessionTelemetry( + token_usage=defaults.pop("token_usage"), + turn_usage=defaults.pop("turn_usage"), + timing_seconds=defaults.pop("timing_seconds"), + audit_record=defaults.pop("audit_record"), + github_api_usage=_api_usage, + github_api_requests=_api_usage.get("total_requests", 0) if _api_usage else 0, + loc_insertions=defaults.pop("loc_insertions"), + loc_deletions=defaults.pop("loc_deletions"), + subagent_model_outcomes=defaults.pop("subagent_model_outcomes"), + child_outcomes=defaults.pop("child_outcomes"), + execution_identity=defaults.pop("execution_identity"), + ) + + # Extract provider/recipe kwargs and build typed containers + _provider_used = defaults.pop("provider_used", "") + _provider_fallback = defaults.pop("provider_fallback", False) + _recipe_name = defaults.pop("recipe_name", "") + _recipe_content_hash = defaults.pop("recipe_content_hash", "") + _recipe_composite_hash = defaults.pop("recipe_composite_hash", "") + _recipe_version = defaults.pop("recipe_version", "") + provider_outcome = ProviderOutcome( + provider_used=_provider_used, + fallback_activated=_provider_fallback, + ) + recipe_identity = RecipeIdentity( + name=_recipe_name, + content_hash=_recipe_content_hash, + composite_hash=_recipe_composite_hash, + version=_recipe_version, + ) + flush_session_log( + **defaults, + telemetry=telemetry, + provider_outcome=provider_outcome, + recipe_identity=recipe_identity, + backend=backend, + session_locator=session_locator, + ) diff --git a/tests/arch/test_complexity_limits.py b/tests/arch/test_complexity_limits.py index 3ae34dca49..ec1d22307b 100644 --- a/tests/arch/test_complexity_limits.py +++ b/tests/arch/test_complexity_limits.py @@ -7,8 +7,8 @@ import pytest +from tests._script_loader import load_script from tests.arch._acceptance_policy_surfaces import POLICY_SURFACES -from tests.infra._complexity_helpers import load_check_script pytestmark = [pytest.mark.layer("arch"), pytest.mark.small] @@ -16,7 +16,7 @@ _CHECK_SCRIPT = REPO_ROOT / "scripts" / "check_complexity.py" _CHECK_MODULE_NAME = "_autoskillit_check_complexity_limits" -check = load_check_script(_CHECK_MODULE_NAME, _CHECK_SCRIPT) +check = load_script(_CHECK_MODULE_NAME, _CHECK_SCRIPT) git_plumbing = sys.modules["_git_plumbing"] diff --git a/tests/arch/test_e2e_sessions_contract.py b/tests/arch/test_e2e_sessions_contract.py new file mode 100644 index 0000000000..3c42ca01de --- /dev/null +++ b/tests/arch/test_e2e_sessions_contract.py @@ -0,0 +1,158 @@ +"""Pin the standalone collector's identities, tools and hook policy to the product.""" + +import json +from collections.abc import Sequence +from pathlib import Path + +import pytest + +from autoskillit.core import DIRECT_PREFIX, PLUGIN_PREFIX, ClaudeFlags, pkg_root +from autoskillit.core.types import ( + GATED_TOOLS, + UNGATED_TOOLS, + ChildTaskTranscript, + SessionLocator, + SessionSummary, +) +from autoskillit.execution.backends import claude_resolvable_agent_names +from tests._hook_protocol_oracle import claude_matcher_admits +from tests._script_loader import load_script +from tests._session_log_helpers import _flush + +pytestmark = pytest.mark.medium + +sessions = load_script( + "_autoskillit_e2e_sessions_contract", + Path(__file__).resolve().parents[2] / "scripts" / "e2e" / "e2e_sessions.py", +) + + +class _TestSessionLocator(SessionLocator): + def __init__(self, transcript: Path, session_id: str) -> None: + self._transcript = transcript + self._session_id = session_id + + def list_sessions(self, cwd: str) -> Sequence[SessionSummary]: + return () + + def locate_session(self, session_id: str) -> Path | None: + return self._transcript if session_id == self._session_id else None + + def project_log_dir(self, cwd: str) -> Path: + return self._transcript.parent + + def session_log_path(self, cwd: str, session_id: str) -> Path | None: + return self._transcript if session_id == self._session_id else None + + def session_context_was_compacted(self, cwd: str, session_id: str) -> bool: + return False + + def read_child_task(self, child_id: str) -> ChildTaskTranscript | None: + return None + + +def test_server_identities_match_canonical_prefixes() -> None: + assert sessions._AUTOSKILLIT_SERVERS == frozenset( + tuple(prefix.split("__")[:-1]) for prefix in (DIRECT_PREFIX, PLUGIN_PREFIX) + ) + + +def test_tracked_tools_are_registered_product_tools() -> None: + assert sessions._TRACKED_TOOLS + assert { + "claim_and_resolve_issue", + "create_and_publish_branch", + "run_skill", + "complete_run_skill_result", + "recover_run_skill_result", + } <= sessions._TRACKED_TOOLS + assert sessions._TRACKED_TOOLS <= GATED_TOOLS | UNGATED_TOOLS + + +def test_agent_hooks_admit_every_resolvable_agent_and_reject_an_unnamed_one() -> None: + names = claude_resolvable_agent_names() + assert names + for event in sessions._AGENT_EVENTS: + matcher = sessions.LIFECYCLE_HOOK_MATCHERS[event] + assert matcher is not None + assert not claude_matcher_admits(matcher, "") + assert all(claude_matcher_admits(matcher, name) for name in names) + + +def test_session_hooks_stay_unfiltered() -> None: + matchers = sessions.LIFECYCLE_HOOK_MATCHERS + assert sessions._SESSION_EVENTS == frozenset({"SessionStart", "SessionEnd"}) + assert set(matchers) == sessions._LIFECYCLE_EVENTS + assert all(matchers[event] is None for event in sessions._SESSION_EVENTS) + + +def test_collector_consumes_production_session_provenance(tmp_path: Path) -> None: + from autoskillit.core.types import ExecutionIdentity + from autoskillit.fleet import DispatchCompleted, DispatchEffectProvenance, DispatchStatus + + log_root = tmp_path / "logs" + dispatch_id = "dispatch-contract" + root_id = "root-contract-session" + child_id = "child-contract-session" + child_identity = ExecutionIdentity(parent_session_id=root_id) + + def write_session(session_id: str, parent_session_id: str, caller_session_id: str) -> None: + transcript = tmp_path / "transcripts" / f"{session_id}.jsonl" + transcript.parent.mkdir(parents=True, exist_ok=True) + transcript.write_text("", encoding="utf-8") + locator: SessionLocator = _TestSessionLocator(transcript, session_id) + _flush( + log_root, + session_id=session_id, + dispatch_id=dispatch_id, + caller_session_id=caller_session_id, + execution_identity=ExecutionIdentity(parent_session_id=parent_session_id), + session_locator=locator, + ) + + write_session(root_id, "", "") + write_session(child_id, root_id, root_id) + + payload = {"result": "completed"} + envelope = json.loads( + DispatchCompleted( + dispatch_status=DispatchStatus.SUCCESS, + dispatch_id=dispatch_id, + dispatched_session_id=root_id, + reason="", + effect_provenance=DispatchEffectProvenance(operation_id=dispatch_id), + l3_payload=payload, + ).to_envelope() + ) + assert envelope["l3_payload"] == payload + + trace_path = tmp_path / "trace.jsonl" + trace_path.write_text( + "".join( + json.dumps({"event": "SessionStart", "timestamp": index, "session_id": session_id}) + + "\n" + for index, session_id in enumerate((root_id, child_id), start=1) + ), + encoding="utf-8", + ) + discovered = sessions._discover_owned(trace_path, envelope, log_root) + + assert discovered["root_id"] == root_id + assert discovered["owned"][root_id]["row"]["dispatch_id"] == dispatch_id + assert set(discovered["owned"]) == {root_id, child_id} + assert ( + discovered["owned"][child_id]["summary"]["execution_identity"] == child_identity.to_dict() + ) + assert not discovered["violations"] + + +def test_product_selects_no_main_agent() -> None: + # Internal-agent callbacks are unnamed only while no main agent is set. + assert {"--agent", "--settings"}.isdisjoint(flag.value for flag in ClaudeFlags) + root = pkg_root() + manifest = json.loads((root / ".claude-plugin" / "plugin.json").read_text(encoding="utf-8")) + assert "agent" not in manifest.get("settings", {}) + plugin_settings = root / "settings.json" + assert not plugin_settings.exists() or "agent" not in json.loads( + plugin_settings.read_text(encoding="utf-8") + ) diff --git a/tests/arch/test_mcp_prefix_literal_containment.py b/tests/arch/test_mcp_prefix_literal_containment.py index aba6020538..268612aae9 100644 --- a/tests/arch/test_mcp_prefix_literal_containment.py +++ b/tests/arch/test_mcp_prefix_literal_containment.py @@ -8,6 +8,7 @@ import pytest from autoskillit.core import find_qualified_autoskillit_tool_names, paths +from tests.arch._helpers import SRC_ROOT pytestmark = [pytest.mark.layer("arch"), pytest.mark.small] @@ -41,18 +42,27 @@ def _string_literals_in_module(path: Path) -> list[tuple[int, str]]: def test_mcp_prefix_literals_confined_to_canonical_module() -> None: """The string literals for both MCP prefix forms appear only in _plugin_ids.py.""" src_root = paths.pkg_root() + scan_roots = ( + (src_root, "src/autoskillit"), + (SRC_ROOT.parents[1] / "scripts", "scripts"), + ) violations: list[str] = [] - for py_file in sorted(src_root.rglob("*.py")): - rel = py_file.relative_to(src_root).as_posix() - if rel == _CANONICAL_MODULE: - continue - for lineno, value in _string_literals_in_module(py_file): - for literal in _CONFINED_LITERALS: - if literal in value: - violations.append(f"{rel}:{lineno}: contains {literal!r} in {value!r}") + for root, root_label in scan_roots: + files = sorted(path for path in root.rglob("*.py") if "__pycache__" not in path.parts) + assert files, f"{root} scan universe must not be empty" + for py_file in files: + rel = py_file.relative_to(root).as_posix() + if root == src_root and rel == _CANONICAL_MODULE: + continue + for lineno, value in _string_literals_in_module(py_file): + for literal in _CONFINED_LITERALS: + if literal in value: + violations.append( + f"{root_label}/{rel}:{lineno}: contains {literal!r} in {value!r}" + ) assert not violations, ( - "MCP prefix literals must appear only in " - f"{_CANONICAL_MODULE}. Violations:\n" + "\n".join(violations) + f"MCP prefix literals must be confined to {_CANONICAL_MODULE} while scanning " + "src/autoskillit/ and scripts/. Violations:\n" + "\n".join(violations) ) diff --git a/tests/contracts/test_skill_result_persistence_ledger.py b/tests/contracts/test_skill_result_persistence_ledger.py index fe05eafc39..2d7970ba4b 100644 --- a/tests/contracts/test_skill_result_persistence_ledger.py +++ b/tests/contracts/test_skill_result_persistence_ledger.py @@ -16,7 +16,7 @@ import pytest from autoskillit.core import ExecutionIdentity, FaultDomain, InfraOutcome, SkillResult -from tests.execution.conftest import _flush +from tests._session_log_helpers import _flush pytestmark = pytest.mark.medium diff --git a/tests/core/types/test_plan_set_authority.py b/tests/core/types/test_plan_set_authority.py index 19d236496f..7b3442c949 100644 --- a/tests/core/types/test_plan_set_authority.py +++ b/tests/core/types/test_plan_set_authority.py @@ -85,19 +85,35 @@ def test_create_digest_and_canonical_round_trip(tmp_path: Path) -> None: }, "unknown part key", ), + ( + { + "inventory_mode": InventoryMode.ENUMERATED, + "coverage": CoverageResultDef( + CoverageStatus.PASS, parts_without_obligations=("P1",) + ), + }, + "invalid for enumerated inventory", + ), ], ) def test_invalid_authority_states_fail_closed( tmp_path: Path, overrides: dict[str, object], message: str ) -> None: - with pytest.raises(ValueError, match=message): + with pytest.raises(ValueError, match=message) as exc_info: _authority(tmp_path, **overrides) + if overrides.get("inventory_mode") is InventoryMode.ENUMERATED: + assert "inventory_mode='enumerated'" in str(exc_info.value) + assert "declared_parts=('P1',)" in str(exc_info.value) @pytest.mark.parametrize("mode", (InventoryMode.NO_ISSUE, InventoryMode.UNENUMERATED)) def test_authority_rejects_omitted_empty_parts(tmp_path: Path, mode: InventoryMode) -> None: - with pytest.raises(ValueError, match="must match actual empty parts"): + with pytest.raises(ValueError, match="must match actual empty parts") as exc_info: _authority(tmp_path, inventory_mode=mode, allocations=()) + message = str(exc_info.value) + assert f"inventory_mode={mode.value!r}" in message + assert "declared_parts=()" in message + assert "actual_empty_parts=('P1',)" in message def test_part_ordinals_and_resolved_locators_are_unique(tmp_path: Path) -> None: diff --git a/tests/execution/conftest.py b/tests/execution/conftest.py index 48bd3c20a6..2d0a36f363 100644 --- a/tests/execution/conftest.py +++ b/tests/execution/conftest.py @@ -7,7 +7,6 @@ import subprocess import textwrap from collections.abc import Callable -from pathlib import Path from types import SimpleNamespace from typing import Any from unittest.mock import Mock @@ -20,8 +19,6 @@ BackendAuthorityTier, BackendCapabilities, CmdSpec, - ExecutionIdentity, - FaultDomain, InfraExitCategory, LaunchPreparation, LaunchResolutionRequest, @@ -421,43 +418,6 @@ def isolated_tracing_config(tmp_path: pathlib.Path): return make_tracing_config(enabled=True, proc_interval=0.05, tmpfs_path=str(shm)) -def _snap( - *, - captured_at: str | None = "2026-03-03T12:00:00+00:00", - state: str = "sleeping", - vm_rss_kb: int = 100000, - oom_score: int = 50, - fd_count: int = 10, - fd_soft_limit: int = 1024, - sig_pnd: str = "0000000000000000", - sig_blk: str = "0000000000000000", - sig_cgt: str = "0000000000000000", - threads: int = 4, - wchan: str = "", - ctx_switches_voluntary: int = 500, - ctx_switches_involuntary: int = 20, - cpu_percent: float = 0.0, -) -> dict[str, object]: - d: dict[str, object] = { - "state": state, - "vm_rss_kb": vm_rss_kb, - "oom_score": oom_score, - "fd_count": fd_count, - "fd_soft_limit": fd_soft_limit, - "sig_pnd": sig_pnd, - "sig_blk": sig_blk, - "sig_cgt": sig_cgt, - "threads": threads, - "wchan": wchan, - "ctx_switches_voluntary": ctx_switches_voluntary, - "ctx_switches_involuntary": ctx_switches_involuntary, - "cpu_percent": cpu_percent, - } - if captured_at is not None: - d["captured_at"] = captured_at - return d - - def _result_ndjson( result_text: str = "done", subtype: str = "success", @@ -677,116 +637,6 @@ def make_write_evidence_scope(**overrides: Any) -> WriteEvidenceScope: return WriteEvidenceScope(**values) -def _flush( - tmp_path: Path, *, backend: str = "claude-code", session_locator=None, **overrides -) -> None: - from autoskillit.core.types.results._type_results import ( - InfraOutcome, - ModelIdentity, - ProviderOutcome, - ) - from autoskillit.core.types.results._type_results_execution import ( - RecipeIdentity, - SessionTelemetry, - ) - from autoskillit.execution.session_log.session_log import flush_session_log - - defaults: dict = { - "log_dir": str(tmp_path), - "cwd": "/home/test/project", - "evidence_worktree": None, - "session_id": "test-session-001", - "pid": 12345, - "skill_command": "/autoskillit:investigate some error", - "success": True, - "needs_retry": False, - "retry_reason": "none", - "infra": InfraOutcome( - exit_category="completed", - fault_domain=FaultDomain.LOGIC, - ), - "api_error_status": None, - "is_error": False, - "subtype": "completed", - "exit_code": 0, - "start_ts": "2026-03-03T12:00:00+00:00", - "proc_snapshots": [_snap(), _snap(), _snap()], - "dispatch_id": "", - "caller_session_id": "", - "github_api_log": None, - "token_usage": None, - "turn_usage": [], - "timing_seconds": None, - "audit_record": None, - "loc_insertions": 0, - "loc_deletions": 0, - "subagent_model_outcomes": (), - "child_outcomes": (), - "model_identity": ModelIdentity.unknown(), - "execution_identity": ExecutionIdentity.empty(), - } - defaults.update(overrides) - - # Convert legacy model_identifier + profile_name to ModelIdentity if callers pass them - _model_identifier = defaults.pop("model_identifier", None) - _profile_name = defaults.pop("profile_name", None) - if _model_identifier is not None or _profile_name is not None: - _mid = _model_identifier or "" - _pn = _profile_name or "" - if _pn and _pn != "anthropic": - defaults["model_identity"] = ModelIdentity.for_provider( - configured=_mid, effective="", profile=_pn - ) - elif _mid: - defaults["model_identity"] = ModelIdentity.anthropic(_mid) - else: - defaults["model_identity"] = ModelIdentity.unknown() - - # Extract telemetry kwargs and build SessionTelemetry before forwarding - _github_api_log = defaults.pop("github_api_log") - _session_id = defaults.get("session_id", "") - _api_usage = _github_api_log.drain(_session_id) if _github_api_log is not None else None - telemetry = SessionTelemetry( - token_usage=defaults.pop("token_usage"), - turn_usage=defaults.pop("turn_usage"), - timing_seconds=defaults.pop("timing_seconds"), - audit_record=defaults.pop("audit_record"), - github_api_usage=_api_usage, - github_api_requests=_api_usage.get("total_requests", 0) if _api_usage else 0, - loc_insertions=defaults.pop("loc_insertions"), - loc_deletions=defaults.pop("loc_deletions"), - subagent_model_outcomes=defaults.pop("subagent_model_outcomes"), - child_outcomes=defaults.pop("child_outcomes"), - execution_identity=defaults.pop("execution_identity"), - ) - - # Extract provider/recipe kwargs and build typed containers - _provider_used = defaults.pop("provider_used", "") - _provider_fallback = defaults.pop("provider_fallback", False) - _recipe_name = defaults.pop("recipe_name", "") - _recipe_content_hash = defaults.pop("recipe_content_hash", "") - _recipe_composite_hash = defaults.pop("recipe_composite_hash", "") - _recipe_version = defaults.pop("recipe_version", "") - provider_outcome = ProviderOutcome( - provider_used=_provider_used, - fallback_activated=_provider_fallback, - ) - recipe_identity = RecipeIdentity( - name=_recipe_name, - content_hash=_recipe_content_hash, - composite_hash=_recipe_composite_hash, - version=_recipe_version, - ) - flush_session_log( - **defaults, - telemetry=telemetry, - provider_outcome=provider_outcome, - recipe_identity=recipe_identity, - backend=backend, - session_locator=session_locator, - ) - - @pytest.fixture def merge_group_only_repo_state() -> dict[str, Any]: """A fixture modelling a repo whose CI only triggers on merge_group. diff --git a/tests/execution/test_anomaly_detection.py b/tests/execution/test_anomaly_detection.py index 7481e3f1a4..526258f6ca 100644 --- a/tests/execution/test_anomaly_detection.py +++ b/tests/execution/test_anomaly_detection.py @@ -14,7 +14,7 @@ detect_model_drift, normalize_model_id, ) -from tests.execution.conftest import _snap as _snap_full +from tests._session_log_helpers import _snap as _snap_full pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] diff --git a/tests/execution/test_flag_contracts.py b/tests/execution/test_flag_contracts.py index a027254c7d..1285dff070 100644 --- a/tests/execution/test_flag_contracts.py +++ b/tests/execution/test_flag_contracts.py @@ -7,10 +7,14 @@ one place, and all tests catch the ripple. """ +from pathlib import Path + import pytest from autoskillit.core import ClaudeFlags +from autoskillit.core.types import OutputFormat from autoskillit.execution.backends.claude import ClaudeCodeBackend +from tests.execution.backends._plugin_binding import plugin_binding pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] @@ -69,6 +73,37 @@ def _extract_flags(cmd: tuple[str, ...] | list[str]) -> set[str]: class TestFlagRegistryAudit: + @pytest.mark.parametrize("fmt", list(OutputFormat)) + def test_skill_food_truck_resume_cmd_flags_are_all_registered(self, fmt: OutputFormat) -> None: + backend = ClaudeCodeBackend() + skill = backend.build_skill_session_cmd( + "/investigate foo", + cwd="/tmp", + completion_marker="%%DONE%%", + model=None, + plugin_binding=None, + output_format=fmt, + ) + with plugin_binding(Path("/p")) as binding: + root = backend.build_food_truck_cmd( + orchestrator_prompt="Orchestrator", + plugin_binding=binding, + cwd="/tmp", + completion_marker="%%DONE%%", + output_format=fmt, + ) + resume = backend.build_resume_cmd( + resume_session_id="abc", prompt="Emit", output_format=fmt + ) + for name, spec in (("skill", skill), ("food_truck", root), ("resume", resume)): + unknown = _extract_flags(spec.cmd) - KNOWN_CLAUDE_FLAGS + assert not unknown, f"{name}: {unknown}" + assert ClaudeFlags.OUTPUT_FORMAT in spec.cmd, f"{name}: missing output-format flag" + output_format_index = spec.cmd.index(ClaudeFlags.OUTPUT_FORMAT) + assert spec.cmd[output_format_index + 1 : output_format_index + 2] == (fmt.value,), ( + f"{name}: expected output format {fmt.value!r}" + ) + def test_interactive_cmd_flags_are_all_registered(self): result = ClaudeCodeBackend().build_interactive_cmd(model="claude-sonnet-4-6") unknown = _extract_flags(result.cmd) - KNOWN_CLAUDE_FLAGS diff --git a/tests/execution/test_flush_completeness_guard.py b/tests/execution/test_flush_completeness_guard.py index 7fa8aa5991..fad1109ab8 100644 --- a/tests/execution/test_flush_completeness_guard.py +++ b/tests/execution/test_flush_completeness_guard.py @@ -7,7 +7,7 @@ import pytest -from tests.execution.conftest import _flush +from tests._session_log_helpers import _flush pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] diff --git a/tests/execution/test_headless_enum_recovery.py b/tests/execution/test_headless_enum_recovery.py index 57dfcf5efc..96c243507d 100644 --- a/tests/execution/test_headless_enum_recovery.py +++ b/tests/execution/test_headless_enum_recovery.py @@ -23,15 +23,16 @@ from autoskillit.recipe._contracts_types import SkillContract, SkillOutput from autoskillit.recipe.contracts import get_skill_contract, load_bundled_manifest from tests.conftest import _make_result -from tests.execution.conftest import _launch_inputs, _mock_backend, _success_session_json +from tests.execution.conftest import ( + _launch_inputs, + _mock_backend, + _success_session_json, + _unmarked_rollout_snapshot, +) pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] -def _unmarked_rollout_snapshot() -> tuple[None, str]: - return None, "2000-01-01T00:00:00+00:00" - - def _make_plan_contract( *, write_behavior: str | None = "conditional", diff --git a/tests/execution/test_linux_tracing_pty_integration.py b/tests/execution/test_linux_tracing_pty_integration.py index 86562cedcb..f597d3a250 100644 --- a/tests/execution/test_linux_tracing_pty_integration.py +++ b/tests/execution/test_linux_tracing_pty_integration.py @@ -17,7 +17,8 @@ import pytest -from tests.execution.conftest import _ALLOCATE_60MB_SCRIPT, _flush +from tests._session_log_helpers import _flush +from tests.execution.conftest import _ALLOCATE_60MB_SCRIPT pytestmark = [ pytest.mark.layer("execution"), diff --git a/tests/execution/test_model_backend_launch_contract.py b/tests/execution/test_model_backend_launch_contract.py index 76d3e4a061..173b2f1843 100644 --- a/tests/execution/test_model_backend_launch_contract.py +++ b/tests/execution/test_model_backend_launch_contract.py @@ -17,7 +17,7 @@ from autoskillit.core import LaunchValueSourceKind from autoskillit.core.types import LaunchContractError, RetryReason, SkillResult -from .conftest import _backend_authority, _launch_inputs, _mock_backend +from .conftest import _backend_authority, _launch_inputs, _mock_backend, _unmarked_rollout_snapshot pytestmark = [pytest.mark.layer("execution"), pytest.mark.small, pytest.mark.model_contract] @@ -180,7 +180,7 @@ async def test_rejected_admission_does_not_capture_rollout_or_invoke_runner(tmp_ from autoskillit.execution.headless._headless_launch import _run_headless_attempt kwargs, runner = _build_codex_attempt_kwargs("gpt-6-sol", tmp_path) - capture = Mock(return_value=(None, "2000-01-01T00:00:00+00:00")) + capture = Mock(side_effect=_unmarked_rollout_snapshot) kwargs["capture_rollout_on_launch"] = capture async def reject(contract): @@ -226,7 +226,7 @@ def build_spec(_binding, _extras, _managed_attempt_id=None): "launch_resolver": resolver, "launch_preparation": preparation, "expected_launch_contract": None, - "capture_rollout_on_launch": lambda: (None, "2000-01-01T00:00:00+00:00"), + "capture_rollout_on_launch": _unmarked_rollout_snapshot, "plugin_authority": None, "plugin_load_mode": PluginLoadMode.NONE, "provider_extras": None, diff --git a/tests/execution/test_otlp_sink.py b/tests/execution/test_otlp_sink.py index b5da8b0937..4379e6f864 100644 --- a/tests/execution/test_otlp_sink.py +++ b/tests/execution/test_otlp_sink.py @@ -19,7 +19,7 @@ from autoskillit.core import CODEX_MODEL_ALIASES from autoskillit.execution.evidence.otlp_sink import _SIGNALS as _OTLP_SIGNALS -from tests.execution.conftest import _flush +from tests._session_log_helpers import _flush pytestmark = [pytest.mark.layer("execution"), pytest.mark.large] diff --git a/tests/execution/test_session_decision_roundtrip.py b/tests/execution/test_session_decision_roundtrip.py index db8f2e9bad..33ba7521c0 100644 --- a/tests/execution/test_session_decision_roundtrip.py +++ b/tests/execution/test_session_decision_roundtrip.py @@ -13,7 +13,7 @@ ) from autoskillit.execution.backends.claude import ClaudeCodeBackend from autoskillit.execution.headless._headless_result import _build_skill_result -from tests.execution.conftest import _flush +from tests._session_log_helpers import _flush pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] diff --git a/tests/execution/test_session_index_roundtrip.py b/tests/execution/test_session_index_roundtrip.py index 8eb6196790..aa94d5c1e8 100644 --- a/tests/execution/test_session_index_roundtrip.py +++ b/tests/execution/test_session_index_roundtrip.py @@ -8,7 +8,7 @@ import pytest from autoskillit.execution.session_log.session_index import iter_tolerant_session_index_lines -from tests.execution.conftest import _flush +from tests._session_log_helpers import _flush pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] diff --git a/tests/execution/test_session_log_backend_source.py b/tests/execution/test_session_log_backend_source.py index b3b821fd4b..6782179ddb 100644 --- a/tests/execution/test_session_log_backend_source.py +++ b/tests/execution/test_session_log_backend_source.py @@ -122,7 +122,7 @@ def test_session_index_entry_default_value(self) -> None: def test_typed_backend_authority_in_summary(self, tmp_path): import json - from tests.execution.conftest import _flush + from tests._session_log_helpers import _flush _flush( tmp_path, @@ -138,7 +138,7 @@ def test_typed_backend_authority_in_summary(self, tmp_path): def test_typed_backend_authority_in_sessions_jsonl(self, tmp_path): import json - from tests.execution.conftest import _flush + from tests._session_log_helpers import _flush _flush( tmp_path, @@ -153,7 +153,7 @@ def test_typed_backend_authority_in_sessions_jsonl(self, tmp_path): def test_launch_evidence_empty_when_unavailable_round_trip(self, tmp_path): import json - from tests.execution.conftest import _flush + from tests._session_log_helpers import _flush _flush(tmp_path) summary = json.loads( @@ -194,7 +194,7 @@ class TestChildOutcomesProjection: def test_child_outcomes_round_trip(self, tmp_path): import json - from tests.execution.conftest import _flush + from tests._session_log_helpers import _flush _flush(tmp_path, child_outcomes=[self._ROW_UNKNOWN]) summary = json.loads( @@ -213,7 +213,7 @@ def test_reused_recovery_refreshes_child_outcomes_in_committed_summary_and_index rewrites unconditionally) reflects it too.""" import json - from tests.execution.conftest import _flush + from tests._session_log_helpers import _flush _flush(tmp_path, child_outcomes=[self._ROW_UNKNOWN]) refined_row = {**self._ROW_UNKNOWN, "terminal_reason": "completed"} @@ -233,7 +233,7 @@ def test_reused_recovery_leaves_other_summary_fields_untouched(self, tmp_path): the already-committed summary (e.g. an unrelated field stays as first-flushed).""" import json - from tests.execution.conftest import _flush + from tests._session_log_helpers import _flush _flush(tmp_path, child_outcomes=[self._ROW_UNKNOWN], skill_command="/first:command") refined_row = {**self._ROW_UNKNOWN, "terminal_reason": "completed"} diff --git a/tests/execution/test_session_log_fields.py b/tests/execution/test_session_log_fields.py index 9b5ce7e04c..00af6e9d6f 100644 --- a/tests/execution/test_session_log_fields.py +++ b/tests/execution/test_session_log_fields.py @@ -22,12 +22,11 @@ ) from autoskillit.core.types.results._type_results import ModelIdentity from tests._helpers import make_rollout_marker +from tests._session_log_helpers import _flush, _snap from tests.execution.conftest import ( - _flush, _make_cc_jsonl_record, _make_thinking_block, _make_tool_block, - _snap, ) pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] diff --git a/tests/execution/test_session_log_flush.py b/tests/execution/test_session_log_flush.py index c5c090e1db..fc8d3aea78 100644 --- a/tests/execution/test_session_log_flush.py +++ b/tests/execution/test_session_log_flush.py @@ -28,7 +28,8 @@ from tests._helpers import INVALID_PATH_INPUTS from tests._helpers import UNKNOWN_MEASURE as _UNKNOWN from tests._helpers import observed_measure as _observed -from tests.execution.conftest import _flush, _make_cc_jsonl_record, _snap +from tests._session_log_helpers import _flush, _snap +from tests.execution.conftest import _make_cc_jsonl_record pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] diff --git a/tests/execution/test_session_log_integration.py b/tests/execution/test_session_log_integration.py index 103590cb60..714b07d4f1 100644 --- a/tests/execution/test_session_log_integration.py +++ b/tests/execution/test_session_log_integration.py @@ -12,7 +12,8 @@ import anyio import pytest -from tests.execution.conftest import _ALLOCATE_60MB_SCRIPT, _flush +from tests._session_log_helpers import _flush +from tests.execution.conftest import _ALLOCATE_60MB_SCRIPT pytestmark = [ pytest.mark.layer("execution"), diff --git a/tests/execution/test_session_log_retention.py b/tests/execution/test_session_log_retention.py index 7355fef8d4..2cb3edd670 100644 --- a/tests/execution/test_session_log_retention.py +++ b/tests/execution/test_session_log_retention.py @@ -32,7 +32,7 @@ RECLAIMER_CONVERGENCE_CASES, assert_second_pass_is_quiet, ) -from tests.execution.conftest import _flush, _snap +from tests._session_log_helpers import _flush, _snap pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] diff --git a/tests/execution/test_smoke_codex.py b/tests/execution/test_smoke_codex.py index 586c581877..d8ca879478 100644 --- a/tests/execution/test_smoke_codex.py +++ b/tests/execution/test_smoke_codex.py @@ -26,8 +26,8 @@ ) from autoskillit.execution.process import run_managed_async from autoskillit.recipe._api import load_and_validate +from tests._session_log_helpers import _flush from tests.execution.backends._plugin_binding import plugin_binding -from tests.execution.conftest import _flush pytestmark = [pytest.mark.layer("execution"), pytest.mark.large] diff --git a/tests/hooks/test_hook_protocol_oracle.py b/tests/hooks/test_hook_protocol_oracle.py index a66d8c8e79..42f3aa34cf 100644 --- a/tests/hooks/test_hook_protocol_oracle.py +++ b/tests/hooks/test_hook_protocol_oracle.py @@ -18,6 +18,7 @@ STATUS_COMPLETED, STATUS_FAILED, STATUS_STOPPED, + claude_matcher_admits, claude_verdict, codex_verdict, ) @@ -25,6 +26,40 @@ pytestmark = [pytest.mark.layer("hooks"), pytest.mark.small] +@pytest.mark.parametrize( + ("matcher", "value", "admitted"), + [ + (None, "Bash", True), + (None, "", True), + ("", "Bash", True), + ("", "", True), + ("*", "Bash", True), + ("*", "", True), + ("Bash", "Bash", True), + ("Bash", "BashOutput", False), + ("Edit|Write", "Edit", True), + ("Edit|Write", "Write", True), + ("Edit|Write", "NotebookEdit", False), + ("Edit, Write", "Edit", True), + ("Edit, Write", "Write", True), + ("Edit, Write", "NotebookEdit", False), + ("code-reviewer", "code-reviewer", True), + ("code-reviewer", "other-code-reviewer", False), + ("^Notebook", "NotebookEdit", True), + ("Edit.*", "NotebookEdit", True), + ("^Edit$", "NotebookEdit", False), + ("^my-plugin:reviewer$", "my-plugin:reviewer", True), + ("general-purpose", "", False), + (".+", "", False), + (".*", "", True), + ], +) +def test_claude_matcher_follows_the_documented_rule( + matcher: str | None, value: str, admitted: bool +) -> None: + assert claude_matcher_admits(matcher, value) is admitted + + @pytest.mark.parametrize("event", ["PreCompact", "PostCompact"]) def test_codex_compact_halt_preserves_system_message(event: str) -> None: # codex-rs/hooks/src/events/compact.rs:260-278 records warnings before stopping. diff --git a/tests/infra/_complexity_helpers.py b/tests/infra/_complexity_helpers.py index eb30befa04..2cd8604e3f 100644 --- a/tests/infra/_complexity_helpers.py +++ b/tests/infra/_complexity_helpers.py @@ -2,57 +2,18 @@ Used by tests/infra/test_check_complexity.py, tests/infra/test_check_complexity_git_e2e.py, tests/infra/test_check_complexity_ruff_parity.py, and tests/arch/test_complexity_limits.py. -Each caller keeps its own _CHECK_MODULE_NAME / module-loading call site (per-file -sys.modules isolation stays a parameter, not shared state) -- only the byte-identical -bodies live here, matching the tests/infra/_pretty_output_helpers.py convention of a -dedicated underscore-prefixed helper module for one feature's split test files. - -tests/infra/test_e2e_select.py and tests/infra/test_e2e_harness.py also load -scripts/e2e/e2e_select.py and scripts/e2e/e2e_harness.py through load_check_script, so -their sibling ``import e2e_catalog`` resolves. +Generic script loading is shared through tests._script_loader. """ from __future__ import annotations -import importlib.util import subprocess -import sys import textwrap from pathlib import Path _CHECK_COMPLEXITY_GIT_TIMEOUT_SECONDS = 30 -def load_check_script(name: str, path: Path): - """Load one gate script fresh via importlib, registering it in sys.modules under *name*. - - Each caller passes its own unique *name* so dataclasses defined in the loaded script - resolve string annotations through sys.modules[cls.__module__] without colliding with a - sibling test file's copy of the same script (see - tests/arch/test_acceptance_policy_relaxation_gate.py). - """ - spec = importlib.util.spec_from_file_location(name, path) - assert spec is not None - assert spec.loader is not None - mod = importlib.util.module_from_spec(spec) - previous_path = list(sys.path) - missing = object() - previous_module = sys.modules.get(name, missing) - sys.modules[name] = mod - sys.path.insert(0, str(path.parent)) - try: - spec.loader.exec_module(mod) - except BaseException: - if previous_module is missing: - sys.modules.pop(name, None) - else: - sys.modules[name] = previous_module - raise - finally: - sys.path[:] = previous_path - return mod - - def _git(repo: Path, *args: str) -> None: subprocess.run( ["git", *args], diff --git a/tests/infra/_e2e_evidence.py b/tests/infra/_e2e_evidence.py new file mode 100644 index 0000000000..2656826adb --- /dev/null +++ b/tests/infra/_e2e_evidence.py @@ -0,0 +1,50 @@ +"""Load and materialize captured E2E session evidence fixtures.""" + +from __future__ import annotations + +import json +import shutil +from pathlib import Path +from typing import Any + +_PROJECTS_TOKEN = "@CLAUDE_PROJECTS@" + + +def fixture_dir(name: str) -> Path: + return Path(__file__).with_name("fixtures") / "e2e_sessions" / name + + +def load_provenance(name: str) -> dict[str, Any]: + """Read the provenance record for a captured E2E run.""" + return json.loads((fixture_dir(name) / "PROVENANCE.json").read_text(encoding="utf-8")) + + +def _replace_projects_token(path: Path, projects: Path) -> None: + text = path.read_text(encoding="utf-8") + replacement = json.dumps(str(projects))[1:-1] + path.write_text(text.replace(_PROJECTS_TOKEN, replacement), encoding="utf-8") + + +def materialize_run( + name: str, + *, + trace: Path, + log_root: Path, + projects: Path, +) -> dict[str, Any]: + """Copy a captured run to the given locations and return its envelope.""" + fixture = fixture_dir(name) + trace.parent.mkdir(parents=True, exist_ok=True) + log_root.mkdir(parents=True, exist_ok=True) + projects.mkdir(parents=True, exist_ok=True) + + shutil.copyfile(fixture / "out/session-events.jsonl", trace) + shutil.copytree(fixture / "logs", log_root, dirs_exist_ok=True) + shutil.copytree(fixture / "projects", projects, dirs_exist_ok=True) + + _replace_projects_token(trace, projects) + _replace_projects_token(log_root / "sessions.jsonl", projects) + for summary in (log_root / "sessions").glob("*/summary.json"): + _replace_projects_token(summary, projects) + + return json.loads((fixture / "out/envelope.json").read_text(encoding="utf-8")) diff --git a/tests/infra/conftest.py b/tests/infra/conftest.py index 074a4e932b..f010b88cd6 100644 --- a/tests/infra/conftest.py +++ b/tests/infra/conftest.py @@ -220,5 +220,5 @@ def _get_formatter_coverage_registry() -> dict[str, FormatterCoverageDef]: return _FORMATTER_COVERAGE_REGISTRY -# check_complexity's shared test infra (load_check_script, _git, _CONSTRUCT_CASES, etc.) +# check_complexity's shared test infra (_git, _CONSTRUCT_CASES, etc.) # lives in tests/infra/_complexity_helpers.py, not here -- see that module's docstring. diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/PROVENANCE.json b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/PROVENANCE.json new file mode 100644 index 0000000000..a654b8af7b --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/PROVENANCE.json @@ -0,0 +1,13 @@ +{ + "producer": "claude-code", + "cli_version": "2.1.280", + "source_run": "38170521660", + "issue_url": "https://github.com/TalonT-Org/autoskillit-e2e-sandbox/issues/1", + "repository": "TalonT-Org/autoskillit-e2e-sandbox", + "claim_label": "in-progress", + "sanitization": "Derived from the retained run artifact. The root transcript retains rows 1-3, 54-55, 61-62, 66-67, 81, 84, 88-89, 93-94, 547-548, 552-553, 569, 679, 680, and 682; each retained row keeps every source key. Worker transcripts retain their first three and final two rows; native subagent transcripts retain their first and final rows. Prose strings longer than 400 characters are truncated with a removed-character marker, while the claim and branch tool inputs remain complete. Serialized JSON carriers, including mcpMeta.structuredContent.result, are parsed, recursively sanitized, and reserialized so structural and ownership fields remain intact. The container prefix /home/autoskillit/.claude/projects is replaced with @CLAUDE_PROJECTS@ in the event trace, session index, and summaries; other /home/ paths are tokenized. Index and summary data retain identity fields needed to resolve the captured lineage. logs/skill-session-contracts/ and logs/otlp.jsonl are excluded.", + "capture_notes": [ + "The retained sessions index and assistant model fields report MiniMax-M3; transcript cost-state modelUsage keys report claude-sonnet-5. Both values are preserved as captured.", + "The raw session-events trace retains an unmatched SubagentStop for agent a610a4bda1f28ca35 with empty agent_type and no matching SubagentStart. The event is preserved as captured; replay under the declared lifecycle matcher excludes the empty-type stop." + ] +} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions.jsonl b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions.jsonl new file mode 100644 index 0000000000..52f67da240 --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions.jsonl @@ -0,0 +1,3 @@ +{"session_id":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","dir_name":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","dispatch_id":"4666c3fd-486f-4ce1-b51f-9d9d660a5560","parent_session_id":"","caller_session_id":"","claude_code_log":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl","session_type":"skill","step_name":"plan","order_id":"4666c3fd-486f-4ce1-b51f-9d9d660a5560","campaign_id":"48915f19-d220-4069-8d55-b81efff84c05","kitchen_id":"48915f19-d220-4069-8d55-b81efff84c05","timestamp":"2026-10-11T20:05:22.655547+00:00","claude_code_version":"2.1.280 (Claude Code)","autoskillit_version":"0.10.1271","backend":"claude-code","provider_used":"anthropic","model_identifier":"MiniMax-M3","configured_model":"sonnet","success":true,"schema_version":17,"recipe_name":"implementation","recipe_version":"1.0.0","requested_parent_backend":"claude-code","requested_parent_model":"sonnet"} +{"session_id":"54f18144-1188-45d7-a392-17b4529f721e","dir_name":"54f18144-1188-45d7-a392-17b4529f721e","dispatch_id":"4666c3fd-486f-4ce1-b51f-9d9d660a5560","parent_session_id":"","caller_session_id":"","claude_code_log":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e.jsonl","session_type":"skill","step_name":"verify","order_id":"4666c3fd-486f-4ce1-b51f-9d9d660a5560","campaign_id":"48915f19-d220-4069-8d55-b81efff84c05","kitchen_id":"48915f19-d220-4069-8d55-b81efff84c05","timestamp":"2026-10-11T20:11:57.256584+00:00","claude_code_version":"2.1.280 (Claude Code)","autoskillit_version":"0.10.1271","backend":"claude-code","provider_used":"anthropic","model_identifier":"MiniMax-M3","configured_model":"sonnet","success":false,"schema_version":17,"recipe_name":"implementation","recipe_version":"1.0.0","requested_parent_backend":"claude-code","requested_parent_model":"sonnet"} +{"session_id":"2292658a-34cf-44a0-a0b1-f7a20218874b","dir_name":"2292658a-34cf-44a0-a0b1-f7a20218874b","dispatch_id":"4666c3fd-486f-4ce1-b51f-9d9d660a5560","parent_session_id":"","caller_session_id":"","claude_code_log":"@CLAUDE_PROJECTS@/-workspace-sandbox/2292658a-34cf-44a0-a0b1-f7a20218874b.jsonl","session_type":"orchestrator","step_name":"","order_id":"4666c3fd-486f-4ce1-b51f-9d9d660a5560","campaign_id":"48915f19-d220-4069-8d55-b81efff84c05","kitchen_id":"48915f19-d220-4069-8d55-b81efff84c05","timestamp":"2026-10-11T20:04:36.539164+00:00","claude_code_version":"2.1.280 (Claude Code)","autoskillit_version":"0.10.1271","backend":"claude-code","provider_used":"anthropic","model_identifier":"MiniMax-M3","configured_model":"sonnet","success":true,"schema_version":17,"recipe_name":"","recipe_version":"","requested_parent_backend":"","requested_parent_model":""} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions/2292658a-34cf-44a0-a0b1-f7a20218874b/summary.json b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions/2292658a-34cf-44a0-a0b1-f7a20218874b/summary.json new file mode 100644 index 0000000000..d13696603e --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions/2292658a-34cf-44a0-a0b1-f7a20218874b/summary.json @@ -0,0 +1,23 @@ +{ + "backend": "claude-code", + "campaign_id": "48915f19-d220-4069-8d55-b81efff84c05", + "dispatch_id": "4666c3fd-486f-4ce1-b51f-9d9d660a5560", + "end_ts": "2026-10-11T20:16:23.869672+00:00", + "execution_identity": { + "children": [], + "cli_version": "", + "effective_parent_backend": "", + "effective_parent_effort": "", + "effective_parent_model": "", + "override_key_path": "", + "override_tier": "", + "parent_session_id": "", + "requested_parent_backend": "", + "requested_parent_effort": "", + "requested_parent_model": "" + }, + "provider_used": "anthropic", + "session_id": "2292658a-34cf-44a0-a0b1-f7a20218874b", + "session_type": "orchestrator", + "start_ts": "2026-10-11T20:04:36.539164+00:00" +} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions/54f18144-1188-45d7-a392-17b4529f721e/summary.json b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions/54f18144-1188-45d7-a392-17b4529f721e/summary.json new file mode 100644 index 0000000000..38000961d4 --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions/54f18144-1188-45d7-a392-17b4529f721e/summary.json @@ -0,0 +1,23 @@ +{ + "backend": "claude-code", + "campaign_id": "48915f19-d220-4069-8d55-b81efff84c05", + "dispatch_id": "4666c3fd-486f-4ce1-b51f-9d9d660a5560", + "end_ts": "2026-10-11T20:14:46.893573+00:00", + "execution_identity": { + "children": [], + "cli_version": "", + "effective_parent_backend": "", + "effective_parent_effort": "", + "effective_parent_model": "", + "override_key_path": "", + "override_tier": "", + "parent_session_id": "", + "requested_parent_backend": "claude-code", + "requested_parent_effort": "", + "requested_parent_model": "sonnet" + }, + "provider_used": "anthropic", + "session_id": "54f18144-1188-45d7-a392-17b4529f721e", + "session_type": "skill", + "start_ts": "2026-10-11T20:11:57.256584+00:00" +} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions/8ef8f993-8caa-47dd-83b0-26f9e5b51139/summary.json b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions/8ef8f993-8caa-47dd-83b0-26f9e5b51139/summary.json new file mode 100644 index 0000000000..e2c1ce5cc3 --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions/8ef8f993-8caa-47dd-83b0-26f9e5b51139/summary.json @@ -0,0 +1,23 @@ +{ + "backend": "claude-code", + "campaign_id": "48915f19-d220-4069-8d55-b81efff84c05", + "dispatch_id": "4666c3fd-486f-4ce1-b51f-9d9d660a5560", + "end_ts": "2026-10-11T20:07:16.866414+00:00", + "execution_identity": { + "children": [], + "cli_version": "", + "effective_parent_backend": "", + "effective_parent_effort": "", + "effective_parent_model": "", + "override_key_path": "", + "override_tier": "", + "parent_session_id": "", + "requested_parent_backend": "claude-code", + "requested_parent_effort": "", + "requested_parent_model": "sonnet" + }, + "provider_used": "anthropic", + "session_id": "8ef8f993-8caa-47dd-83b0-26f9e5b51139", + "session_type": "skill", + "start_ts": "2026-10-11T20:05:22.655547+00:00" +} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/out/envelope.json b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/out/envelope.json new file mode 100644 index 0000000000..ed1e998ea1 --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/out/envelope.json @@ -0,0 +1,114 @@ +{ + "kind": "completed", + "success": false, + "dispatch_status": "resumable", + "dispatch_id": "4666c3fd-486f-4ce1-b51f-9d9d660a5560", + "dispatched_session_id": "2292658a-34cf-44a0-a0b1-f7a20218874b", + "reason": "fleet_l3_no_result_block", + "effect_provenance": { + "operation_id": "b6908330d53b40b98656d73e3a8fdb05", + "aggregate_phase": "started", + "retry_disposition": "resume_by_identity", + "effects": [ + { + "name": "campaign_state_write", + "phase": "confirmed", + "effect_id": "b6908330d53b40b98656d73e3a8fdb05:campaign_state_write", + "retry_relevant": true, + "confirmation_receipt": "per-dispatch state and captures persisted", + "known_downstream_identities": { + "dispatch_id": "4666c3fd-486f-4ce1-b51f-9d9d660a5560", + "state_path": "/workspace/sandbox/.autoskillit/temp/dispatches/4666c3fd-486f-4ce1-b51f-9d9d660a5560.json" + }, + "ambiguity": "" + }, + { + "name": "child_discovery", + "phase": "confirmed", + "effect_id": "b6908330d53b40b98656d73e3a8fdb05:child_discovery", + "retry_relevant": true, + "confirmation_receipt": "captured one process identity tuple", + "known_downstream_identities": { + "boot_id": "95f60de8-f823-4b7a-abc9-9e3f15310bc6", + "create_time": "1791749076.16", + "dispatch_id": "4666c3fd-486f-4ce1-b51f-9d9d660a5560", + "identity_degraded": "False", + "pid": "106", + "starttime_ticks": "15116" + }, + "ambiguity": "" + }, + { + "name": "dispatch_allocation", + "phase": "confirmed", + "effect_id": "b6908330d53b40b98656d73e3a8fdb05:dispatch_allocation", + "retry_relevant": true, + "confirmation_receipt": "per-dispatch state identity persisted", + "known_downstream_identities": { + "dispatch_id": "4666c3fd-486f-4ce1-b51f-9d9d660a5560", + "state_path": "/workspace/sandbox/.autoskillit/temp/dispatches/4666c3fd-486f-4ce1-b51f-9d9d660a5560.json" + }, + "ambiguity": "" + }, + { + "name": "process_spawn", + "phase": "confirmed", + "effect_id": "b6908330d53b40b98656d73e3a8fdb05:process_spawn", + "retry_relevant": true, + "confirmation_receipt": "executor reported spawned process and authoritative session identity", + "known_downstream_identities": { + "dispatch_id": "4666c3fd-486f-4ce1-b51f-9d9d660a5560", + "dispatched_session_id": "2292658a-34cf-44a0-a0b1-f7a20218874b", + "pid": "106", + "starttime_ticks": "15116" + }, + "ambiguity": "" + } + ], + "cancel_requested": false, + "local_cleanup": null, + "state_cleanup_confirmed": false, + "labels_cleanup_confirmed": false + }, + "token_usage": { + "backend": "claude-code", + "provider_used": "anthropic", + "input_tokens": { + "state": "measured", + "value": 205063 + }, + "output_tokens": { + "state": "measured", + "value": 39864 + }, + "cache_read_tokens": { + "state": "measured", + "value": 11571251 + }, + "cache_write_tokens": { + "state": "measured_zero", + "value": 0 + }, + "peak_context": { + "state": "unknown", + "value": null + } + }, + "l3_payload": null, + "l3_parse_source": "stdout", + "lifespan_started": true, + "stderr": "", + "elapsed_seconds": 708.0250108242035, + "error": "fleet_l3_no_result_block", + "user_visible_message": "fleet_l3_no_result_block", + "resume_checkpoint": { + "completed_items": [ + "plan" + ], + "step_name": "plan", + "progress_pct": 0.007407407407407408, + "ts": "2026-10-11T20:07:18.776245+00:00", + "backend_name": "claude-code", + "skill_name": "implementation" + } +} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/out/session-events.jsonl b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/out/session-events.jsonl new file mode 100644 index 0000000000..54806953b0 --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/out/session-events.jsonl @@ -0,0 +1,16 @@ +{"event":"SessionStart","timestamp":1791749076.9067402,"session_id":"2292658a-34cf-44a0-a0b1-f7a20218874b","source":"startup","transcript_path":"@CLAUDE_PROJECTS@/-workspace-sandbox/2292658a-34cf-44a0-a0b1-f7a20218874b.jsonl"} +{"event":"SessionStart","timestamp":1791749123.0673993,"session_id":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","source":"startup","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl"} +{"event":"SubagentStart","timestamp":1791749157.9630604,"session_id":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","agent_id":"a9ed51b7c02b828dd","agent_type":"autoskillit:plan-interface-mapper","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl"} +{"event":"SubagentStart","timestamp":1791749158.0156765,"session_id":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","agent_id":"aeb96cc0ddbdab294","agent_type":"autoskillit:plan-foundation-auditor","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl"} +{"event":"SubagentStart","timestamp":1791749158.1193779,"session_id":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","agent_id":"a516cbae85151dc7c","agent_type":"autoskillit:plan-registry-tracer","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl"} +{"event":"SubagentStop","timestamp":1791749183.3175266,"session_id":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","agent_id":"a516cbae85151dc7c","agent_type":"autoskillit:plan-registry-tracer","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl","agent_transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-a516cbae85151dc7c.jsonl"} +{"event":"SubagentStop","timestamp":1791749183.933718,"session_id":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","agent_id":"a9ed51b7c02b828dd","agent_type":"autoskillit:plan-interface-mapper","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl","agent_transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-a9ed51b7c02b828dd.jsonl"} +{"event":"SubagentStop","timestamp":1791749190.3404713,"session_id":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","agent_id":"aeb96cc0ddbdab294","agent_type":"autoskillit:plan-foundation-auditor","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl","agent_transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-aeb96cc0ddbdab294.jsonl"} +{"event":"SessionEnd","timestamp":1791749233.8509,"session_id":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl"} +{"event":"SessionStart","timestamp":1791749517.653047,"session_id":"54f18144-1188-45d7-a392-17b4529f721e","source":"startup","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e.jsonl"} +{"event":"SubagentStart","timestamp":1791749551.111115,"session_id":"54f18144-1188-45d7-a392-17b4529f721e","agent_id":"a7e613554b849d6b2","agent_type":"general-purpose","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e.jsonl"} +{"event":"SubagentStop","timestamp":1791749679.8737457,"session_id":"54f18144-1188-45d7-a392-17b4529f721e","agent_id":"a7e613554b849d6b2","agent_type":"general-purpose","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e.jsonl","agent_transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e/subagents/agent-a7e613554b849d6b2.jsonl"} +{"event":"SessionEnd","timestamp":1791749686.7281744,"session_id":"54f18144-1188-45d7-a392-17b4529f721e","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e.jsonl"} +{"event":"SubagentStop","timestamp":1791749722.6609342,"session_id":"2292658a-34cf-44a0-a0b1-f7a20218874b","agent_id":"a610a4bda1f28ca35","agent_type":"","transcript_path":"@CLAUDE_PROJECTS@/-workspace-sandbox/2292658a-34cf-44a0-a0b1-f7a20218874b.jsonl","agent_transcript_path":"@CLAUDE_PROJECTS@/-workspace-sandbox/2292658a-34cf-44a0-a0b1-f7a20218874b/subagents/agent-a610a4bda1f28ca35.jsonl"} +{"event":"SessionStart","timestamp":1791749722.700151,"session_id":"2292658a-34cf-44a0-a0b1-f7a20218874b","source":"compact","transcript_path":"@CLAUDE_PROJECTS@/-workspace-sandbox/2292658a-34cf-44a0-a0b1-f7a20218874b.jsonl"} +{"event":"SessionEnd","timestamp":1791749780.901994,"session_id":"2292658a-34cf-44a0-a0b1-f7a20218874b","transcript_path":"@CLAUDE_PROJECTS@/-workspace-sandbox/2292658a-34cf-44a0-a0b1-f7a20218874b.jsonl"} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e.jsonl b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e.jsonl new file mode 100644 index 0000000000..5fc748643b --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e.jsonl @@ -0,0 +1,5 @@ +{"type":"queue-operation","operation":"enqueue","timestamp":"2026-10-11T20:12:00.782Z","sessionId":"54f18144-1188-45d7-a392-17b4529f721e","content":"Use the /dry-walkthrough skill AUTOSKILLIT_BOUND_INVOCATION_V1\n{\"skill_inputs\":[{\"name\":\"plan_path\",\"value\":\"/workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/make-plan/normalize_whitespace_plan_2026-10-11_200500.md\"},{\"name\":\"issue_url\",\"value\":\"https://github.com/TalonT-Org/autoskillit-e2e-sandbox/issues/1\"},{\"name\":\"review_path\",\"value\":\"\"},{\"name\":\"audit_cycle_path\",\"va...[fixture-truncated 2001 chars]"} +{"type":"queue-operation","operation":"dequeue","timestamp":"2026-10-11T20:12:00.783Z","sessionId":"54f18144-1188-45d7-a392-17b4529f721e"} +{"parentUuid":null,"isSidechain":false,"promptId":"6ec22124-b069-42f2-a3d7-361204649ceb","type":"user","message":{"role":"user","content":"Use the /dry-walkthrough skill AUTOSKILLIT_BOUND_INVOCATION_V1\n{\"skill_inputs\":[{\"name\":\"plan_path\",\"value\":\"/workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/make-plan/normalize_whitespace_plan_2026-10-11_200500.md\"},{\"name\":\"issue_url\",\"value\":\"https://github.com/TalonT-Org/autoskillit-e2e-sandbox/issues/1\"},{\"name\":\"review_path\",\"value\":\"\"},{\"name\":\"audit_cycle_path\",\"va...[fixture-truncated 2001 chars]"},"uuid":"1fa895db-0f63-4e4d-9319-5e0a1ea3bd2e","timestamp":"2026-10-11T20:12:00.823Z","permissionMode":"bypassPermissions","promptSource":"sdk","turnOrigin":"sdk","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","sessionId":"54f18144-1188-45d7-a392-17b4529f721e","version":"2.1.280","gitBranch":"e2e-implementation-fixture-normalize-whitespace/1-3"} +{"type":"last-prompt","lastPrompt":"Use the /dry-walkthrough skill AUTOSKILLIT_BOUND_INVOCATION_V1 {\"skill_inputs\":[{\"name\":\"plan_path\",\"value\":\"/workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/make-plan/normali…","leafUuid":"9a7737fa-95fb-4950-9f81-a0f69f85140a","sessionId":"54f18144-1188-45d7-a392-17b4529f721e"} +{"type":"cost-state","sessionId":"54f18144-1188-45d7-a392-17b4529f721e","totalCostUSD":0.4940016000000001,"totalAPIDuration":149446,"totalAPIDurationWithoutRetries":149406,"totalToolDuration":130687,"totalLinesAdded":3,"totalLinesRemoved":1,"totalDuration":169414,"startTime":1791749517277,"modelUsage":{"claude-sonnet-5":{"inputTokens":93799,"outputTokens":19776,"thinkingTokens":13280,"cacheReadInputTokens":543218,"cacheCreationInputTokens":0,"webSearchRequests":0,"costUSD":0.4940016000000001}},"hasUnknownModelCost":false} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e/subagents/agent-a7e613554b849d6b2.jsonl b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e/subagents/agent-a7e613554b849d6b2.jsonl new file mode 100644 index 0000000000..9b09bf65bd --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e/subagents/agent-a7e613554b849d6b2.jsonl @@ -0,0 +1,2 @@ +{"parentUuid":null,"isSidechain":true,"promptId":"6ec22124-b069-42f2-a3d7-361204649ceb","agentId":"a7e613554b849d6b2","type":"user","message":{"role":"user","content":"You are validating a proposed implementation plan by performing a dry walkthrough of each change without implementing. Your role is technical validation, not strategic decision-making. Fix factual inaccuracies (wrong file paths, nonexistent functions, incorrect line numbers). Preserve all goals and scope.\n\n## Plan to Validate\nRead the exact plan file at:\n`/workspace/autoskillit-runs/impl-20261011-...[fixture-truncated 7212 chars]"},"uuid":"bd262550-a49c-499e-8443-0dad17d21c1a","timestamp":"2026-10-11T20:12:31.074Z","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","sessionId":"54f18144-1188-45d7-a392-17b4529f721e","version":"2.1.280","gitBranch":"e2e-implementation-fixture-normalize-whitespace/1-3"} +{"parentUuid":"1bee3141-ddeb-4c5a-8e05-5c397c01fc75","isSidechain":true,"agentId":"a7e613554b849d6b2","message":{"id":"071b212510333ffa9b724744f66211b1","type":"message","role":"assistant","content":[{"type":"text","text":"# Dry Walkthrough Report\n\n## Plan Validation Summary\n\n**File**: `/workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/make-plan/normalize_whitespace_plan_2026-10-11_200500.md`\n**Stamp**: `Dry-walkthrough verified = TRUE` (verified present on line 1)\n\n---\n\n## Step 1: Multi-Part Detection\n- Filename has no `_part_` suffix. PASS\n- Plan contains no PART scope warning block. PASS\n\n...[fixture-truncated 4672 chars]"}],"model":"MiniMax-M3","stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":188,"cache_creation_input_tokens":0,"cache_read_input_tokens":45709,"output_tokens":1882,"output_tokens_details":{"thinking_tokens":532},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionAgent":"general-purpose","attributionSkill":"dry-walkthrough","type":"assistant","uuid":"3819a31b-bbba-46b9-83ca-61def8b3f88f","timestamp":"2026-10-11T20:14:39.838Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","sessionId":"54f18144-1188-45d7-a392-17b4529f721e","version":"2.1.280","gitBranch":"e2e-implementation-fixture-normalize-whitespace/1-3"} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl new file mode 100644 index 0000000000..3f16a49374 --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl @@ -0,0 +1,5 @@ +{"type":"queue-operation","operation":"enqueue","timestamp":"2026-10-11T20:05:26.169Z","sessionId":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","content":"Use the /make-plan skill AUTOSKILLIT_BOUND_INVOCATION_V1\n{\"skill_inputs\":[{\"name\":\"task\",\"value\":\"E2E implementation fixture: normalize whitespace\\n\\n# Reusable implementation E2E fixture\\n\\nAdd `normalize_whitespace(text: str) -> str` to `sandbox/text.py`.\\n\\nThe function must remove leading and trailing whitespace and replace each run of\\nwhitespace (including spaces, tabs, and newlines) with on...[fixture-truncated 2035 chars]"} +{"type":"queue-operation","operation":"dequeue","timestamp":"2026-10-11T20:05:26.169Z","sessionId":"8ef8f993-8caa-47dd-83b0-26f9e5b51139"} +{"parentUuid":null,"isSidechain":false,"promptId":"b304e3c8-f9a9-4e0e-8d13-8e7d6617cc4e","type":"user","message":{"role":"user","content":"Use the /make-plan skill AUTOSKILLIT_BOUND_INVOCATION_V1\n{\"skill_inputs\":[{\"name\":\"task\",\"value\":\"E2E implementation fixture: normalize whitespace\\n\\n# Reusable implementation E2E fixture\\n\\nAdd `normalize_whitespace(text: str) -> str` to `sandbox/text.py`.\\n\\nThe function must remove leading and trailing whitespace and replace each run of\\nwhitespace (including spaces, tabs, and newlines) with on...[fixture-truncated 2035 chars]"},"uuid":"0b38c9ec-2c12-4619-8935-2121f8c249ad","timestamp":"2026-10-11T20:05:26.210Z","permissionMode":"bypassPermissions","promptSource":"sdk","turnOrigin":"sdk","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","sessionId":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","version":"2.1.280","gitBranch":"e2e-implementation-fixture-normalize-whitespace/1-3"} +{"type":"last-prompt","lastPrompt":"Use the /make-plan skill AUTOSKILLIT_BOUND_INVOCATION_V1 {\"skill_inputs\":[{\"name\":\"task\",\"value\":\"E2E implementation fixture: normalize whitespace\\n\\n# Reusable implementation E2E fixture\\n\\nAdd `norm…","leafUuid":"f50148a8-7b86-4926-a4d2-961755148f52","sessionId":"8ef8f993-8caa-47dd-83b0-26f9e5b51139"} +{"type":"cost-state","sessionId":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","totalCostUSD":0.41991880000000004,"totalAPIDuration":123555,"totalAPIDurationWithoutRetries":123501,"totalToolDuration":91414,"totalLinesAdded":199,"totalLinesRemoved":0,"totalDuration":111137,"startTime":1791749122675,"modelUsage":{"claude-sonnet-5":{"inputTokens":79993,"outputTokens":16833,"thinkingTokens":8494,"cacheReadInputTokens":458014,"cacheCreationInputTokens":0,"webSearchRequests":0,"costUSD":0.41991880000000004}},"hasUnknownModelCost":false} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-a516cbae85151dc7c.jsonl b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-a516cbae85151dc7c.jsonl new file mode 100644 index 0000000000..e37cb2a70e --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-a516cbae85151dc7c.jsonl @@ -0,0 +1,2 @@ +{"parentUuid":null,"isSidechain":true,"promptId":"b304e3c8-f9a9-4e0e-8d13-8e7d6617cc4e","agentId":"a516cbae85151dc7c","type":"user","message":{"role":"user","content":"You are the plan-registry-tracer. Perform a registry and artifact trace audit of the following draft implementation plan.\n\nContrastive frame: \"A junior reviewer found this plan's registry coverage complete — what did they miss?\"\n\nDo NOT suggest scope expansion. Identify only gaps in what the plan already claims to do.\n\nCodebase root: /workspace/autoskillit-runs/impl-20261011-200500-587134\n\nWorking...[fixture-truncated 715 chars]"},"uuid":"0a0a8820-6e0f-43bd-9b90-5a9dadd38fb9","timestamp":"2026-10-11T20:05:58.067Z","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","sessionId":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","version":"2.1.280","gitBranch":"e2e-implementation-fixture-normalize-whitespace/1-3"} +{"parentUuid":"8349df76-a174-48ed-87fa-55fcd3bb3647","isSidechain":true,"agentId":"a516cbae85151dc7c","message":{"id":"071b1f367fc2617d89e17e0512b28e11","type":"message","role":"assistant","content":[{"type":"text","text":"# Registry Trace Findings\n\n## Touched Symbols\n- `normalize_whitespace` — new function in `sandbox/text.py` (no rename, no removal)\n\n## Grep/LSP Results\n\n| Symbol | File Found | Reference Type | Updated by Plan? |\n|--------|-----------|----------------|------------------|\n| normalize_whitespace | (none) | N/A — net-new symbol, no prior references anywhere in repo | N/A |\n\nA recursive grep across `/...[fixture-truncated 1847 chars]"}],"model":"MiniMax-M3","stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":1894,"cache_creation_input_tokens":0,"cache_read_input_tokens":9755,"output_tokens":1025,"output_tokens_details":{"thinking_tokens":474},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionAgent":"autoskillit:plan-registry-tracer","attributionSkill":"make-plan","attributionPlugin":"autoskillit","type":"assistant","uuid":"27b5b29b-5719-4134-b996-7009d7bee590","timestamp":"2026-10-11T20:06:23.283Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","sessionId":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","version":"2.1.280","gitBranch":"e2e-implementation-fixture-normalize-whitespace/1-3"} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-a9ed51b7c02b828dd.jsonl b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-a9ed51b7c02b828dd.jsonl new file mode 100644 index 0000000000..b2c3fede89 --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-a9ed51b7c02b828dd.jsonl @@ -0,0 +1,2 @@ +{"parentUuid":null,"isSidechain":true,"promptId":"b304e3c8-f9a9-4e0e-8d13-8e7d6617cc4e","agentId":"a9ed51b7c02b828dd","type":"user","message":{"role":"user","content":"You are the plan-interface-mapper. Perform a variable and data-flow trace audit of the following draft implementation plan.\n\nContrastive frame: \"A junior reviewer found this plan's variable usage correct — what did they miss?\"\n\nDo NOT suggest scope expansion. Identify only gaps in what the plan already claims to do.\n\nCodebase root: /workspace/autoskillit-runs/impl-20261011-200500-587134\n\nWorking b...[fixture-truncated 780 chars]"},"uuid":"5067983e-c8ee-44df-8677-6acfb5f2d6d9","timestamp":"2026-10-11T20:05:57.864Z","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","sessionId":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","version":"2.1.280","gitBranch":"e2e-implementation-fixture-normalize-whitespace/1-3"} +{"parentUuid":"9ab1fecd-1984-470f-a6a6-e6a60a361312","isSidechain":true,"agentId":"a9ed51b7c02b828dd","message":{"id":"071b1f337786beccec1da5b44669d627","type":"message","role":"assistant","content":[{"type":"text","text":"# Interface Mapping Audit\n\n## SET/READ Trace Table\n\n| Variable | SET Location | SET Value | READ Location | READ Expectation | Match? |\n|----------|-------------|-----------|---------------|------------------|--------|\n| `tests/test_text.py` content | plan step 2 (create) | new file with `normalize_whitespace` tests only | `Taskfile.yml:11-12` (`unittest discover`) | discover runs over existing `t...[fixture-truncated 2117 chars]"}],"model":"MiniMax-M3","stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":918,"cache_creation_input_tokens":0,"cache_read_input_tokens":6939,"output_tokens":1767,"output_tokens_details":{"thinking_tokens":1127},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionAgent":"autoskillit:plan-interface-mapper","attributionSkill":"make-plan","attributionPlugin":"autoskillit","type":"assistant","uuid":"53c38458-ca9f-45dd-a78c-38dcfd68125f","timestamp":"2026-10-11T20:06:23.901Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","sessionId":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","version":"2.1.280","gitBranch":"e2e-implementation-fixture-normalize-whitespace/1-3"} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-aeb96cc0ddbdab294.jsonl b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-aeb96cc0ddbdab294.jsonl new file mode 100644 index 0000000000..b98512bab6 --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-aeb96cc0ddbdab294.jsonl @@ -0,0 +1,2 @@ +{"parentUuid":null,"isSidechain":true,"promptId":"b304e3c8-f9a9-4e0e-8d13-8e7d6617cc4e","agentId":"aeb96cc0ddbdab294","type":"user","message":{"role":"user","content":"You are the plan-foundation-auditor. Perform a step-by-step control-flow audit of the following draft implementation plan.\n\nContrastive frame: \"A junior reviewer found this plan's control flow acceptable — what did they miss?\"\n\nDo NOT suggest scope expansion. Identify only gaps in what the plan already claims to do.\n\nCodebase root: /workspace/autoskillit-runs/impl-20261011-200500-587134\n\nWorking b...[fixture-truncated 1144 chars]"},"uuid":"1178680b-6e3b-4d4b-9ee4-894c30881464","timestamp":"2026-10-11T20:05:57.914Z","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","sessionId":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","version":"2.1.280","gitBranch":"e2e-implementation-fixture-normalize-whitespace/1-3"} +{"parentUuid":"b28e2ef4-916c-41b8-a55f-ec64f01eaece","isSidechain":true,"agentId":"aeb96cc0ddbdab294","message":{"id":"071b1f3371a476db86a8860eb750a903","type":"message","role":"assistant","content":[{"type":"text","text":"# Plan Audit — Findings\n\n## Step 1–2 — Enumerate & Draw Control Flow\n\n**Target function:** `normalize_whitespace(text: str) -> str` (new, in `sandbox/text.py`).\n\nPlanned body is a two-call pipeline (`text.strip()` then `re.sub(r'\\s+', ' ', ...)`). No `if`/`match`/`return` branches inside the function — every input traverses the same path. The audit therefore pivots to the **test file and the plan'...[fixture-truncated 2234 chars]"}],"model":"MiniMax-M3","stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":594,"cache_creation_input_tokens":0,"cache_read_input_tokens":7345,"output_tokens":3617,"output_tokens_details":{"thinking_tokens":2955},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionAgent":"autoskillit:plan-foundation-auditor","attributionSkill":"make-plan","attributionPlugin":"autoskillit","type":"assistant","uuid":"f522d9e2-3194-4afc-9b88-8a072e3ff485","timestamp":"2026-10-11T20:06:30.307Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","sessionId":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","version":"2.1.280","gitBranch":"e2e-implementation-fixture-normalize-whitespace/1-3"} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-sandbox/2292658a-34cf-44a0-a0b1-f7a20218874b.jsonl b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-sandbox/2292658a-34cf-44a0-a0b1-f7a20218874b.jsonl new file mode 100644 index 0000000000..d92ba7be4c --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-sandbox/2292658a-34cf-44a0-a0b1-f7a20218874b.jsonl @@ -0,0 +1,23 @@ +{"type":"queue-operation","operation":"enqueue","timestamp":"2026-10-11T20:04:40.218Z","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","content":"You are an L2 food truck orchestrator. Execute the recipe 'implementation' autonomously.\nTimeout: 7200s. Campaign: 48915f19-d220-4069-8d55-b81efff84c05. Dispatch: 4666c3fd-486f-4ce1-b51f-9d9d660a5560.\n\n--- SECTION 1: ADMIRAL DISCIPLINE (L2/FOOD-TRUCK SUBSET) ---\n\n## CONTEXT LIMIT ROUTING — MANDATORY\n\nWhen `run_skill` returns `needs_retry=true` for **any step**:\n\n- **If `retry_reason: resume` AND `...[fixture-truncated 33134 chars]"} +{"type":"queue-operation","operation":"dequeue","timestamp":"2026-10-11T20:04:40.218Z","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b"} +{"parentUuid":null,"isSidechain":false,"promptId":"886212ea-29c8-475f-a4bb-05be860f18dd","type":"user","message":{"role":"user","content":"You are an L2 food truck orchestrator. Execute the recipe 'implementation' autonomously.\nTimeout: 7200s. Campaign: 48915f19-d220-4069-8d55-b81efff84c05. Dispatch: 4666c3fd-486f-4ce1-b51f-9d9d660a5560.\n\n--- SECTION 1: ADMIRAL DISCIPLINE (L2/FOOD-TRUCK SUBSET) ---\n\n## CONTEXT LIMIT ROUTING — MANDATORY\n\nWhen `run_skill` returns `needs_retry=true` for **any step**:\n\n- **If `retry_reason: resume` AND `...[fixture-truncated 33134 chars]"},"uuid":"e1be918a-cba5-4f0b-9960-60a08420f2b2","timestamp":"2026-10-11T20:04:40.246Z","permissionMode":"bypassPermissions","promptSource":"sdk","turnOrigin":"sdk","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"9aa8bfde-3315-49d7-b568-c907349a3dbd","isSidechain":false,"message":{"id":"071b1eea9dc5d3c03c383ed25b5ec610","type":"message","role":"assistant","content":[{"type":"tool_use","id":"call_01a12c9191907360b2a092bf","name":"mcp__plugin_autoskillit_autoskillit__bootstrap_clone","input":{"source_dir":"/workspace/sandbox","run_name":"impl","base_branch":"main","step_name":"clone"}}],"model":"MiniMax-M3","stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":1301,"cache_creation_input_tokens":0,"cache_read_input_tokens":42278,"output_tokens":222,"output_tokens_details":{"thinking_tokens":143},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionMcpServer":"plugin:autoskillit:autoskillit","attributionMcpTool":"get_recipe_section","type":"assistant","uuid":"66199695-80b0-448d-9390-b9bcb3712e6e","timestamp":"2026-10-11T20:04:59.937Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"66199695-80b0-448d-9390-b9bcb3712e6e","isSidechain":false,"promptId":"886212ea-29c8-475f-a4bb-05be860f18dd","type":"user","message":{"role":"user","content":[{"tool_use_id":"call_01a12c9191907360b2a092bf","type":"tool_result","content":"## bootstrap_clone\n\nwork_dir: /workspace/autoskillit-runs/impl-20261011-200500-587134\nremote_url: https://github.com/TalonT-Org/autoskillit-e2e-sandbox.git\nbase_sha: d6c143d2494488deeb6680d7d5cf2e55e3e01732\nmerge_target: main\ntimings:\n clone_ms: 851\n rev_parse_ms: 3\n\n--- RECIPE SEGMENT ---\n{\"bodies\":[{\"body\":\"{\\\"claim_and_resolve\\\":{\\\"tool\\\":\\\"claim_and_resolve_issue\\\",\\\"with\\\":{\\\"step_name\\\":\\\"...[fixture-truncated 8153 chars]"}]},"uuid":"6cdc0d3f-fc36-426f-97c3-206e93e21f49","timestamp":"2026-10-11T20:05:01.170Z","toolUseResult":"## bootstrap_clone\n\nwork_dir: /workspace/autoskillit-runs/impl-20261011-200500-587134\nremote_url: https://github.com/TalonT-Org/autoskillit-e2e-sandbox.git\nbase_sha: d6c143d2494488deeb6680d7d5cf2e55e3e01732\nmerge_target: main\ntimings:\n clone_ms: 851\n rev_parse_ms: 3\n\n--- RECIPE SEGMENT ---\n{\"bodies\":[{\"body\":\"{\\\"claim_and_resolve\\\":{\\\"tool\\\":\\\"claim_and_resolve_issue\\\",\\\"with\\\":{\\\"step_name\\\":\\\"...[fixture-truncated 8153 chars]","mcpMeta":{"_meta":{"fastmcp":{"wrap_result":true}},"structuredContent":{"result":"{\"work_dir\":\"/workspace/autoskillit-runs/impl-20261011-200500-587134\",\"remote_url\":\"https://github.com/TalonT-Org/autoskillit-e2e-sandbox.git\",\"base_sha\":\"d6c143d2494488deeb6680d7d5cf2e55e3e01732\",\"merge_target\":\"main\",\"timings\":{\"clone_ms\":851,\"rev_parse_ms\":3},\"recipe_segment\":{\"kind\":\"success\",\"source_step\":\"clone\",\"segments\":[{\"index\":1,\"name\":\"I1\"}],\"bodies\":[{\"step\":\"claim_and_resolve\",\"body\":\"{\\\"claim_and_resolve\\\":{\\\"tool\\\":\\\"claim_and_resolve_issue\\\",\\\"with\\\":{\\\"step_name\\\":\\\"claim_and_resolve\\\",\\\"issue_url\\\":\\\"${{ inputs.issue_url }}\\\",\\\"allow_reentry\\\":false},\\\"capture\\\":{\\\"issue_number\\\":\\\"${{ result.issue_number }}\\\",\\\"issue_title\\\":\\\"${{ result.issue_title }}\\\",\\\"issue_slug\\\":\\\"${{ result.issue_slug }}\\\",\\\"claimed\\\":\\\"${{ result.claimed }}\\\"},\\\"on_result\\\":[{\\\"when\\\":\\\"${{ result.claimed }} == true\\\",\\\"route\\\":\\\"create_and_publish\\\"},{\\\"route\\\":\\\"register_clone_failure\\\"}],\\\"on_failure\\\":\\\"register_clone_failure\\\",\\\"note\\\":\\\"Fetches issue title/slug and claims the issue in one turn. Returns issue_number, issue_title, issue_slug, and claimed boolean. When claimed=false (another session owns it), routes to register_clone_failure before escalate_stop so the clone directory is registered for cleanup. issue_slug and issue_number are passed to create_and_publish for branch naming.\\\\n\\\"}}\",\"body_sha256\":\"sha256:2dff84b001815660dc762ea29e4eeeb9eea19b11c77638b46aec51039509a257\"},{\"step\":\"create_and_publish\",\"body\":\"{\\\"create_and_publish\\\":{\\\"tool\\\":\\\"create_and_publish_branch\\\",\\\"with\\\":{\\\"issue_slug\\\":\\\"${{ context.issue_slug }}\\\",\\\"run_name\\\":\\\"${{ inputs.run_name }}\\\",\\\"issue_number\\\":\\\"${{ context.issue_number }}\\\",\\\"work_dir\\\":\\\"${{ context.work_dir }}\\\",\\\"remote_url\\\":\\\"${{ context.remote_url }}\\\",\\\"step_name\\\":\\\"create_and_publish\\\"},\\\"capture\\\":{\\\"merge_target\\\":\\\"${{ result.merge_target }}\\\"},\\\"on_success\\\":\\\"plan\\\",\\\"on_failure\\\":\\\"release_issue_failure\\\",\\\"note\\\":\\\"Computes branch name, creates it (with collision suffix if needed), and pushes to origin in one turn. Overrides merge_target with the feature branch name. merge_worktree requires refs/remotes/origin/{branch} to exist after fetch (step 5.5). When open_pr=false, this step is skipped and merge_target remains as base_branch.\\\\n\\\"}}\",\"body_sha256\":\"sha256:f102a40c0ec99f47038934ff163d56cee9d5df486246d7575221ce2c6a89fcf1\"},{\"step\":\"plan\",\"body\":\"{\\\"plan\\\":{\\\"tool\\\":\\\"run_skill\\\",\\\"model\\\":\\\"\\\",\\\"with\\\":{\\\"skill_command\\\":\\\"/autoskillit:make-plan\\\",\\\"skill_inputs\\\":{\\\"adversarial_review_level\\\":\\\"auto\\\",\\\"task\\\":\\\"${{ inputs.task }}\\\",\\\"issue_url\\\":\\\"${{ inputs.issue_url }}\\\",\\\"audit_cycle_path\\\":\\\"${{ context.audit_cycle_path }}\\\",\\\"plan_set_coverage_gaps\\\":\\\"${{ context.plan_set_coverage_gaps }}\\\"},\\\"cwd\\\":\\\"${{ context.work_dir }}\\\",\\\"step_name\\\":\\\"plan\\\"},\\\"capture\\\":{\\\"plan_path\\\":\\\"${{ result.plan_path }}\\\",\\\"all_plan_paths\\\":\\\"${{ result.plan_path }}\\\",\\\"verdict\\\":\\\"${{ result.verdict }}\\\",\\\"plan_disposition_path\\\":\\\"${{ result.plan_disposition_path }}\\\"},\\\"capture_list\\\":{\\\"plan_parts\\\":\\\"${{ result.plan_parts }}\\\"},\\\"retries\\\":0,\\\"on_result\\\":[{\\\"when\\\":\\\"${{ result.verdict }} == plan\\\",\\\"route\\\":\\\"bind_plan_set\\\"}],\\\"on_failure\\\":\\\"release_issue_failure\\\",\\\"on_rate_limit\\\":\\\"release_issue_failure\\\",\\\"on_context_limit\\\":\\\"salvage_plan\\\",\\\"optional_context_refs\\\":[\\\"audit_cycle_path\\\",\\\"plan_set_coverage_gaps\\\"],\\\"note\\\":\\\"Produces an ordinary plan when no audit_cycle_path is present. On NO GO re-entry, consumes that exact current authority and emits both plan_path and plan_disposition_path. all_plan_paths appends result.plan_path across re-entries; verdict=plan is the only successful result.\\\"}}\",\"body_sha256\":\"sha256:c44e11b44d35bcdf9620f407c17b7936a9b47a1f348e3f377cfd7b3ae9c4765a\",\"invocation_template_digest\":\"sha256:deec5b08a76bdabe0f7f520b9bb66f78789645f615012ac8eaf24806bef8e05e\"},{\"step\":\"salvage_plan\",\"body\":\"{\\\"salvage_plan\\\":{\\\"tool\\\":\\\"run_python\\\",\\\"with\\\":{\\\"step_name\\\":\\\"salvage_plan\\\",\\\"callable\\\":\\\"autoskillit.recipe._cmd_rpc.verify_plan_artifacts\\\",\\\"plan_parts\\\":\\\"${{ context.plan_parts }}\\\",\\\"audit_cycle_path\\\":\\\"${{ context.audit_cycle_path }}\\\"},\\\"capture\\\":{\\\"plan_path\\\":\\\"${{ result.plan_path }}\\\",\\\"plan_disposition_path\\\":\\\"${{ result.plan_disposition_path }}\\\",\\\"all_plan_paths\\\":\\\"${{ result.plan_path }}\\\"},\\\"capture_list\\\":{\\\"plan_parts\\\":\\\"${{ result.plan_parts }}\\\"},\\\"optional_context_refs\\\":[\\\"audit_cycle_path\\\"],\\\"retries\\\":0,\\\"on_result\\\":[{\\\"when\\\":\\\"${{ result.verdict }} == salvaged\\\",\\\"route\\\":\\\"bind_plan_set\\\"},{\\\"route\\\":\\\"release_issue_failure\\\"}],\\\"on_failure\\\":\\\"release_issue_failure\\\",\\\"note\\\":\\\"Deterministic salvage for plan context-limit stumbles: verifies captured plan_parts artifacts exist on disk. salvaged routes onward as if plan succeeded; unsalvageable falls through to plan's original on_failure destination. Substitute the captured ${{ context.plan_parts }} value into the plan_parts argument verbatim, unmodified — do not reformat, quote, or re-derive it; the callable's input gramm...[fixture-truncated 30 chars]\\\"}}\",\"body_sha256\":\"sha256:08f1e03e1927e09d6d304a2925760a2215f0f78b768fb856ab0261f695558d1f\"},{\"step\":\"bind_plan_set\",\"body\":\"{\\\"bind_plan_set\\\":{\\\"tool\\\":\\\"bind_plan_set\\\",\\\"with\\\":{\\\"plan_parts\\\":\\\"${{ context.plan_parts }}\\\",\\\"cwd\\\":\\\"${{ context.work_dir }}\\\",\\\"issue_url\\\":\\\"${{ inputs.issue_url }}\\\",\\\"seal\\\":\\\"true\\\",\\\"step_name\\\":\\\"bind_plan_set\\\"},\\\"capture\\\":{\\\"plan_set_authority_path\\\":\\\"${{ result.plan_set_authority_path }}\\\",\\\"plan_set_authority_digest\\\":\\\"${{ result.plan_set_authority_digest }}\\\",\\\"plan_set_parts\\\":\\\"${{ result.plan_set_parts }}\\\",\\\"plan_set_coverage_gaps\\\":\\\"${{ result.coverage_gaps }}\\\"},\\\"on_result\\\":[{\\\"when\\\":\\\"${{ result.success }} == true\\\",\\\"route\\\":\\\"verify\\\"},{\\\"when\\\":\\\"${{ result.reason }} == coverage_failed\\\",\\\"route\\\":\\\"check_replan_iteration\\\"},{\\\"route\\\":\\\"release_issue_failure\\\"}],\\\"on_failure\\\":\\\"release_issue_failure\\\",\\\"note\\\":\\\"Bind exactly the plan_parts value captured from the producer that just ran. A successful replan replaces the earlier list with its returned list: do not append rejected paths or rebuild the list from a directory listing. When salvage_plan is the producer, bind its returned list unchanged under the existing salvage recovery and strict coverage checks. Aggregate plan-set gate before every walkthroug...[fixture-truncated 277 chars]\\\"}}\",\"body_sha256\":\"sha256:ac7464747141373f7d61c06815e6021b52cda0a41170cde8e83296541e2229e6\"},{\"step\":\"check_replan_iteration\",\"body\":\"{\\\"check_replan_iteration\\\":{\\\"tool\\\":\\\"run_python\\\",\\\"with\\\":{\\\"step_name\\\":\\\"check_replan_iteration\\\",\\\"callable\\\":\\\"autoskillit.smoke_utils.check_loop_iteration\\\",\\\"current_iteration\\\":\\\"${{ context.plan_set_replan_count }}\\\",\\\"max_iterations\\\":\\\"2\\\"},\\\"capture\\\":{\\\"plan_set_replan_count\\\":\\\"${{ result.next_iteration }}\\\"},\\\"optional_context_refs\\\":[\\\"plan_set_replan_count\\\"],\\\"on_result\\\":[{\\\"when\\\":\\\"${{ result.max_exceeded }} == true\\\",\\\"route\\\":\\\"release_issue_failure\\\"},{\\\"route\\\":\\\"plan\\\"}],\\\"on_failure\\\":\\\"release_issue_failure\\\"}}\",\"body_sha256\":\"sha256:6ef878d992680fbac602153f1c0ac829f8a5c0657be7718c4426c2d25812f9c3\"}],\"recipe_execution\":{\"execution_id\":\"1dda3d6d29e545cbae216029ec657356\",\"invocation_template_digests\":{\"plan\":\"sha256:deec5b08a76bdabe0f7f520b9bb66f78789645f615012ac8eaf24806bef8e05e\"},\"skill_input_shapes\":{\"plan\":{\"keys\":[\"task\",\"issue_url\",\"adversarial_review_level\",\"audit_cycle_path\",\"plan_set_coverage_gaps\"],\"absence_values\":{\"audit_cycle_path\":\"\",\"plan_set_coverage_gaps\":\"\"}}},\"snapshot_digest\":\"sha256:1269b37298577647aec24baeb183326c56ad0d0bf2fecbd0231b0ad6e5b81fba\"},\"pull_closures\":[],\"recipe_pull\":{\"producer_tool\":\"open_kitchen\",\"recipe_name\":\"implementation\",\"descriptor_version\":2,\"schema_version\":2,\"payload_sha256\":\"sha256:ea2e62f28435745f27a7a742cbb98d2e34cdb00d82eef53117118c3b574d9497\",\"artifact_blob_sha256\":\"sha256:cd5e7fff1ca8145532aa9ce8ca2a26ddbfe48fb704316310789b8123319f8d63\",\"artifact_blob_size_bytes\":576427,\"body_sha256\":\"sha256:3f14e54f1505f8a5c4d13c2bc5658394855e4c7dc624cc527ddb4bf0711954d5\",\"body_size_bytes\":115869,\"flow_schema_version\":2,\"flow_sha256\":\"sha256:3b9ff899035aad7555d5863de6c70c6ed95e6ffac5f19cea0c46cc17261f4f2a\",\"flow_size_bytes\":101558,\"flow_record_count\":694,\"pull_tool\":\"get_recipe_section\"}}}"}},"sourceToolAssistantUUID":"66199695-80b0-448d-9390-b9bcb3712e6e","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"0b144546-f16f-4dd1-94b8-87b4f092a3e6","isSidechain":false,"message":{"id":"071b1eedfe3e75016da0bd16d2a6ae53","type":"message","role":"assistant","content":[{"type":"tool_use","id":"call_01a12c919df971e2b8b09c37","name":"mcp__plugin_autoskillit_autoskillit__claim_and_resolve_issue","input":{"issue_url":"https://github.com/TalonT-Org/autoskillit-e2e-sandbox/issues/1","allow_reentry":false,"step_name":"claim_and_resolve"}}],"model":"MiniMax-M3","stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":2643,"cache_creation_input_tokens":0,"cache_read_input_tokens":43579,"output_tokens":218,"output_tokens_details":{"thinking_tokens":124},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionMcpServer":"plugin:autoskillit:autoskillit","attributionMcpTool":"bootstrap_clone","type":"assistant","uuid":"e8638011-011a-4b7d-8a7f-a09899915dfb","timestamp":"2026-10-11T20:05:03.120Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"e8638011-011a-4b7d-8a7f-a09899915dfb","isSidechain":false,"promptId":"886212ea-29c8-475f-a4bb-05be860f18dd","type":"user","message":{"role":"user","content":[{"tool_use_id":"call_01a12c919df971e2b8b09c37","type":"tool_result","content":"## claim_and_resolve_issue\n\nsuccess: True\nclaimed: True\nlabel: in-progress\nreview_approach_recommended: False\ninvestigation_complete: False\nissue_number: 1\nissue_title: E2E implementation fixture: normalize whitespace\nissue_slug: e2e-implementation-fixture-normalize-whitespace\ntimings:\n fetch_title_ms: 359\n claim_ms: 2475\n\n--- RECIPE SEGMENT ---\n{\"bodies\":[],\"kind\":\"success\",\"pull_closures\":[],\"...[fixture-truncated 943 chars]"}]},"uuid":"b689f776-8c6f-4831-bd03-e33b6ded7492","timestamp":"2026-10-11T20:05:06.239Z","toolUseResult":"## claim_and_resolve_issue\n\nsuccess: True\nclaimed: True\nlabel: in-progress\nreview_approach_recommended: False\ninvestigation_complete: False\nissue_number: 1\nissue_title: E2E implementation fixture: normalize whitespace\nissue_slug: e2e-implementation-fixture-normalize-whitespace\ntimings:\n fetch_title_ms: 359\n claim_ms: 2475\n\n--- RECIPE SEGMENT ---\n{\"bodies\":[],\"kind\":\"success\",\"pull_closures\":[],\"...[fixture-truncated 943 chars]","mcpMeta":{"_meta":{"fastmcp":{"wrap_result":true}},"structuredContent":{"result":"{\"success\":true,\"claimed\":true,\"label\":\"in-progress\",\"review_approach_recommended\":false,\"investigation_complete\":false,\"issue_number\":1,\"issue_title\":\"E2E implementation fixture: normalize whitespace\",\"issue_slug\":\"e2e-implementation-fixture-normalize-whitespace\",\"timings\":{\"fetch_title_ms\":359,\"claim_ms\":2475},\"recipe_segment\":{\"kind\":\"success\",\"source_step\":\"claim_and_resolve\",\"segments\":[],\"bodies\":[],\"recipe_execution\":{\"execution_id\":\"1dda3d6d29e545cbae216029ec657356\",\"invocation_template_digests\":{},\"skill_input_shapes\":{},\"snapshot_digest\":\"sha256:1269b37298577647aec24baeb183326c56ad0d0bf2fecbd0231b0ad6e5b81fba\"},\"pull_closures\":[],\"recipe_pull\":{\"producer_tool\":\"open_kitchen\",\"recipe_name\":\"implementation\",\"descriptor_version\":2,\"schema_version\":2,\"payload_sha256\":\"sha256:ea2e62f28435745f27a7a742cbb98d2e34cdb00d82eef53117118c3b574d9497\",\"artifact_blob_sha256\":\"sha256:cd5e7fff1ca8145532aa9ce8ca2a26ddbfe48fb704316310789b8123319f8d63\",\"artifact_blob_size_bytes\":576427,\"body_sha256\":\"sha256:3f14e54f1505f8a5c4d13c2bc5658394855e4c7dc624cc527ddb4bf0711954d5\",\"body_size_bytes\":115869,\"flow_schema_version\":2,\"flow_sha256\":\"sha256:3b9ff899035aad7555d5863de6c70c6ed95e6ffac5f19cea0c46cc17261f4f2a\",\"flow_size_bytes\":101558,\"flow_record_count\":694,\"pull_tool\":\"get_recipe_section\"}}}"}},"sourceToolAssistantUUID":"e8638011-011a-4b7d-8a7f-a09899915dfb","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"09cc0463-fc85-4985-ae35-09732330b279","isSidechain":false,"message":{"id":"071b1ef27e5cad826dc3ce73f99787ac","type":"message","role":"assistant","content":[{"type":"tool_use","id":"call_01a12c91b227749fbd03b2b0","name":"mcp__plugin_autoskillit_autoskillit__create_and_publish_branch","input":{"issue_slug":"e2e-implementation-fixture-normalize-whitespace","run_name":"impl","issue_number":"1","work_dir":"/workspace/autoskillit-runs/impl-20261011-200500-587134","remote_url":"https://github.com/TalonT-Org/autoskillit-e2e-sandbox.git","step_name":"create_and_publish"}}],"model":"MiniMax-M3","stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":737,"cache_creation_input_tokens":0,"cache_read_input_tokens":46222,"output_tokens":182,"output_tokens_details":{"thinking_tokens":32},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionMcpServer":"plugin:autoskillit:autoskillit","attributionMcpTool":"claim_and_resolve_issue","type":"assistant","uuid":"fd55e15c-8988-49ee-8a19-20eff2eb5919","timestamp":"2026-10-11T20:05:08.278Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"fd55e15c-8988-49ee-8a19-20eff2eb5919","isSidechain":false,"promptId":"886212ea-29c8-475f-a4bb-05be860f18dd","type":"user","message":{"role":"user","content":[{"tool_use_id":"call_01a12c91b227749fbd03b2b0","type":"tool_result","content":"## create_and_publish_branch\n\nmerge_target: e2e-implementation-fixture-normalize-whitespace/1-3\nwas_unique: False\ntimings:\n compute_ms: 0\n branch_create_ms: 1131\n push_ms: 1068"}]},"uuid":"d5b2d789-548b-4f26-be64-599540dab17e","timestamp":"2026-10-11T20:05:10.770Z","toolUseResult":"## create_and_publish_branch\n\nmerge_target: e2e-implementation-fixture-normalize-whitespace/1-3\nwas_unique: False\ntimings:\n compute_ms: 0\n branch_create_ms: 1131\n push_ms: 1068","mcpMeta":{"_meta":{"fastmcp":{"wrap_result":true}},"structuredContent":{"result":"{\"merge_target\":\"e2e-implementation-fixture-normalize-whitespace/1-3\",\"was_unique\":false,\"timings\":{\"compute_ms\":0,\"branch_create_ms\":1131,\"push_ms\":1068}}"}},"sourceToolAssistantUUID":"fd55e15c-8988-49ee-8a19-20eff2eb5919","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"914bc97f-052b-46b7-ad36-0fa7d53b614f","isSidechain":false,"message":{"id":"071b1efbdcae033e3bee1954f2c790c3","type":"message","role":"assistant","content":[{"type":"tool_use","id":"call_01a12c91d9517369b630b48c","name":"mcp__plugin_autoskillit_autoskillit__run_skill","input":{"skill_command":"/autoskillit:make-plan","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","step_name":"plan","skill_inputs":{"adversarial_review_level":"auto","task":"E2E implementation fixture: normalize whitespace\n\n# Reusable implementation E2E fixture\n\nAdd `normalize_whitespace(text: str) -> str` to `sandbox/text.py`.\n\nThe function must remove leading and trailing whitespace and replace each run of\nwhitespace (including spaces, tabs, and newlines) with one ASCII space. Preserve\nthe case and punctuation of non-whitespace characters. Empty and whitespace-only\n...[fixture-truncated 737 chars]","issue_url":"https://github.com/TalonT-Org/autoskillit-e2e-sandbox/issues/1","audit_cycle_path":"","plan_set_coverage_gaps":""}}}],"model":"MiniMax-M3","stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":209,"cache_creation_input_tokens":0,"cache_read_input_tokens":47866,"output_tokens":435,"output_tokens_details":{"thinking_tokens":9},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionMcpServer":"plugin:autoskillit:autoskillit","attributionMcpTool":"record_pipeline_step","type":"assistant","uuid":"25cae19a-d6ea-4f11-a8b3-c8937bd543b9","timestamp":"2026-10-11T20:05:18.314Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"25cae19a-d6ea-4f11-a8b3-c8937bd543b9","isSidechain":false,"promptId":"886212ea-29c8-475f-a4bb-05be860f18dd","type":"user","message":{"role":"user","content":[{"tool_use_id":"call_01a12c91d9517369b630b48c","type":"tool_result","content":"run_skill: FAIL [FAIL]\nexit_code: \nneeds_retry: False\nerror: RECIPE EXECUTION REJECTED [recipe_execution_attestation_missing]: an active recipe requires recipe_execution_id and invocation_template_digest; take both from the recipe_execution block of the complete_recipe_initialization receipt (bounded delivery) or of the open_kitchen response (inline delivery), using invocation_template_digests[ste...[fixture-truncated 274 chars]"}]},"uuid":"3936d669-1f07-41d4-bc84-3770955f1e04","timestamp":"2026-10-11T20:05:18.870Z","toolUseResult":"run_skill: FAIL [FAIL]\nexit_code: \nneeds_retry: False\nerror: RECIPE EXECUTION REJECTED [recipe_execution_attestation_missing]: an active recipe requires recipe_execution_id and invocation_template_digest; take both from the recipe_execution block of the complete_recipe_initialization receipt (bounded delivery) or of the open_kitchen response (inline delivery), using invocation_template_digests[ste...[fixture-truncated 274 chars]","mcpMeta":{"_meta":{"fastmcp":{"wrap_result":true}},"structuredContent":{"result":"{\"success\":false,\"is_error\":true,\"error\":\"RECIPE EXECUTION REJECTED [recipe_execution_attestation_missing]: an active recipe requires recipe_execution_id and invocation_template_digest; take both from the recipe_execution block of the complete_recipe_initialization receipt (bounded delivery) or of the open_kitchen response (inline delivery), using invocation_template_digests[step_name] for this step; structured calls must initialize skill...[fixture-truncated 213 chars]\",\"stage\":\"preflight:recipe_execution\",\"retriable\":false}"}},"sourceToolAssistantUUID":"25cae19a-d6ea-4f11-a8b3-c8937bd543b9","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"02c03783-dcd5-4563-8311-e06aa7b8c603","isSidechain":false,"message":{"id":"071b1efe8d9be3c62f44b3fd56e52baa","type":"message","role":"assistant","content":[{"type":"tool_use","id":"call_01a12c91e6c07518b24e8d31","name":"mcp__plugin_autoskillit_autoskillit__run_skill","input":{"skill_command":"/autoskillit:make-plan","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","step_name":"plan","recipe_execution_id":"1dda3d6d29e545cbae216029ec657356","invocation_template_digest":"sha256:deec5b08a76bdabe0f7f520b9bb66f78789645f615012ac8eaf24806bef8e05e","skill_inputs":{"adversarial_review_level":"auto","task":"E2E implementation fixture: normalize whitespace\n\n# Reusable implementation E2E fixture\n\nAdd `normalize_whitespace(text: str) -> str` to `sandbox/text.py`.\n\nThe function must remove leading and trailing whitespace and replace each run of\nwhitespace (including spaces, tabs, and newlines) with one ASCII space. Preserve\nthe case and punctuation of non-whitespace characters. Empty and whitespace-only\n...[fixture-truncated 737 chars]","issue_url":"https://github.com/TalonT-Org/autoskillit-e2e-sandbox/issues/1","audit_cycle_path":"","plan_set_coverage_gaps":""}}}],"model":"MiniMax-M3","stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":630,"cache_creation_input_tokens":0,"cache_read_input_tokens":48075,"output_tokens":607,"output_tokens_details":{"thinking_tokens":94},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionMcpServer":"plugin:autoskillit:autoskillit","attributionMcpTool":"run_skill","type":"assistant","uuid":"ef71da75-f7f0-44ec-a940-f1b9c8affbba","timestamp":"2026-10-11T20:05:21.760Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"ef71da75-f7f0-44ec-a940-f1b9c8affbba","isSidechain":false,"promptId":"886212ea-29c8-475f-a4bb-05be860f18dd","type":"user","message":{"role":"user","content":[{"tool_use_id":"call_01a12c91e6c07518b24e8d31","type":"tool_result","content":"run_skill: OK [success]\nsession_id: 8ef8f993-8caa-47dd-83b0-26f9e5b51139\nreceipt_id: 0739af7533c34089af07c982a53a62c1\nexit_code: 0 (infra-terminated after completion — grace exceeded)\nneeds_retry: False\noutcome_fields: {'plan_parts': '/workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/make-plan/normalize_whitespace_plan_2026-10-11_200500.md', 'plan_path': '/workspace/autoski...[fixture-truncated 444 chars]"}]},"uuid":"bd81bead-a8ba-4a75-8388-a924aadc126d","timestamp":"2026-10-11T20:07:17.201Z","toolUseResult":"run_skill: OK [success]\nsession_id: 8ef8f993-8caa-47dd-83b0-26f9e5b51139\nreceipt_id: 0739af7533c34089af07c982a53a62c1\nexit_code: 0 (infra-terminated after completion — grace exceeded)\nneeds_retry: False\noutcome_fields: {'plan_parts': '/workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/make-plan/normalize_whitespace_plan_2026-10-11_200500.md', 'plan_path': '/workspace/autoski...[fixture-truncated 444 chars]","mcpMeta":{"_meta":{"fastmcp":{"wrap_result":true}},"structuredContent":{"result":"{\"success\":true,\"result\":\"verdict = plan\\nplan_path = /workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/make-plan/normalize_whitespace_plan_2026-10-11_200500.md\\nplan_parts = /workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/make-plan/normalize_whitespace_plan_2026-10-11_200500.md\",\"session_id\":\"8ef8f993-8caa-47dd-83b0-26f9e5b51139\",\"subtype\":\"success\",\"cli_subtype\":\"success\",\"is_error\":false,\"exit_code\":0,\"kill_reason\":\"kill_after_completion\",\"needs_retry\":false,\"retry_reason\":\"none\",\"stderr\":\"\",\"token_usage\":{\"backend\":\"claude-code\",\"provider_used\":\"anthropic\",\"input_tokens\":{\"state\":\"measured\",\"value\":54534},\"output_tokens\":{\"state\":\"measured\",\"value\":8817},\"cache_write_tokens\":{\"state\":\"measured_zero\",\"value\":0},\"cache_read_tokens\":{\"state\":\"measured\",\"value\":405101},\"model_breakdown\":{\"MiniMax-M3\":{\"input_tokens\":{\"state\":\"measured_zero\",\"value\":0},\"output_tokens\":{\"state\":\"measured_zero\",\"value\":0},\"cache_read_tokens\":{\"state\":\"unknown\",\"value\":null},\"cache_write_tokens\":{\"state\":\"unknown\",\"value\":null}}},\"peak_context\":{\"state\":\"unknown\",\"value\":null},\"turn_count\":10},\"write_path_warnings\":[],\"write_call_count\":1,\"fs_writes_detected\":true,\"git_writes_detected\":false,\"file_changes_count\":0,\"has_progress_evidence\":true,\"has_implementation_progress\":true,\"completion_required\":true,\"outcome_fields\":{\"plan_parts\":\"/workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/make-plan/normalize_whitespace_plan_2026-10-11_200500.md\",\"plan_path\":\"/workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/make-plan/normalize_whitespace_plan_2026-10-11_200500.md\",\"verdict\":\"plan\"},\"outcome_invariant_violated\":false,\"outcome_qualifier\":null,\"adjudication_verdict\":null,\"last_stop_reason\":\"\",\"lifespan_started\":true,\"provider_fallback\":false,\"provider_used\":\"anthropic\",\"infra_exit_category\":\"completed\",\"infra_cleanup_incomplete\":false,\"infra_fault_domain\":\"logic\",\"api_retry_count\":0,\"api_retry_last_error\":\"\",\"api_retry_last_status\":null,\"api_retry_exhausted\":false,\"api_error_status\":null,\"api_terminal_reason\":\"completed\",\"api_error_code\":\"\",\"api_error_message_seen\":false,\"candidate_exhausted\":false,\"execution_selection\":{\"selection_id\":\"838046e6cee249598edf9a01cc35eb1e\",\"campaign_id\":\"48915f19-d220-4069-8d55-b81efff84c05\",\"attempts\":[{\"candidate_id\":\"838046e6cee249598edf9a01cc35eb1e:0\",\"ordinal\":0,\"attempt\":1,\"parent_backend\":\"claude-code\",\"parent_provider\":\"anthropic\",\"parent_model\":\"sonnet\",\"requested_backend\":\"\",\"requested_provider\":\"\",\"requested_model\":\"\",\"effective_backend\":\"claude-code\",\"effective_provider\":\"anthropic\",\"effective_model\":\"sonnet\",\"backend_source_path\":\"agent_backend.backend\",\"provider_source_path\":\"backend.native_provider\",\"model_source_path\":\"backend.claude-code.model\",\"admission_status\":\"started\",\"admission_at_epoch\":1791749122,\"admission_cache_status\":\"quota_guard_disabled\",\"admission_cache_age_seconds\":null,\"rate_limit_status\":\"\",\"rate_limit_type\":\"\",\"rate_limit_resets_at_epoch\":null,\"rejection_reason\":null,\"transition\":\"primary\",\"execution_started\":true,\"child_session_id\":\"8ef8f993-8caa-47dd-83b0-26f9e5b51139\"}],\"terminal_candidate_id\":\"838046e6cee249598edf9a01cc35eb1e:0\",\"manifest_ref\":\"execution-candidates/838046e6cee249598edf9a01cc35eb1e.json\",\"invocation_deadline_epoch\":1791756276,\"remaining_retry_budget\":2,\"backend_rerouted\":false,\"candidate_fallback\":false,\"provider_fallback\":false,\"continuation\":null,\"completed\":true},\"rate_limit_status\":\"\",\"rate_limit_type\":\"\",\"rate_limit_resets_at_epoch\":null,\"pre_contamination_retry_reason\":\"none\",\"pre_contamination_subtype\":\"\",\"ndjson_unknown_event_count\":0,\"ndjson_unknown_item_count\":0,\"session_errors\":[],\"audit_status\":null,\"audit_verdict\":null,\"audit_cycle_path\":null,\"audit_attempt_id\":null,\"execution_identity\":{\"requested_parent_backend\":\"claude-code\",\"effective_parent_backend\":\"\",\"requested_parent_model\":\"sonnet\",\"effective_parent_model\":\"\",\"requested_parent_effort\":\"\",\"effective_parent_effort\":\"\",\"cli_version\":\"\",\"override_tier\":\"\",\"override_key_path\":\"\",\"parent_session_id\":\"\",\"children\":[]},\"order_id\":\"4666c3fd-486f-4ce1-b51f-9d9d660a5560\",\"receipt_id\":\"0739af7533c34089af07c982a53a62c1\"}"}},"sourceToolAssistantUUID":"ef71da75-f7f0-44ec-a940-f1b9c8affbba","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"9583a00d-f2bf-4f54-9e39-859d6ac3e38d","isSidechain":false,"message":{"id":"071b1f7556fb87c19d84e09c50262e1d","type":"message","role":"assistant","content":[{"type":"tool_use","id":"call_01a12c93aed67125a3a7504e","name":"mcp__plugin_autoskillit_autoskillit__complete_run_skill_result","input":{"receipt_id":"0739af7533c34089af07c982a53a62c1"}}],"model":"MiniMax-M3","stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":959,"cache_creation_input_tokens":0,"cache_read_input_tokens":48705,"output_tokens":89,"output_tokens_details":{"thinking_tokens":24},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionMcpServer":"plugin:autoskillit:autoskillit","attributionMcpTool":"run_skill","type":"assistant","uuid":"fd7e9b53-087c-4247-9289-e909ff5fd4e5","timestamp":"2026-10-11T20:07:18.500Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"fd7e9b53-087c-4247-9289-e909ff5fd4e5","isSidechain":false,"promptId":"886212ea-29c8-475f-a4bb-05be860f18dd","type":"user","message":{"role":"user","content":[{"tool_use_id":"call_01a12c93aed67125a3a7504e","type":"tool_result","content":"## complete_run_skill_result\n\nsuccess: True\nreceipt_id: 0739af7533c34089af07c982a53a62c1\nrun_skill_success: True\nclassification: success\nsession_id: 8ef8f993-8caa-47dd-83b0-26f9e5b51139\ntracker:\n success: True\n incarnation_matches: True\n step: plan\n order_id: 4666c3fd-486f-4ce1-b51f-9d9d660a5560\n status: complete\n pipeline_id: 4666c3fd-486f-4ce1-b51f-9d9d660a5560\n done: 1\n total: 135\ntrack...[fixture-truncated 1025 chars]"}]},"uuid":"ac39e42b-7c63-4205-b7c9-a61025b1b4d6","timestamp":"2026-10-11T20:07:18.890Z","toolUseResult":"## complete_run_skill_result\n\nsuccess: True\nreceipt_id: 0739af7533c34089af07c982a53a62c1\nrun_skill_success: True\nclassification: success\nsession_id: 8ef8f993-8caa-47dd-83b0-26f9e5b51139\ntracker:\n success: True\n incarnation_matches: True\n step: plan\n order_id: 4666c3fd-486f-4ce1-b51f-9d9d660a5560\n status: complete\n pipeline_id: 4666c3fd-486f-4ce1-b51f-9d9d660a5560\n done: 1\n total: 135\ntrack...[fixture-truncated 1025 chars]","mcpMeta":{"_meta":{"fastmcp":{"wrap_result":true}},"structuredContent":{"result":"{\"success\":true,\"receipt_id\":\"0739af7533c34089af07c982a53a62c1\",\"run_skill_success\":true,\"classification\":\"success\",\"session_id\":\"8ef8f993-8caa-47dd-83b0-26f9e5b51139\",\"tracker\":{\"success\":true,\"incarnation_matches\":true,\"step\":\"plan\",\"order_id\":\"4666c3fd-486f-4ce1-b51f-9d9d660a5560\",\"status\":\"complete\",\"pipeline_id\":\"4666c3fd-486f-4ce1-b51f-9d9d660a5560\",\"done\":1,\"total\":135},\"tracker_repairable\":false,\"recipe_segment\":{\"kind\":\"success\",\"source_step\":\"plan\",\"segments\":[],\"bodies\":[],\"recipe_execution\":{\"execution_id\":\"1dda3d6d29e545cbae216029ec657356\",\"invocation_template_digests\":{},\"skill_input_shapes\":{},\"snapshot_digest\":\"sha256:1269b37298577647aec24baeb183326c56ad0d0bf2fecbd0231b0ad6e5b81fba\"},\"pull_closures\":[],\"recipe_pull\":{\"producer_tool\":\"open_kitchen\",\"recipe_name\":\"implementation\",\"descriptor_version\":2,\"schema_version\":2,\"payload_sha256\":\"sha256:ea2e62f28435745f27a7a742cbb98d2e34cdb00d82eef53117118c3b574d9497\",\"artifact_blob_sha256\":\"sha256:cd5e7fff1ca8145532aa9ce8ca2a26ddbfe48fb704316310789b8123319f8d63\",\"artifact_blob_size_bytes\":576427,\"body_sha256\":\"sha256:3f14e54f1505f8a5c4d13c2bc5658394855e4c7dc624cc527ddb4bf0711954d5\",\"body_size_bytes\":115869,\"flow_schema_version\":2,\"flow_sha256\":\"sha256:3b9ff899035aad7555d5863de6c70c6ed95e6ffac5f19cea0c46cc17261f4f2a\",\"flow_size_bytes\":101558,\"flow_record_count\":694,\"pull_tool\":\"get_recipe_section\"}}}"}},"sourceToolAssistantUUID":"fd7e9b53-087c-4247-9289-e909ff5fd4e5","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"ce90af08-dd2d-4343-a661-8c8b229e79c6","isSidechain":false,"message":{"id":"071b20897697dd2a864edf9dfcc2d180","type":"message","role":"assistant","content":[{"type":"tool_use","id":"call_01a12c97eda97598bed1c282","name":"mcp__plugin_autoskillit_autoskillit__run_skill","input":{"skill_command":"/autoskillit:dry-walkthrough","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","step_name":"verify","recipe_execution_id":"1dda3d6d29e545cbae216029ec657356","invocation_template_digest":"sha256:7447506c5306026f763953953af00ee3cf4a723f01b786bf9aa989751756c5cf","output_dir":".autoskillit/temp","skill_inputs":{"plan_path":"/workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/make-plan/normalize_whitespace_plan_2026-10-11_200500.md","issue_url":"https://github.com/TalonT-Org/autoskillit-e2e-sandbox/issues/1","review_path":"","audit_cycle_path":"","plan_disposition_path":"","plan_set_authority_path":"/workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/plan-set-authority/planset-407e119e8ff1c06eeaf93992/0c6efb4faa404ec4fb97f4c0.json"}}}],"model":"MiniMax-M3","stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":3378,"cache_creation_input_tokens":0,"cache_read_input_tokens":160144,"output_tokens":534,"output_tokens_details":{"thinking_tokens":131},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionMcpServer":"plugin:autoskillit:autoskillit","attributionMcpTool":"run_python","type":"assistant","uuid":"b02275c1-d87d-4a7b-a28e-c87b020aac8d","timestamp":"2026-10-11T20:11:56.730Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"b02275c1-d87d-4a7b-a28e-c87b020aac8d","isSidechain":false,"promptId":"886212ea-29c8-475f-a4bb-05be860f18dd","type":"user","message":{"role":"user","content":[{"tool_use_id":"call_01a12c97eda97598bed1c282","type":"tool_result","content":"run_skill: FAIL [zero_writes]\nsession_id: 54f18144-1188-45d7-a392-17b4529f721e\nreceipt_id: 18af52568f97412b9eecc304d8a75e7c\nexit_code: 0\nneeds_retry: True\nretry_reason: zero_writes\nhas_progress_evidence: True\nadjudication_verdict: {'reason_kind': 'zero_writes', 'subtype': 'zero_writes', 'detail': 'expected implementation writes were not observed', 'outcome_fields': None, 'defects': []}\n\nresult:\nex...[fixture-truncated 46 chars]"}]},"uuid":"e945da13-1cc3-48a0-9ce5-06578b883602","timestamp":"2026-10-11T20:14:47.218Z","toolUseResult":"run_skill: FAIL [zero_writes]\nsession_id: 54f18144-1188-45d7-a392-17b4529f721e\nreceipt_id: 18af52568f97412b9eecc304d8a75e7c\nexit_code: 0\nneeds_retry: True\nretry_reason: zero_writes\nhas_progress_evidence: True\nadjudication_verdict: {'reason_kind': 'zero_writes', 'subtype': 'zero_writes', 'detail': 'expected implementation writes were not observed', 'outcome_fields': None, 'defects': []}\n\nresult:\nex...[fixture-truncated 46 chars]","mcpMeta":{"_meta":{"fastmcp":{"wrap_result":true}},"structuredContent":{"result":"{\"success\":false,\"result\":\"expected implementation writes were not observed\",\"session_id\":\"54f18144-1188-45d7-a392-17b4529f721e\",\"subtype\":\"zero_writes\",\"cli_subtype\":\"success\",\"is_error\":false,\"exit_code\":0,\"kill_reason\":\"natural_exit\",\"needs_retry\":true,\"retry_reason\":\"zero_writes\",\"stderr\":\"\",\"token_usage\":{\"backend\":\"claude-code\",\"provider_used\":\"anthropic\",\"input_tokens\":{\"state\":\"measured\",\"value\":48030},\"output_tokens\":{\"state\":\"measured\",\"value\":4349},\"cache_write_tokens\":{\"state\":\"measured_zero\",\"value\":0},\"cache_read_tokens\":{\"state\":\"measured\",\"value\":166797},\"model_breakdown\":{\"MiniMax-M3\":{\"input_tokens\":{\"state\":\"measured_zero\",\"value\":0},\"output_tokens\":{\"state\":\"measured_zero\",\"value\":0},\"cache_read_tokens\":{\"state\":\"unknown\",\"value\":null},\"cache_write_tokens\":{\"state\":\"unknown\",\"value\":null}}},\"peak_context\":{\"state\":\"unknown\",\"value\":null},\"turn_count\":5},\"write_path_warnings\":[],\"write_call_count\":0,\"fs_writes_detected\":true,\"git_writes_detected\":false,\"file_changes_count\":0,\"has_progress_evidence\":true,\"has_implementation_progress\":false,\"completion_required\":false,\"outcome_fields\":null,\"outcome_invariant_violated\":false,\"outcome_qualifier\":null,\"adjudication_verdict\":{\"reason_kind\":\"zero_writes\",\"subtype\":\"zero_writes\",\"detail\":\"expected implementation writes were not observed\",\"outcome_fields\":null,\"defects\":[]},\"last_stop_reason\":\"\",\"lifespan_started\":true,\"provider_fallback\":false,\"provider_used\":\"anthropic\",\"infra_exit_category\":\"completed\",\"infra_cleanup_incomplete\":false,\"infra_fault_domain\":\"logic\",\"api_retry_count\":0,\"api_retry_last_error\":\"\",\"api_retry_last_status\":null,\"api_retry_exhausted\":false,\"api_error_status\":null,\"api_terminal_reason\":\"completed\",\"api_error_code\":\"\",\"api_error_message_seen\":false,\"candidate_exhausted\":false,\"execution_selection\":{\"selection_id\":\"77495ee8add84a1b8334155a172ef192\",\"campaign_id\":\"48915f19-d220-4069-8d55-b81efff84c05\",\"attempts\":[{\"candidate_id\":\"77495ee8add84a1b8334155a172ef192:0\",\"ordinal\":0,\"attempt\":1,\"parent_backend\":\"claude-code\",\"parent_provider\":\"anthropic\",\"parent_model\":\"sonnet\",\"requested_backend\":\"\",\"requested_provider\":\"\",\"requested_model\":\"\",\"effective_backend\":\"claude-code\",\"effective_provider\":\"anthropic\",\"effective_model\":\"sonnet\",\"backend_source_path\":\"agent_backend.backend\",\"provider_source_path\":\"backend.native_provider\",\"model_source_path\":\"backend.claude-code.model\",\"admission_status\":\"started\",\"admission_at_epoch\":1791749517,\"admission_cache_status\":\"quota_guard_disabled\",\"admission_cache_age_seconds\":null,\"rate_limit_status\":\"\",\"rate_limit_type\":\"\",\"rate_limit_resets_at_epoch\":null,\"rejection_reason\":null,\"transition\":\"primary\",\"execution_started\":true,\"child_session_id\":\"54f18144-1188-45d7-a392-17b4529f721e\"}],\"terminal_candidate_id\":\"77495ee8add84a1b8334155a172ef192:0\",\"manifest_ref\":\"execution-candidates/77495ee8add84a1b8334155a172ef192.json\",\"invocation_deadline_epoch\":1791756276,\"remaining_retry_budget\":2,\"backend_rerouted\":false,\"candidate_fallback\":false,\"provider_fallback\":false,\"continuation\":null,\"completed\":true},\"rate_limit_status\":\"\",\"rate_limit_type\":\"\",\"rate_limit_resets_at_epoch\":null,\"pre_contamination_retry_reason\":\"none\",\"pre_contamination_subtype\":\"\",\"ndjson_unknown_event_count\":0,\"ndjson_unknown_item_count\":0,\"session_errors\":[],\"audit_status\":null,\"audit_verdict\":null,\"audit_cycle_path\":null,\"audit_attempt_id\":null,\"execution_identity\":{\"requested_parent_backend\":\"claude-code\",\"effective_parent_backend\":\"\",\"requested_parent_model\":\"sonnet\",\"effective_parent_model\":\"\",\"requested_parent_effort\":\"\",\"effective_parent_effort\":\"\",\"cli_version\":\"\",\"override_tier\":\"\",\"override_key_path\":\"\",\"parent_session_id\":\"\",\"children\":[]},\"order_id\":\"4666c3fd-486f-4ce1-b51f-9d9d660a5560\",\"receipt_id\":\"18af52568f97412b9eecc304d8a75e7c\"}"}},"sourceToolAssistantUUID":"b02275c1-d87d-4a7b-a28e-c87b020aac8d","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"3a518c19-addd-4ba2-a391-10c4149a6cd4","isSidechain":false,"message":{"id":"071b21375dfedbd6689fd160582e771b","type":"message","role":"assistant","content":[{"type":"tool_use","id":"call_01a12c9a935971faa349a923","name":"mcp__plugin_autoskillit_autoskillit__complete_run_skill_result","input":{"receipt_id":"18af52568f97412b9eecc304d8a75e7c"}}],"model":"MiniMax-M3","stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":727,"cache_creation_input_tokens":0,"cache_read_input_tokens":163522,"output_tokens":385,"output_tokens_details":{"thinking_tokens":319},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionMcpServer":"plugin:autoskillit:autoskillit","attributionMcpTool":"run_skill","type":"assistant","uuid":"d5d574c9-e7be-4957-b3c9-4f1786d9de2a","timestamp":"2026-10-11T20:14:50.215Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"d5d574c9-e7be-4957-b3c9-4f1786d9de2a","isSidechain":false,"promptId":"886212ea-29c8-475f-a4bb-05be860f18dd","type":"user","message":{"role":"user","content":[{"tool_use_id":"call_01a12c9a935971faa349a923","type":"tool_result","content":"## complete_run_skill_result\n\nsuccess: True\nreceipt_id: 18af52568f97412b9eecc304d8a75e7c\nrun_skill_success: False\nclassification: zero_writes\nsession_id: 54f18144-1188-45d7-a392-17b4529f721e\ntracker:\n success: True\n status: not_applied\ntracker_repairable: False\n\n--- RECIPE SEGMENT ---\n{\"kind\":\"recovery\",\"pull_closure\":[\"renew_plan_set\",\"create_impl_worktree\",\"implement\",\"release_issue_failure\",\"...[fixture-truncated 1059 chars]"}]},"uuid":"fc21e657-3713-460e-b508-f14b28a65117","timestamp":"2026-10-11T20:14:50.513Z","toolUseResult":"## complete_run_skill_result\n\nsuccess: True\nreceipt_id: 18af52568f97412b9eecc304d8a75e7c\nrun_skill_success: False\nclassification: zero_writes\nsession_id: 54f18144-1188-45d7-a392-17b4529f721e\ntracker:\n success: True\n status: not_applied\ntracker_repairable: False\n\n--- RECIPE SEGMENT ---\n{\"kind\":\"recovery\",\"pull_closure\":[\"renew_plan_set\",\"create_impl_worktree\",\"implement\",\"release_issue_failure\",\"...[fixture-truncated 1059 chars]","mcpMeta":{"_meta":{"fastmcp":{"wrap_result":true}},"structuredContent":{"result":"{\"success\":true,\"receipt_id\":\"18af52568f97412b9eecc304d8a75e7c\",\"run_skill_success\":false,\"classification\":\"zero_writes\",\"session_id\":\"54f18144-1188-45d7-a392-17b4529f721e\",\"tracker\":{\"success\":true,\"status\":\"not_applied\"},\"tracker_repairable\":false,\"recipe_segment\":{\"kind\":\"recovery\",\"source_step\":\"verify\",\"target_steps\":[\"renew_plan_set\",\"release_issue_failure\"],\"pull_closure\":[\"renew_plan_set\",\"create_impl_worktree\",\"implement\",\"release_issue_failure\",\"register_clone_failure\",\"escalate_stop\"],\"pull_requests\":[{\"section\":\"renew_plan_set\",\"part\":0},{\"section\":\"create_impl_worktree\",\"part\":0},{\"section\":\"implement\",\"part\":0},{\"section\":\"release_issue_failure\",\"part\":0},{\"section\":\"register_clone_failure\",\"part\":0},{\"section\":\"escalate_stop\",\"part\":0}],\"recipe_pull\":{\"producer_tool\":\"open_kitchen\",\"recipe_name\":\"implementation\",\"descriptor_version\":2,\"schema_version\":2,\"payload_sha256\":\"sha256:ea2e62f28435745f27a7a742cbb98d2e34cdb00d82eef53117118c3b574d9497\",\"artifact_blob_sha256\":\"sha256:cd5e7fff1ca8145532aa9ce8ca2a26ddbfe48fb704316310789b8123319f8d63\",\"artifact_blob_size_bytes\":576427,\"body_sha256\":\"sha256:3f14e54f1505f8a5c4d13c2bc5658394855e4c7dc624cc527ddb4bf0711954d5\",\"body_size_bytes\":115869,\"flow_schema_version\":2,\"flow_sha256\":\"sha256:3b9ff899035aad7555d5863de6c70c6ed95e6ffac5f19cea0c46cc17261f4f2a\",\"flow_size_bytes\":101558,\"flow_record_count\":694,\"pull_tool\":\"get_recipe_section\"}}}"}},"sourceToolAssistantUUID":"d5d574c9-e7be-4957-b3c9-4f1786d9de2a","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":null,"logicalParentUuid":"282278b8-9dd2-4a6f-bbaa-1105acc27bfb","isSidechain":false,"type":"system","subtype":"compact_boundary","content":"Conversation compacted","level":"info","compactMetadata":{"trigger":"auto","preTokens":166958,"postTokens":3121,"cumulativeDroppedTokens":163837,"durationMs":21279,"preservedSegment":{"headUuid":"b00cd523-124f-4830-9333-a77e74e0f7f5","anchorUuid":"c73a3283-5a21-4482-8124-50d7d3251b94","tailUuid":"282278b8-9dd2-4a6f-bbaa-1105acc27bfb"},"preservedMessages":{"anchorUuid":"c73a3283-5a21-4482-8124-50d7d3251b94","uuids":["b00cd523-124f-4830-9333-a77e74e0f7f5","5c92e852-aedc-4502-aabd-4ecc5cb13843","f2cc56c1-0cfa-4ba2-bf38-64115b02adea","282278b8-9dd2-4a6f-bbaa-1105acc27bfb"],"allUuids":["b00cd523-124f-4830-9333-a77e74e0f7f5","2aac9b5f-a477-43f9-bb2f-bf74d1f49f8c","05edcb98-33b3-4462-b670-bca0163361fe","c9579bc1-e850-46cf-8f41-586f87a894ae","8afdb5bf-7d28-49c8-af1a-08f5a3c343fe","62b75b38-cc5e-4922-b5a5-398281472724","52c3981f-6147-4b1f-8256-531acee3828c","4f22a38f-6841-42e3-ba27-40a45b71387f","d09734c1-860a-4015-8590-1c8491e75c14","18b90c12-6bfb-4bd0-8c79-f7d3c7602c87","0287af2b-1d5a-4266-8e75-29c3e96ec868","0a3f287e-7607-4ba9-b042-dd1701ebea9d","0e407ba0-f60d-4f94-9ba8-4543674e2e3f","cb4e9a54-e7ed-433e-950c-e93211025deb","6aa3d78b-4514-456f-a42a-a76e136692d6","04dd0b8b-f81f-4fdd-a777-59a57d69eeb5","5c92e852-aedc-4502-aabd-4ecc5cb13843","fb3efad1-a773-4c1d-9d47-b6760694a153","f2cc56c1-0cfa-4ba2-bf38-64115b02adea","282278b8-9dd2-4a6f-bbaa-1105acc27bfb"]}},"uuid":"9a99fc5b-0ab0-4f91-a84b-4b2196d3b702","timestamp":"2026-10-11T20:15:22.874Z","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main","slug":"shimmying-tinkering-dream"} +{"parentUuid":"a286e33c-9ae5-4118-90ea-99d928d70a15","isSidechain":false,"message":{"id":"071b2190376d0b42f31507604c7b85ae","type":"message","role":"assistant","content":[{"type":"text","text":"The analyze_pipeline_health_error is `optional: true` with `skip_when_false: inputs.pipeline_health`. Since pipeline_health wasn't set, this will skip to escalate_stop, which is the stop action.\n\nThe verify step's zero_writes failure has been correctly routed to release_issue_failure → register_clone_failure → escalate_stop. The pipeline is now at the terminal stop point. Per the message:\n\n> \"Pipe...[fixture-truncated 1476 chars]"}],"model":"MiniMax-M3","stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":455,"cache_creation_input_tokens":0,"cache_read_input_tokens":41427,"output_tokens":472,"output_tokens_details":{"thinking_tokens":0},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":0,"attributionMcpServer":"plugin:autoskillit:autoskillit","attributionMcpTool":"run_cmd","type":"assistant","uuid":"8136c073-129f-45bf-a17c-71a2d795dd38","timestamp":"2026-10-11T20:16:20.582Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main","slug":"shimmying-tinkering-dream"} +{"parentUuid":"8136c073-129f-45bf-a17c-71a2d795dd38","isSidechain":false,"type":"system","subtype":"stop_hook_summary","hookCount":2,"hookInfos":[{"command":"python3 -B \"${CLAUDE_PLUGIN_ROOT}/hooks/_dispatch.py\" guards/join_stop_guard","durationMs":117},{"command":"python3 -B \"${CLAUDE_PLUGIN_ROOT}/hooks/_dispatch.py\" session_lifetime_notice_hook","durationMs":99}],"hookErrors":[],"hookAdditionalContext":[],"preventedContinuation":false,"stopReason":"","hasOutput":false,"level":"suggestion","timestamp":"2026-10-11T20:16:20.704Z","uuid":"28868d68-9669-4c8c-874c-7b76c09003b0","toolUseID":"fdb4c1c9-f5b5-4999-8095-64dd918d9388","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main","slug":"shimmying-tinkering-dream"} +{"type":"cost-state","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","totalCostUSD":3.1986767999999994,"totalAPIDuration":314689,"totalAPIDurationWithoutRetries":314423,"totalToolDuration":300139,"totalLinesAdded":0,"totalLinesRemoved":0,"totalDuration":704309,"startTime":1791749076557,"modelUsage":{"claude-sonnet-5":{"inputTokens":206383,"outputTokens":43841,"thinkingTokens":17593,"cacheReadInputTokens":11737504,"cacheCreationInputTokens":0,"webSearchRequests":0,"costUSD":3.1986767999999994}},"hasUnknownModelCost":false} diff --git a/tests/infra/test_check_complexity.py b/tests/infra/test_check_complexity.py index 147fe49ea8..a936d8e080 100644 --- a/tests/infra/test_check_complexity.py +++ b/tests/infra/test_check_complexity.py @@ -15,12 +15,12 @@ import pytest +from tests._script_loader import load_script from tests.infra._complexity_helpers import ( _CONSTRUCT_CASES, _MINIMAL_LIMITS, _git, _source_with_function, - load_check_script, ) pytestmark = [pytest.mark.layer("infra"), pytest.mark.small] @@ -29,7 +29,7 @@ _CHECK_SCRIPT = REPO_ROOT / "scripts" / "check_complexity.py" _CHECK_MODULE_NAME = "_autoskillit_check_complexity" -check = load_check_script(_CHECK_MODULE_NAME, _CHECK_SCRIPT) +check = load_script(_CHECK_MODULE_NAME, _CHECK_SCRIPT) git_plumbing = sys.modules["_git_plumbing"] @@ -657,7 +657,7 @@ def test_undecodable_working_tree_source_exits_2( assert "encoding" in capsys.readouterr().err -def test_load_check_script_restores_sys_path_and_keeps_success_registration( +def test_load_script_restores_sys_path_and_keeps_success_registration( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -667,7 +667,7 @@ def test_load_check_script_restores_sys_path_and_keeps_success_registration( monkeypatch.delitem(sys.modules, name, raising=False) previous_path = list(sys.path) - loaded = load_check_script(name, script) + loaded = load_script(name, script) assert sys.path == previous_path assert sys.modules[name] is loaded @@ -675,7 +675,7 @@ def test_load_check_script_restores_sys_path_and_keeps_success_registration( @pytest.mark.parametrize("has_previous", [False, True]) -def test_load_check_script_restores_registration_after_failure( +def test_load_script_restores_registration_after_failure( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, has_previous: bool, @@ -693,7 +693,7 @@ def test_load_check_script_restores_registration_after_failure( previous_path = list(sys.path) with pytest.raises(RuntimeError, match="load failed"): - load_check_script(name, script) + load_script(name, script) assert sys.path == previous_path if has_previous: diff --git a/tests/infra/test_check_complexity_git_e2e.py b/tests/infra/test_check_complexity_git_e2e.py index f7ecfed34c..0785f753b0 100644 --- a/tests/infra/test_check_complexity_git_e2e.py +++ b/tests/infra/test_check_complexity_git_e2e.py @@ -13,12 +13,8 @@ import pytest -from tests.infra._complexity_helpers import ( - _MINIMAL_LIMITS, - _git, - _source_with_function, - load_check_script, -) +from tests._script_loader import load_script +from tests.infra._complexity_helpers import _MINIMAL_LIMITS, _git, _source_with_function pytestmark = [pytest.mark.layer("infra"), pytest.mark.medium] @@ -26,7 +22,7 @@ _CHECK_SCRIPT = REPO_ROOT / "scripts" / "check_complexity.py" _CHECK_MODULE_NAME = "_autoskillit_check_complexity_git_e2e" -check = load_check_script(_CHECK_MODULE_NAME, _CHECK_SCRIPT) +check = load_script(_CHECK_MODULE_NAME, _CHECK_SCRIPT) # --- disposable repo fixture: seeding differs from test_check_complexity.py (branch diff --git a/tests/infra/test_check_complexity_ruff_parity.py b/tests/infra/test_check_complexity_ruff_parity.py index d3612caeab..a9b3c9db0a 100644 --- a/tests/infra/test_check_complexity_ruff_parity.py +++ b/tests/infra/test_check_complexity_ruff_parity.py @@ -15,8 +15,9 @@ import pytest +from tests._script_loader import load_script from tests.conftest import production_interpreter_env -from tests.infra._complexity_helpers import _CONSTRUCT_CASES, load_check_script +from tests.infra._complexity_helpers import _CONSTRUCT_CASES pytestmark = [pytest.mark.layer("infra"), pytest.mark.medium] @@ -25,7 +26,7 @@ _CHECK_MODULE_NAME = "_autoskillit_check_complexity_ruff_parity" _RUFF_TIMEOUT_SECONDS = 30 -check = load_check_script(_CHECK_MODULE_NAME, _CHECK_SCRIPT) +check = load_script(_CHECK_MODULE_NAME, _CHECK_SCRIPT) # --- ruff invocation: a required dependency, never skipped -------------------------------- diff --git a/tests/infra/test_e2e_evidence.py b/tests/infra/test_e2e_evidence.py new file mode 100644 index 0000000000..071b36e3f3 --- /dev/null +++ b/tests/infra/test_e2e_evidence.py @@ -0,0 +1,27 @@ +"""Behavioral checks for the captured E2E evidence helper.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from tests.infra._e2e_evidence import _PROJECTS_TOKEN, _replace_projects_token + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.medium] + + +def test_replace_projects_token_json_escapes_paths(tmp_path: Path) -> None: + trace = tmp_path / "trace.jsonl" + projects = Path(f'{tmp_path}/project "quoted" path\\with\\backslashes') + trace.write_text( + json.dumps({"transcript": f"{_PROJECTS_TOKEN}/session.jsonl"}), + encoding="utf-8", + ) + + _replace_projects_token(trace, projects) + + assert json.loads(trace.read_text(encoding="utf-8")) == { + "transcript": f"{projects}/session.jsonl" + } diff --git a/tests/infra/test_e2e_harness.py b/tests/infra/test_e2e_harness.py index e9f9ad26dd..104938a659 100644 --- a/tests/infra/test_e2e_harness.py +++ b/tests/infra/test_e2e_harness.py @@ -5,6 +5,7 @@ import json import math import os +import shlex import stat import subprocess import tempfile @@ -12,17 +13,21 @@ from dataclasses import dataclass, replace from pathlib import Path from types import SimpleNamespace +from typing import Any +from urllib.parse import quote import pytest -from tests.infra._complexity_helpers import load_check_script +from tests._hook_protocol_oracle import claude_matcher_admits +from tests._script_loader import load_script +from tests.infra._e2e_evidence import load_provenance, materialize_run pytestmark = [pytest.mark.layer("infra"), pytest.mark.medium] REPO_ROOT = Path(__file__).resolve().parents[2] _HARNESS_SCRIPT = REPO_ROOT / "scripts" / "e2e" / "e2e_harness.py" -harness = load_check_script("_autoskillit_e2e_harness", _HARNESS_SCRIPT) +harness = load_script("_autoskillit_e2e_harness", _HARNESS_SCRIPT) e2e_catalog = harness.e2e_catalog _AUDIT_SLOT = f"sha256:{'a' * 64}" @@ -500,7 +505,7 @@ def test_unretained_lists_recorded_files_that_were_not_kept( {"cwd": str(clone_a), "outcome_fields": {"remediation_path": missing}}, ) home = tmp_path / "home" - index = harness.autoskillit_log_dir(home) / "sessions.jsonl" + index = harness.e2e_sessions.default_log_root(home) / "sessions.jsonl" index.parent.mkdir(parents=True, exist_ok=True) index.write_text( "".join(json.dumps(row) + "\n" for row in rows) + "{malformed line\n", @@ -524,7 +529,7 @@ def test_index_read_error_preserves_collected_evidence( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, evidence_sources, error ) -> None: kept, excluded = evidence_sources - index = harness.autoskillit_log_dir(tmp_path / "home") / "sessions.jsonl" + index = harness.e2e_sessions.default_log_root(tmp_path / "home") / "sessions.jsonl" original = Path.read_text def unreadable(path, *args, **kwargs): @@ -670,9 +675,13 @@ def test_worktrees_directory_matches_clone_source_of_truth() -> None: "message": "model reported failure", "issue": "https://github.com/TalonT-Org/AutoSkillit/issues/5232", } +_SANDBOX_IMPLEMENTATION_ISSUE = "https://github.com/owner/sandbox/issues/5233" +_UPSTREAM_EXPECTED_IMPLEMENTATION_ISSUE = "https://github.com/TalonT-Org/AutoSkillit/issues/5233" +_PIPELINE_HEAD = "0123456789abcdef" +_PIPELINE_BRANCH = "codex/issue-5233" -@dataclass +@dataclass(eq=False) class RecordedCall: argv: list[str] cwd: Path | None @@ -706,6 +715,39 @@ def _completed(argv, stdout: str = "", returncode: int = 0, stderr: str = ""): return subprocess.CompletedProcess(list(argv), returncode, stdout, stderr) +def _recorded_calls( + calls: list[RecordedCall], + *, + prefix: list[str] | None = None, + exact: list[str] | None = None, + contains: str | None = None, +) -> list[RecordedCall]: + return [ + call + for call in calls + if (prefix is None or call.argv[: len(prefix)] == prefix) + and (exact is None or call.argv == exact) + and (contains is None or contains in " ".join(call.argv)) + ] + + +def _recorded_pr_list_calls(calls: list[RecordedCall], *, baseline: bool) -> list[RecordedCall]: + lists = _recorded_calls(calls, prefix=["gh", "pr", "list"]) + selected = [] + for call in lists: + is_baseline = call.argv[call.argv.index("--limit") + 1] == "1" + if (baseline and is_baseline) or (not baseline and not is_baseline): + selected.append(call) + return selected + + +def _recorded_pr_detail_calls(calls: list[RecordedCall], number: int) -> list[RecordedCall]: + suffix = f"/pulls/{number}" + return [ + call for call in calls if call.argv[:2] == ["gh", "api"] and call.argv[2].endswith(suffix) + ] + + def _pr(number: int, state: str, additions: int = 3, deletions: int = 1) -> dict: return {"number": number, "state": state, "additions": additions, "deletions": deletions} @@ -777,7 +819,7 @@ def _seed_sessions(log_dir: Path, sessions: dict[str, list[int | None] | None]) def _log_dir(tmp_path: Path) -> Path: - return tmp_path / "home" / ".local" / "share" / "autoskillit" / "logs" + return harness.e2e_sessions.default_log_root(tmp_path / "home") def _env(**overrides: str) -> dict[str, str]: @@ -793,6 +835,394 @@ def _env(**overrides: str) -> dict[str, str]: return {name: value for name, value in env.items() if value} +def _pipeline_catalog( + expected_failures: list[dict[str, str]] | None = None, *, repository: str = SANDBOX +) -> e2e_catalog.Catalog: + entry = { + "name": "implementation", + "kind": "recipe", + "pipeline": True, + "peak_sessions": 6, + "timeout_sec": 600, + "trigger_paths": ["src/*"], + "recipe": "implementation", + "ingredients": {"source_dir": "/workspace/sandbox", "open_pr": "true"}, + "expected_pull_request_state": "open", + "issue_url_env": "E2E_IMPLEMENTATION_ISSUE_URL", + "required_changed_paths": ["sandbox/", "tests/"], + "test_command": ["task", "test-check"], + "peak_sessions_evidence": "recorded session events", + } + if expected_failures is not None: + entry["expected_failures"] = expected_failures + return e2e_catalog.parse_catalog({"sandbox_repository": repository, "tests": [entry]}) + + +def _pipeline_pr(number: int = 8, *, branch: str = _PIPELINE_BRANCH) -> dict[str, Any]: + return { + "number": number, + "state": "open", + "merged": False, + "merged_at": None, + "auto_merge": None, + "body": f"Implements {_SANDBOX_IMPLEMENTATION_ISSUE}", + "additions": 5, + "deletions": 2, + "head": { + "ref": branch, + "sha": _PIPELINE_HEAD, + "repo": {"full_name": SANDBOX}, + }, + "base": {"ref": "main", "repo": {"full_name": SANDBOX}}, + } + + +def _pipeline_setup_response(argv: list[str], state: dict[str, Any]): + if argv[:3] == ["autoskillit", "config", "show"]: + return _completed(argv, json.dumps(state["config"])) + if argv[:3] == ["gh", "issue", "view"]: + fields = argv[argv.index("--json") + 1].split(",") + payload = state["seed"] if "url" in fields else state["final_issue"] + return _completed(argv, json.dumps(payload)) + if argv[:3] == ["gh", "repo", "view"]: + payload = {"defaultBranchRef": {"name": state["default_branch"]}} + return _completed(argv, json.dumps(payload)) + if argv[:3] == ["gh", "pr", "list"]: + rows = [{"number": 7}] if argv[argv.index("--limit") + 1] == "1" else state["listed_prs"] + return _completed(argv, json.dumps(rows)) + return None + + +def _pipeline_execution_response(argv: list[str], state: dict[str, Any]): + if argv[:3] == ["autoskillit", "fleet", "run"]: + if state["fleet_exception"] is not None: + raise state["fleet_exception"] + state["head_refs"].extend(state["published_refs"]) + return _completed(argv, json.dumps(state["fleet_envelope"]) + "\n") + if argv[:2] == ["task", "test-check"]: + state["test_check_count"] += 1 + if state["test_check_count"] > 1 and state["pr_test_returncode"]: + return _completed(argv, returncode=state["pr_test_returncode"], stderr="tests failed") + return None + if argv[:2] == ["git", "rev-parse"]: + return _completed(argv, state["fetch_oid"] + "\n") + return None + + +def _pipeline_ci_api_response(argv: list[str], endpoint: str, state: dict[str, Any]): + if endpoint.endswith("/actions/workflows?per_page=100"): + workflows = [{"id": 1, "name": "CI", "state": "active"}] + return _completed(argv, json.dumps([{"workflows": workflows}])) + if "/actions/workflows/1/runs?" in endpoint: + runs = [] + if state["workflow_present"]: + runs.append( + { + "id": 2, + "status": state["workflow_status"], + "conclusion": state["workflow_conclusion"], + "pull_requests": [ + { + "number": 8, + "head": {"sha": _PIPELINE_HEAD, "repo": {"full_name": SANDBOX}}, + "base": {"repo": {"full_name": SANDBOX}}, + } + ], + } + ) + return _completed(argv, json.dumps([{"workflow_runs": runs}])) + if endpoint.endswith("/actions/runs/2/jobs?per_page=100"): + job = {"name": "test", "status": "completed", "conclusion": state["job_conclusion"]} + return _completed(argv, json.dumps([{"jobs": [job]}])) + return None + + +def _pipeline_pull_request_api_response(argv: list[str], endpoint: str, state: dict[str, Any]): + if "/pulls/" not in endpoint: + return None + if endpoint.endswith("/files?per_page=100"): + return _completed(argv, json.dumps([state["files"]])) + number = int(endpoint.split("/pulls/", 1)[1].split("/", 1)[0]) + return _completed(argv, json.dumps(state["pr_details"][number])) + + +def _pipeline_refs_response(argv: list[str], endpoint: str, state: dict[str, Any]): + if "/git/matching-refs/heads/" not in endpoint: + return None + state["refs_list_calls"] += 1 + error = state["refs_list_faults"].get(state["refs_list_calls"]) + if error is not None: + raise error + response = state["refs_list_response"] + if response is None: + response = [[{"ref": f"refs/heads/{name}"} for name in state["head_refs"]]] + return _completed(argv, json.dumps(response)) + + +def _pipeline_api_request(argv: list[str]) -> tuple[str, str]: + if argv[:2] != ["gh", "api"]: + raise AssertionError(f"unsupported gh api argv: {argv!r}") + method = "GET" + endpoint = None + arguments = iter(argv[2:]) + for argument in arguments: + if argument == "--method": + method = next(arguments, "").upper() + elif argument in {"--include", "--paginate", "--slurp"}: + continue + elif not argument.startswith("-") and endpoint is None: + endpoint = argument + else: + raise AssertionError(f"unsupported gh api argv: {argv!r}") + if method not in {"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD"} or endpoint is None: + raise AssertionError(f"unsupported gh api argv: {argv!r}") + return method, endpoint + + +def _pipeline_cleanup_api_response(argv: list[str], endpoint: str, state: dict[str, Any]): + method, endpoint = _pipeline_api_request(argv) + if method == "DELETE" and "/git/refs/heads/" in endpoint: + error = state["cleanup_errors"].get("branch_delete") + if error is not None: + raise error + ref_name = endpoint.split("/git/refs/heads/", 1)[1] + state["head_refs"] = [ + name for name in state["head_refs"] if quote(name, safe="") != ref_name + ] + return None + if method == "GET" and "/git/refs/heads/" in endpoint: + error = state["cleanup_errors"].get("branch_get") + if error is not None: + raise error + return _completed(argv, "{}") + return None + + +def _pipeline_api_response(argv: list[str], state: dict[str, Any]): + if argv[:2] != ["gh", "api"]: + return None + _method, endpoint = _pipeline_api_request(argv) + for route in ( + _pipeline_ci_api_response, + _pipeline_pull_request_api_response, + _pipeline_refs_response, + _pipeline_cleanup_api_response, + ): + response = route(argv, endpoint, state) + if response is not None: + return response + return None + + +def test_pipeline_api_request_parses_supported_flags_and_rejects_unknown_shapes() -> None: + assert _pipeline_api_request( + ["gh", "api", "--method", "DELETE", "repos/owner/repo/git/refs/heads/example"] + ) == ("DELETE", "repos/owner/repo/git/refs/heads/example") + assert _pipeline_api_request( + ["gh", "api", "repos/owner/repo/pulls", "--paginate", "--slurp"] + ) == ("GET", "repos/owner/repo/pulls") + + assert _pipeline_api_request( + ["gh", "api", "--include", "--method", "PATCH", "repos/owner/repo/issues/1"] + ) == ("PATCH", "repos/owner/repo/issues/1") + + malformed = ["gh", "api", "--unknown", "repos/owner/repo"] + with pytest.raises(AssertionError, match="unsupported gh api argv") as error: + _pipeline_api_request(malformed) + assert repr(malformed) in str(error.value) + + +@pytest.mark.parametrize( + "argv", + [ + ["gh", "api", "--method", "PATCH", "repos/owner/repo/git/refs/heads/example"], + ["gh", "api", "--method", "DELETE", "repos/owner/repo/pulls/8"], + ], +) +def test_pipeline_branch_delete_fault_only_applies_to_branch_delete_requests( + argv: list[str], +) -> None: + state = {"cleanup_errors": {"branch_delete": RuntimeError("delete failed")}} + endpoint = _pipeline_api_request(argv)[1] + + assert _pipeline_cleanup_api_response(argv, endpoint, state) is None + + +def _pipeline_cleanup_response(argv: list[str], state: dict[str, Any]): + if argv[:3] == ["gh", "pr", "close"]: + error = state["cleanup_errors"].get("pr_close") + if error is not None: + raise error + return None + + +def _pipeline_response(argv: list[str], state: dict[str, Any]): + for route in ( + _pipeline_setup_response, + _pipeline_execution_response, + _pipeline_api_response, + _pipeline_cleanup_response, + ): + response = route(argv, state) + if response is not None: + return response + return None + + +def _pipeline_runner() -> tuple[FakeRunner, dict[str, Any]]: + seed = { + "number": 5233, + "url": _SANDBOX_IMPLEMENTATION_ISSUE, + "state": "OPEN", + "title": "Add a greeting", + "body": "Implement the requested behavior.", + "labels": [], + } + listed = {"number": 8, "headRefOid": _PIPELINE_HEAD} + state: dict[str, Any] = { + "seed": seed, + "default_branch": "main", + "config": { + "github": {"in_progress_label": "in-progress"}, + "branching": {"default_base_branch": "main"}, + }, + "listed_prs": [listed], + "pr_details": {8: _pipeline_pr()}, + "files": [ + {"filename": "sandbox/worker.py", "status": "added", "additions": 3, "deletions": 0}, + { + "filename": "tests/test_worker.py", + "status": "modified", + "additions": 2, + "deletions": 1, + }, + ], + "final_issue": {"state": "OPEN", "labels": [{"name": "in-progress"}]}, + "fetch_oid": _PIPELINE_HEAD, + "test_check_count": 0, + "pr_test_returncode": 0, + "workflow_conclusion": "success", + "workflow_present": True, + "workflow_status": "completed", + "job_conclusion": "success", + "fleet_envelope": { + "success": True, + "dispatch_status": "success", + "dispatch_id": "dispatch-5233", + "dispatched_session_id": "session-5233", + "l3_payload": {"reason": "implementation_complete"}, + }, + "fleet_exception": None, + "head_refs": ["main"], + "published_refs": [], + "refs_list_calls": 0, + "refs_list_faults": {}, + "refs_list_response": None, + "cleanup_errors": {}, + "proof": {"branches": [_PIPELINE_BRANCH], "claimed": True, "violations": []}, + "summary": {"coverage": {"complete": True}, "peak_sessions": 4, "violations": []}, + } + + return FakeRunner(lambda argv: _pipeline_response(argv, state)), state + + +def _configure_failed_pipeline_state(state: dict[str, Any]) -> None: + state["listed_prs"] = [] + state["pr_details"] = {} + state["final_issue"] = {"state": "OPEN", "labels": []} + state["proof"] = {"branches": [], "claimed": False, "violations": []} + + +def _retry_exhausted_envelope() -> dict[str, Any]: + return { + "success": False, + "dispatch_status": "retry_exhausted", + "dispatch_id": "dispatch-5233", + "dispatched_session_id": "session-5233", + "reason": "known orchestration failure", + } + + +def _patch_pipeline_measurement( + monkeypatch: pytest.MonkeyPatch, state: dict[str, Any] +) -> dict[str, list[tuple[Any, ...]]]: + observed: dict[str, list[tuple[Any, ...]]] = {"ownership": [], "summaries": []} + + def collect(trace_path, envelope, log_root, issue_url, repository, *, claim_label=None): + observed["ownership"].append( + (trace_path, envelope, log_root, issue_url, repository, claim_label) + ) + return state["proof"] + + def summarize(trace_path, envelope, log_root): + observed["summaries"].append((trace_path, envelope, log_root)) + return state["summary"] + + monkeypatch.setattr(harness.e2e_sessions, "collect_ownership", collect) + monkeypatch.setattr(harness.e2e_sessions, "summarize", summarize) + monkeypatch.setattr(harness.time, "sleep", lambda _seconds: None) + return observed + + +def _run_pipeline( + tmp_path: Path, + runner: FakeRunner, + state: dict[str, Any], + monkeypatch: pytest.MonkeyPatch, + *, + env: dict[str, str] | None = None, + expected_failures: list[dict[str, str]] | None = None, + real_measurement: bool = False, + repository: str = SANDBOX, +) -> tuple[ + list[str], + dict[str, Any], + Path, + Path, + e2e_catalog.Catalog, + dict[str, list[tuple[Any, ...]]] | None, +]: + catalog = _pipeline_catalog(expected_failures, repository=repository) + original_handler = runner.handler + + def handle(argv: list[str]): + if argv[:3] == ["autoskillit", "fleet", "run"]: + _seed_sessions( + harness.e2e_sessions.default_log_root(tmp_path / "home"), + state.get( + "sessions", {"pipeline-session": [_EXPECTED_REQUIRED_CONTEXT_WINDOW_TOKENS]} + ), + ) + return original_handler(argv) + + runner.handler = handle + out = tmp_path / "out" + home = tmp_path / "home" + if real_measurement: + monkeypatch.setattr(harness.time, "sleep", lambda _seconds: None) + observed = None + else: + observed = _patch_pipeline_measurement(monkeypatch, state) + failures = harness.run_test( + catalog.get("implementation"), + catalog, + out=out, + home=home, + env=env + if env is not None + else _env(E2E_IMPLEMENTATION_ISSUE_URL=_SANDBOX_IMPLEMENTATION_ISSUE), + runner=runner, + ) + return ( + failures, + json.loads((out / "result.json").read_text(encoding="utf-8")), + out, + home, + catalog, + observed, + ) + + def _run_recipe( tmp_path: Path, runner: FakeRunner, @@ -2309,10 +2739,10 @@ def test_steps_run_in_order_with_the_token_only_on_stdin(self, tmp_path: Path) - def test_outputs_keep_full_stdout_and_only_the_envelope_line(self, tmp_path: Path) -> None: _run_recipe(tmp_path, FakeRunner(_recipe_handler(log_dir=_log_dir(tmp_path)))) out = tmp_path / "out" - stdout = (out / "fleet-run.stdout.log").read_text(encoding="utf-8") + stdout = (out / "fleet-run.stdout.txt").read_text(encoding="utf-8") assert stdout == f'{_SKILL_NOTICE}\n{{"success": true}}\n' assert (out / "envelope.json").read_text(encoding="utf-8") == '{"success": true}\n' - assert (out / "fleet-run.stderr.log").exists() + assert (out / "fleet-run.stderr.txt").exists() def test_harness_writes_provider_settings_and_user_config(self, tmp_path: Path) -> None: _run_recipe(tmp_path, FakeRunner(_recipe_handler(log_dir=_log_dir(tmp_path)))) @@ -2346,7 +2776,9 @@ def reject_login(argv: list[str]): assert ("autoskillit", "fleet", "run") not in prefixes def test_fleet_timeout_and_window_failures_keep_cleanup(self, tmp_path: Path) -> None: - timeout = subprocess.TimeoutExpired(["autoskillit"], 720) + timeout = subprocess.TimeoutExpired( + ["autoskillit"], 720, output=b"partial stdout", stderr=b"partial stderr" + ) runner = FakeRunner( _recipe_handler( log_dir=_log_dir(tmp_path), @@ -2361,6 +2793,8 @@ def test_fleet_timeout_and_window_failures_keep_cleanup(self, tmp_path: Path) -> f"session windows: timed-out: observed [200000] != required " f"{_EXPECTED_REQUIRED_CONTEXT_WINDOW_TOKENS}" ) in failures + assert (tmp_path / "out" / "fleet-run.stdout.txt").read_text() == "partial stdout" + assert (tmp_path / "out" / "fleet-run.stderr.txt").read_text() == "partial stderr" assert runner.calls[-1].argv[:4] == ["gh", "pr", "close", "8"] def test_unexpected_fleet_error_records_failure_and_still_cleans_up( @@ -2489,6 +2923,785 @@ def test_expected_failure_keeps_a_window_failure(self, tmp_path: Path) -> None: assert result["expected_findings"] == [expected] +class TestPipelineRecipeFlow: + def test_pipeline_rejects_small_context_window_and_still_cleans_up( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + runner, state = _pipeline_runner() + state["sessions"] = {"pipeline-session": [200000]} + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert result["passed"] is False + assert ( + "session windows: pipeline-session: observed [200000] != required " + f"{_EXPECTED_REQUIRED_CONTEXT_WINDOW_TOKENS}" + ) in failures + assert ("gh", "pr", "close", "8") in runner.argv_prefixes(4) + + def test_session_hooks_install_the_declared_policy_and_keep_existing_settings( + self, tmp_path: Path + ) -> None: + home = tmp_path / "home" + settings_path = home / ".claude" / "settings.json" + settings_path.parent.mkdir(parents=True, exist_ok=True) + foreign_subagent_stop = { + "matcher": "ForeignAgent", + "hooks": [{"type": "command", "command": "foreign"}], + } + other_pre_tool_use = { + "matcher": "Bash", + "hooks": [{"type": "command", "command": "other"}], + } + settings_path.write_text( + json.dumps( + { + "theme": "dark", + "env": {"KEEP1": "1"}, + "hooks": { + "SubagentStop": [foreign_subagent_stop], + "PreToolUse": [other_pre_tool_use], + }, + } + ), + encoding="utf-8", + ) + trace = tmp_path / "trace with space" / "events.jsonl" + + harness.install_session_hooks(home, trace) + + settings = json.loads(settings_path.read_text(encoding="utf-8")) + lifecycle_events = {"SessionStart", "SessionEnd", "SubagentStart", "SubagentStop"} + declarations = harness.e2e_sessions.LIFECYCLE_HOOK_MATCHERS + assert set(declarations) == lifecycle_events + assert set(settings["hooks"]) == lifecycle_events | {"PreToolUse"} + assert settings["theme"] == "dark" + assert settings["env"] == {"KEEP1": "1"} + assert settings["hooks"]["PreToolUse"] == [other_pre_tool_use] + assert settings["hooks"]["SubagentStop"][0] == foreign_subagent_stop + + expected_command_hook = { + "type": "command", + "command": f"python3 /opt/e2e/e2e_sessions.py {shlex.quote(str(trace))}", + "timeout": 10, + } + for event in lifecycle_events: + installed = settings["hooks"][event] + if event == "SubagentStop": + installed = installed[1:] + assert len(installed) == 1 + entry = installed[0] + assert entry["hooks"] == [expected_command_hook] + declared_matcher = declarations[event] + if declared_matcher is None: + assert "matcher" not in entry + else: + assert entry["matcher"] == declared_matcher + assert not claude_matcher_admits(entry["matcher"], "") + + def test_session_hooks_refuse_a_custom_main_agent(self, tmp_path: Path) -> None: + home = tmp_path / "home" + settings_path = home / ".claude" / "settings.json" + settings_path.parent.mkdir(parents=True, exist_ok=True) + settings_path.write_bytes(b'{"agent":"custom-main"}') + original = settings_path.read_bytes() + + with pytest.raises(ValueError, match="main agent"): + harness.install_session_hooks(home, tmp_path / "events.jsonl") + + assert settings_path.read_bytes() == original + + def test_custom_main_agent_setting_fails_preflight( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + settings_path = tmp_path / "home" / ".claude" / "settings.json" + settings_path.parent.mkdir(parents=True, exist_ok=True) + settings_path.write_text('{"agent":"custom-main"}', encoding="utf-8") + runner, state = _pipeline_runner() + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert failures == ["preflight: session hooks: settings.json sets a main agent"] + assert result["outcome"] == "failed" + assert not any(call.argv[:3] == ["autoskillit", "fleet", "run"] for call in runner.calls) + + def test_pipeline_uses_the_seed_issue_and_cleans_only_its_owned_pr( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + runner, state = _pipeline_runner() + unrelated = {**state["listed_prs"][0], "number": 9} + state["listed_prs"].append(unrelated) + state["pr_details"][9] = _pipeline_pr(9, branch="other/branch") + failures, result, out, home, catalog, observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert failures == [] + assert result == { + "test": "implementation", + "passed": True, + "failures": [], + "outcome": "passed", + "expected_findings": [], + } + test = catalog.get("implementation") + assert test.pipeline is True + assert test.issue_url_env == "E2E_IMPLEMENTATION_ISSUE_URL" + assert test.required_changed_paths == ("sandbox/", "tests/") + assert test.test_command == ("task", "test-check") + + calls = runner.calls + config_call = _recorded_calls(calls, prefix=["autoskillit", "config", "show"])[0] + assert config_call.cwd == harness.SANDBOX_CLONE + assert config_call.env == { + **harness.child_env(_env(E2E_IMPLEMENTATION_ISSUE_URL=_SANDBOX_IMPLEMENTATION_ISSUE)), + "AUTOSKILLIT_LOG_DIR": str(_log_dir(tmp_path)), + } + seed_call = _recorded_calls(calls, prefix=["gh", "issue", "view"])[0] + repo_call = _recorded_calls(calls, prefix=["gh", "repo", "view"])[0] + baseline_install = _recorded_calls(calls, exact=["task", "install-worktree"])[0] + checks = _recorded_calls(calls, exact=["task", "test-check"]) + baseline_list = _recorded_pr_list_calls(calls, baseline=True)[0] + fleet_call = _recorded_calls(calls, prefix=["autoskillit", "fleet", "run"])[0] + assert ( + calls.index(config_call) + < calls.index(seed_call) + < calls.index(repo_call) + < calls.index(baseline_install) + < calls.index(checks[0]) + < calls.index(baseline_list) + < calls.index(fleet_call) + ) + assert baseline_install.cwd == checks[0].cwd == harness.SANDBOX_CLONE + assert len(checks) == 2 and checks[1].cwd == harness.SANDBOX_CLONE + + listed_calls = _recorded_pr_list_calls(calls, baseline=False) + assert all( + call.argv[call.argv.index("--json") + 1] == "number,headRefOid" + for call in listed_calls + ) + file_call = _recorded_calls(calls, contains="/files?per_page=100")[0] + fetch_call = _recorded_calls(calls, prefix=["git", "fetch"])[0] + oid_call = _recorded_calls(calls, prefix=["git", "rev-parse"])[0] + checkout_call = _recorded_calls(calls, prefix=["git", "checkout", "--detach"])[0] + install_calls = _recorded_calls(calls, exact=["task", "install-worktree"]) + workflow_call = _recorded_calls( + calls, prefix=["gh", "api"], contains="/actions/workflows?per_page=100" + )[0] + runs_call = _recorded_calls(calls, contains="/actions/workflows/1/runs?")[0] + jobs_call = _recorded_calls(calls, contains="/actions/runs/2/jobs?")[0] + pr_reads = _recorded_pr_detail_calls(calls, 8) + close = _recorded_calls(calls, prefix=["gh", "pr", "close"])[0] + branch_delete = _recorded_calls(calls, prefix=["gh", "api", "--method", "DELETE"])[0] + final_issue = _recorded_calls( + calls, prefix=["gh", "issue", "view"], contains="state,labels" + )[-1] + issue_edit = _recorded_calls(calls, prefix=["gh", "issue", "edit"])[0] + assert len(listed_calls) == 2 and len(install_calls) == 2 and len(pr_reads) == 3 + assert ( + calls.index(fleet_call) + < calls.index(listed_calls[0]) + < calls.index(file_call) + < calls.index(fetch_call) + < calls.index(oid_call) + < calls.index(checkout_call) + < calls.index(install_calls[1]) + < calls.index(checks[1]) + < calls.index(workflow_call) + < calls.index(runs_call) + < calls.index(jobs_call) + < calls.index(pr_reads[1]) + < calls.index(listed_calls[1]) + < calls.index(close) + < calls.index(branch_delete) + < calls.index(final_issue) + < calls.index(issue_edit) + ) + + ingredient_args = [ + fleet_call.argv[index + 1] + for index, arg in enumerate(fleet_call.argv[:-1]) + if arg == "-i" + ] + assert ingredient_args.count(f"issue_url={_SANDBOX_IMPLEMENTATION_ISSUE}") == 1 + assert ( + ingredient_args.count("task=Add a greeting\n\nImplement the requested behavior.") == 1 + ) + ingredient_names = {key for key, _value in test.ingredients} + assert "issue_url" not in ingredient_names and "task" not in ingredient_names + assert fleet_call.timeout == test.timeout_sec + e2e_catalog.HARNESS_GRACE_SEC + assert fleet_call.cwd == harness.SANDBOX_CLONE + assert [call.input for call in calls if call.input] == [SANDBOX_TOKEN] + assert all(SANDBOX_TOKEN not in " ".join(call.argv) for call in calls) + assert all(call.env == config_call.env for call in calls) + + trace = out / "session-events.jsonl" + log_root = harness.e2e_sessions.default_log_root(home) + envelope = json.loads((out / "envelope.json").read_text(encoding="utf-8")) + assert observed["ownership"] == [ + (trace, envelope, log_root, _SANDBOX_IMPLEMENTATION_ISSUE, SANDBOX, "in-progress") + ] + assert observed["summaries"] == [(trace, envelope, log_root)] + capacity = json.loads((out / "catalog-capacity.json").read_text(encoding="utf-8")) + assert capacity == { + "peak_sessions": test.peak_sessions, + "peak_sessions_evidence": test.peak_sessions_evidence, + } + settings = json.loads((home / ".claude" / "settings.json").read_text(encoding="utf-8")) + assert settings["env"] == _EXPECTED_SETTINGS_ENV + assert all( + len(settings["hooks"][event]) == 1 + for event in harness.e2e_sessions.LIFECYCLE_HOOK_MATCHERS + ) + + assert close.argv == ["gh", "pr", "close", "8", "--repo", SANDBOX] + assert not any(call.argv[:4] == ["gh", "pr", "close", "9"] for call in calls) + assert branch_delete.argv[:4] == ["gh", "api", "--method", "DELETE"] + assert branch_delete.argv[4].endswith("codex%2Fissue-5233") + assert any( + call.argv[:3] == ["gh", "issue", "edit"] + and call.argv[-2:] == ["--remove-label", "in-progress"] + for call in calls + ) + + @pytest.mark.parametrize( + ("problem", "expected"), + [ + ( + "malformed", + "preflight: seed: expected the configured OPEN sandbox issue with valid metadata", + ), + ( + "closed", + "preflight: seed: expected the configured OPEN sandbox issue with valid metadata", + ), + ("claimed", "preflight: seed: issue is already claimed"), + ], + ) + def test_invalid_seed_stops_before_baseline_or_fleet( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + problem: str, + expected: str, + ) -> None: + runner, state = _pipeline_runner() + if problem == "malformed": + state["seed"].pop("title") + elif problem == "closed": + state["seed"]["state"] = "CLOSED" + else: + state["seed"]["labels"] = [{"name": "in-progress"}] + + failures, result, _out, _home, _catalog, observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert failures == [expected] + assert result["outcome"] == "failed" + assert not any(call.argv[:2] == ["task", "install-worktree"] for call in runner.calls) + assert not any(call.argv[:2] == ["autoskillit", "fleet"] for call in runner.calls) + assert not any(call.argv[:3] == ["gh", "pr", "list"] for call in runner.calls) + assert observed["ownership"] == observed["summaries"] == [] + + def test_missing_seed_url_stops_before_github_issue_lookup( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + runner, state = _pipeline_runner() + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, + runner, + state, + monkeypatch, + env=_env(E2E_IMPLEMENTATION_ISSUE_URL=""), + ) + + assert failures == ["preflight: seed: missing or noncanonical sandbox issue URL"] + assert result["outcome"] == "failed" + assert not any(call.argv[:3] == ["gh", "issue", "view"] for call in runner.calls) + assert not any(call.argv[:3] == ["autoskillit", "fleet", "run"] for call in runner.calls) + + @pytest.mark.parametrize("wrong_base", ["sandbox", "effective-config"]) + def test_seed_requires_the_main_branch_in_repository_and_config( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, wrong_base: str + ) -> None: + runner, state = _pipeline_runner() + if wrong_base == "sandbox": + state["default_branch"] = "develop" + else: + state["config"]["branching"]["default_base_branch"] = "trunk" + + failures, _result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert any( + "sandbox and effective configured base branch must both be main" in item + for item in failures + ) + assert not any(call.argv[:2] == ["task", "install-worktree"] for call in runner.calls) + assert not any(call.argv[:3] == ["autoskillit", "fleet", "run"] for call in runner.calls) + + def test_linked_stable_finding_is_recorded_and_skips_pr_assertions( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + runner, state = _pipeline_runner() + state["fleet_envelope"] = _retry_exhausted_envelope() + _configure_failed_pipeline_state(state) + expected = { + "check": "fleet_envelope", + "message": "retry_exhausted: known orchestration failure", + "issue": _UPSTREAM_EXPECTED_IMPLEMENTATION_ISSUE, + } + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch, expected_failures=[expected] + ) + + assert failures == [] + assert result["outcome"] == "expected_failure" + assert result["passed"] is False + assert result["expected_findings"] == [expected] + assert not any(call.argv[:2] == ["git", "fetch"] for call in runner.calls) + assert not any( + "/pulls/" in " ".join(call.argv) and call.argv[0:2] == ["gh", "api"] + for call in runner.calls + ) + assert not any("actions/workflows" in " ".join(call.argv) for call in runner.calls) + + @pytest.mark.parametrize("envelope_kind", ["stale", "fatal"]) + def test_stale_or_fatal_pipeline_failure_cannot_be_allowlisted( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, envelope_kind: str + ) -> None: + runner, state = _pipeline_runner() + expected = { + "check": "fleet_envelope", + "message": "retry_exhausted: known orchestration failure", + "issue": _UPSTREAM_EXPECTED_IMPLEMENTATION_ISSUE, + } + _configure_failed_pipeline_state(state) + if envelope_kind == "fatal": + state["fleet_envelope"] = { + "success": False, + "dispatch_status": "failed", + "dispatch_id": "dispatch-5233", + "dispatched_session_id": "session-5233", + "l3_payload": {}, + } + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch, expected_failures=[expected] + ) + + assert result["outcome"] == "failed" + assert result["expected_findings"] == [] + if envelope_kind == "stale": + assert any("expected recipe finding absent" in failure for failure in failures) + else: + assert any( + "unsuccessful envelope has no specific failure reason" in f for f in failures + ) + assert any("expected recipe finding absent" in failure for failure in failures) + assert not any(call.argv[:2] == ["git", "fetch"] for call in runner.calls) + + @pytest.mark.parametrize( + ("problem", "expected"), + [ + ("pr", "implementation PR: base branch must be main"), + ("body", "implementation PR: body does not reference the canonical seed URL"), + ("paths", "implementation PR: missing non-removed changes under tests/"), + ("ci", "sandbox CI: CI run and test job must succeed"), + ("head", "implementation PR: fetched head differs from captured head"), + ("ci-job", "sandbox CI: CI run and test job must succeed"), + ("test", "pr-test: exit 1: tests failed"), + ], + ) + def test_pipeline_rejects_unverified_pr_or_validation_failure( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + problem: str, + expected: str, + ) -> None: + runner, state = _pipeline_runner() + pr = state["pr_details"][8] + if problem == "pr": + pr["base"]["ref"] = "develop" + elif problem == "body": + pr["body"] = "No linked issue" + elif problem == "paths": + state["files"] = state["files"][:1] + elif problem == "ci": + state["workflow_conclusion"] = "failure" + elif problem == "ci-job": + state["job_conclusion"] = "failure" + elif problem == "head": + state["fetch_oid"] = "fetched-a-different-head" + else: + state["pr_test_returncode"] = 1 + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert result["outcome"] == "failed" + assert any(expected in failure for failure in failures) + if problem in {"ci", "ci-job"}: + assert not any(failure.startswith("harness error:") for failure in failures) + assert any(call.argv[:3] == ["gh", "issue", "edit"] for call in runner.calls) + + def test_implementation_pr_reports_each_failed_precondition(self, tmp_path: Path) -> None: + test = _pipeline_catalog().get("implementation") + pr = _pipeline_pr() + pr.update( + state="closed", + merged=True, + merged_at="2026-10-01T00:00:00Z", + auto_merge={"enabled": True}, + ) + pr["base"]["ref"] = "develop" + pr["base"]["repo"] = {"full_name": "other/repo"} + pr["head"]["sha"] = "" + pr["head"]["repo"] = {"full_name": "other/repo"} + pr["_listed_head"] = _PIPELINE_HEAD + + failures = harness.validate_implementation_pr( + test, + [pr], + {"url": _SANDBOX_IMPLEMENTATION_ISSUE}, + SANDBOX, + tmp_path, + _env(), + FakeRunner(lambda argv: None), + ) + + assert failures == [ + "implementation PR: state must be open", + "implementation PR: merged must be false", + "implementation PR: merged_at must be empty", + "implementation PR: auto-merge must be disabled", + "implementation PR: base branch must be main", + "implementation PR: head SHA must be a non-empty string", + "implementation PR: base repository must match the sandbox", + "implementation PR: head repository must match the sandbox", + ] + + def test_listed_pr_head_must_match_rest_head_before_validation( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + runner, state = _pipeline_runner() + state["listed_prs"][0]["headRefOid"] = "stale-listed-head" + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert result["outcome"] == "failed" + assert "implementation PR: REST head differs from captured headRefOid" in failures + assert not any("/files?" in " ".join(call.argv) for call in runner.calls) + assert not any("actions/workflows" in " ".join(call.argv) for call in runner.calls) + + @pytest.mark.parametrize( + ("problem", "expected"), + [ + ("coverage", "session measurement: incomplete lifecycle coverage"), + ("capacity", "session measurement: observed peak exceeds catalog capacity"), + ], + ) + def test_session_evidence_must_be_complete_and_within_catalog_capacity( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + problem: str, + expected: str, + ) -> None: + runner, state = _pipeline_runner() + if problem == "coverage": + state["summary"]["coverage"]["complete"] = False + else: + state["summary"]["peak_sessions"] = 7 + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert result["outcome"] == "failed" + assert any(expected in failure for failure in failures) + + def test_ownership_violations_fail_the_pipeline_result( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + runner, state = _pipeline_runner() + state["proof"]["violations"] = ["transcript evidence is incomplete"] + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert result["outcome"] == "failed" + assert "ownership: transcript evidence is incomplete" in failures + + def test_cleanup_errors_do_not_prevent_remaining_owned_cleanup( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + runner, state = _pipeline_runner() + state["cleanup_errors"] = { + "pr_close": RuntimeError("close failed"), + "branch_delete": RuntimeError("branch delete failed"), + } + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert result["outcome"] == "failed" + assert any("cleanup PR: close failed" in failure for failure in failures) + assert any("cleanup branch: branch delete failed" in failure for failure in failures) + assert any(call.argv[:3] == ["gh", "issue", "edit"] for call in runner.calls) + + def test_fleet_exception_and_cleanup_exception_are_both_preserved( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + runner, state = _pipeline_runner() + state["fleet_exception"] = RuntimeError("fleet launch crashed") + state["cleanup_errors"] = {"branch_delete": RuntimeError("branch delete failed")} + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert result["outcome"] == "failed" + assert any( + "harness error: RuntimeError: fleet launch crashed" in failure + and "Traceback (most recent call last)" in failure + for failure in failures + ) + assert any("cleanup branch: branch delete failed" in failure for failure in failures) + assert any(call.argv[:3] == ["gh", "issue", "edit"] for call in runner.calls) + + def test_branch_created_without_ownership_proof_fails_cleanup_and_is_preserved( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + runner, state = _pipeline_runner() + state["fleet_envelope"] = _retry_exhausted_envelope() + _configure_failed_pipeline_state(state) + state["published_refs"] = ["impl/leaked"] + + failures, _result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert ( + "cleanup: branch impl/leaked was created during this run without ownership proof" + in failures + ) + assert not any( + call.argv[:4] == ["gh", "api", "--method", "DELETE"] for call in runner.calls + ) + assert "impl/leaked" in state["head_refs"] + + def test_proven_branch_is_deleted_and_leaves_nothing_unexplained( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + runner, state = _pipeline_runner() + state["published_refs"] = [_PIPELINE_BRANCH] + original_list_head_refs = harness.list_head_refs + snapshots = {} + + def record_head_refs(repository, out, env, runner, stage): + refs = original_list_head_refs(repository, out, env, runner, stage) + snapshots[stage] = refs + return refs + + monkeypatch.setattr(harness, "list_head_refs", record_head_refs) + failures, _result, out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + delete = [ + "gh", + "api", + "--method", + "DELETE", + f"repos/{SANDBOX}/git/refs/heads/{quote(_PIPELINE_BRANCH, safe='')}", + ] + assert not any(failure.startswith("cleanup") for failure in failures) + assert any(call.argv == delete for call in runner.calls) + assert state["head_refs"] == ["main"] + assert snapshots["before"] == snapshots["after"] == frozenset({"main"}) + for stage in ("before", "after"): + assert json.loads( + (out / f"head-refs-{stage}.stdout.txt").read_text(encoding="utf-8") + ) == [[{"ref": "refs/heads/main"}]] + + def test_branch_present_before_the_run_is_never_reported( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + runner, state = _pipeline_runner() + state["fleet_envelope"] = _retry_exhausted_envelope() + _configure_failed_pipeline_state(state) + state["head_refs"] = ["main", "impl/old-leftover"] + + failures, _result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert not any("impl/old-leftover" in failure for failure in failures) + assert not any( + call.argv[:4] == ["gh", "api", "--method", "DELETE"] for call in runner.calls + ) + + @pytest.mark.parametrize( + ("call_number", "expected_prefix"), [(1, "preflight:"), (2, "cleanup refs:")] + ) + def test_ref_listing_failure_is_reported( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + call_number: int, + expected_prefix: str, + ) -> None: + runner, state = _pipeline_runner() + state["published_refs"] = [_PIPELINE_BRANCH] + state["refs_list_faults"] = {call_number: RuntimeError("refs unavailable")} + + failures, _result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + listing_failures = [failure for failure in failures if failure.startswith(expected_prefix)] + assert len(listing_failures) == 1 + assert "refs unavailable" in listing_failures[0] + if call_number == 1: + assert not any( + call.argv[:3] == ["autoskillit", "fleet", "run"] for call in runner.calls + ) + else: + delete = [ + "gh", + "api", + "--method", + "DELETE", + f"repos/{SANDBOX}/git/refs/heads/{quote(_PIPELINE_BRANCH, safe='')}", + ] + assert any(call.argv == delete for call in runner.calls) + assert any( + call.argv[:3] == ["gh", "issue", "edit"] + and call.argv[-2:] == ["--remove-label", "in-progress"] + for call in runner.calls + ) + + @pytest.mark.parametrize( + "response", + [[], [[{}]], [[{"ref": "refs/tags/v1"}]]], + ) + def test_malformed_ref_listing_fails_preflight( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, response: Any + ) -> None: + runner, state = _pipeline_runner() + state["refs_list_response"] = response + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert result["outcome"] == "failed" + assert sum(failure.startswith("preflight:") for failure in failures) == 1 + assert not any(call.argv[:3] == ["autoskillit", "fleet", "run"] for call in runner.calls) + + def test_proven_branch_whose_delete_fails_is_not_reported_as_unexplained( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + runner, state = _pipeline_runner() + state["published_refs"] = [_PIPELINE_BRANCH] + state["cleanup_errors"] = {"branch_delete": RuntimeError("branch delete failed")} + + failures, _result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert sum("cleanup branch: branch delete failed" in failure for failure in failures) == 1 + assert not any("without ownership proof" in failure for failure in failures) + + def test_failed_run_teardown_deletes_the_branch_proven_by_captured_evidence( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + provenance = load_provenance("run_38170521660_attempt_1") + repository = provenance["repository"] + issue_url = provenance["issue_url"] + claim_label = provenance["claim_label"] + branch = "e2e-implementation-fixture-normalize-whitespace/1-3" + runner, state = _pipeline_runner() + state["seed"] = { + **state["seed"], + "number": int(issue_url.rsplit("/", 1)[1]), + "url": issue_url, + } + state["config"]["github"]["in_progress_label"] = claim_label + state["listed_prs"] = [] + state["pr_details"] = {} + state["final_issue"] = {"state": "OPEN", "labels": []} + state["published_refs"] = [branch] + out = tmp_path / "out" + home = tmp_path / "home" + original_handler = runner.handler + + def materialize_before_fleet(argv): + if argv[:3] == ["autoskillit", "fleet", "run"]: + state["fleet_envelope"] = materialize_run( + "run_38170521660_attempt_1", + trace=out / "session-events.jsonl", + log_root=harness.e2e_sessions.default_log_root(home), + projects=home / ".claude" / "projects", + ) + return original_handler(argv) + + runner.handler = materialize_before_fleet + failures, result, out, _home, _catalog, _observed = _run_pipeline( + tmp_path, + runner, + state, + monkeypatch, + env=_env(E2E_IMPLEMENTATION_ISSUE_URL=issue_url), + real_measurement=True, + repository=repository, + ) + + delete = [ + "gh", + "api", + "--method", + "DELETE", + f"repos/{repository}/git/refs/heads/{quote(branch, safe='')}", + ] + assert any(call.argv == delete for call in runner.calls) + assert state["head_refs"] == ["main"] + assert not any(failure.startswith(("ownership:", "cleanup")) for failure in failures) + assert result["outcome"] == "failed" + assert ( + sum( + failure.startswith("fleet_envelope:") + and "resumable: fleet_l3_no_result_block" in failure + for failure in failures + ) + == 1 + ) + assert ( + sum( + failure.startswith("session measurement: incomplete lifecycle coverage") + for failure in failures + ) + == 1 + ) + assert json.loads((out / "ownership.json").read_text(encoding="utf-8"))["branches"] == [ + branch + ] + + class TestCanaryFlow: def test_canary_runs_one_claude_prompt(self, tmp_path: Path, monkeypatch) -> None: runner = FakeRunner(lambda argv: _completed(argv, _CANARY_REPLY)) @@ -2906,3 +4119,210 @@ def test_redact_command_reads_secrets_from_the_environment( assert (tmp_path / "upload" / "out" / "log.txt").read_text(encoding="utf-8") == ( "key [REDACTED]" ) + + +def test_delete_owned_branch_does_not_treat_ref_name_404_as_http_status() -> None: + runner = FakeRunner( + lambda argv: _completed( + argv, + returncode=1, + stderr=( + "gh: HTTP 500: Internal Server Error " + "(https://api.github.com/repos/owner/repo/git/refs/heads/release%2F404-fix)" + ), + ) + ) + + with pytest.raises(ValueError, match="HTTP 500"): + harness.delete_owned_branch(SANDBOX, "release/404-fix", _env(), runner) + + assert len(runner.calls) == 1 + assert runner.calls[0].argv[:2] == ["gh", "api"] + + +@pytest.mark.parametrize( + "stderr", + ["gh: HTTP 404: Not Found", "gh: Not Found (HTTP 404)"], +) +def test_delete_owned_branch_treats_gh_404_formats_as_missing_ref(stderr: str) -> None: + runner = FakeRunner(lambda argv: _completed(argv, returncode=1, stderr=stderr)) + + harness.delete_owned_branch(SANDBOX, "feature/already-deleted", _env(), runner) + + assert len(runner.calls) == 1 + assert runner.calls[0].argv[:2] == ["gh", "api"] + + +@pytest.mark.parametrize( + ("workflow_present", "workflow_status", "expected"), + [ + ( + True, + "in_progress", + "sandbox CI: matching run did not complete before the deadline", + ), + ( + False, + "completed", + "sandbox CI: no run associated with the exact PR head before the deadline", + ), + ], +) +def test_sandbox_ci_deadline_distinguishes_pending_and_missing_runs( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + workflow_present: bool, + workflow_status: str, + expected: str, +) -> None: + runner, state = _pipeline_runner() + state["workflow_present"] = workflow_present + state["workflow_status"] = workflow_status + out = tmp_path / "ci" + out.mkdir() + monkeypatch.setattr(harness, "_CI_POLL_TIMEOUT_SEC", 0) + monkeypatch.setattr(harness.time, "sleep", lambda _seconds: pytest.fail("unexpected sleep")) + + with pytest.raises(ValueError) as error: + harness.validate_sandbox_ci(SANDBOX, 8, _PIPELINE_HEAD, out, _env(), runner) + + assert str(error.value) == expected + assert len(_recorded_calls(runner.calls, contains="/actions/workflows/1/runs?")) == 1 + + +def test_malformed_effective_config_reports_the_section(tmp_path: Path, monkeypatch) -> None: + runner, state = _pipeline_runner() + state["config"] = {"github": [], "branching": {"default_base_branch": "main"}} + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert failures == ["preflight: seed: effective config github section must be an object"] + assert result["outcome"] == "failed" + assert not any(call.argv[:3] == ["gh", "issue", "view"] for call in runner.calls) + assert not any(call.argv[:3] == ["autoskillit", "fleet", "run"] for call in runner.calls) + + +def test_malformed_final_issue_labels_report_cleanup_shape(tmp_path: Path, monkeypatch) -> None: + runner, state = _pipeline_runner() + state["final_issue"] = {"state": "OPEN", "labels": None} + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert result["outcome"] == "failed" + assert any( + "cleanup claim: cleanup: final seed issue labels must be an array of named objects" + in failure + for failure in failures + ) + assert not any(call.argv[:3] == ["gh", "issue", "edit"] for call in runner.calls) + + +@pytest.mark.parametrize( + ("document", "expected"), + [ + ("[]", "settings.json must be an object"), + ('"agent-shaped string"', "settings.json must be an object"), + ('{"hooks": []}', "settings.json hooks must be an object"), + ( + '{"hooks": {"SessionStart": "invalid"}}', + "settings.json hooks.SessionStart must be an array", + ), + ], +) +def test_session_hook_install_rejects_malformed_settings_without_rewriting( + tmp_path: Path, + document: str, + expected: str, +) -> None: + settings_path = tmp_path / "home" / ".claude" / "settings.json" + settings_path.parent.mkdir(parents=True, exist_ok=True) + settings_path.write_text(document, encoding="utf-8") + original = settings_path.read_bytes() + + with pytest.raises(ValueError) as error: + harness.install_session_hooks(tmp_path / "home", tmp_path / "events.jsonl") + + assert expected in str(error.value) + assert settings_path.read_bytes() == original + + +def test_branch_lookup_failure_still_releases_claim(tmp_path: Path, monkeypatch) -> None: + runner, state = _pipeline_runner() + state["cleanup_errors"] = {"branch_get": RuntimeError("branch lookup failed")} + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert result["outcome"] == "failed" + assert any("cleanup branch: branch lookup failed" in failure for failure in failures) + assert any(call.argv[:3] == ["gh", "issue", "edit"] for call in runner.calls) + assert not any("--method" in call.argv and "DELETE" in call.argv for call in runner.calls) + + +def test_pipeline_setup_response_routes_seed_with_extra_json_fields() -> None: + _runner, state = _pipeline_runner() + argv = [ + "gh", + "issue", + "view", + _SANDBOX_IMPLEMENTATION_ISSUE, + "--json", + "number,url,state,title,body,labels,assignees", + ] + + response = _pipeline_setup_response(argv, state) + + assert response is not None + assert json.loads(response.stdout) == state["seed"] + + +def test_fleet_and_collectors_share_the_exported_log_root(tmp_path: Path, monkeypatch) -> None: + runner, state = _pipeline_runner() + custom_root = tmp_path / "custom-session-logs" + monkeypatch.setattr(harness.e2e_sessions, "default_log_root", lambda _home: custom_root) + failures, result, _out, _home, _catalog, observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + assert failures == [] + assert result["outcome"] == "passed" + fleet = next(call for call in runner.calls if call.argv[:3] == ["autoskillit", "fleet", "run"]) + assert fleet.env["AUTOSKILLIT_LOG_DIR"] == str(custom_root) + assert observed["summaries"][0][2] == custom_root + assert all(call[2] == custom_root for call in observed["ownership"]) + + +def test_match_recipe_findings_preserves_the_input_failures_list() -> None: + failures = ["existing failure"] + + result, matched = harness.match_recipe_findings( + failures, + [{"check": "new_check", "message": "unexpected finding"}], + [], + ) + + assert failures == ["existing failure"] + assert result == ["existing failure", "new_check: unexpected finding"] + assert matched == [] + + +def test_json_call_reports_parse_location_and_preserves_output_evidence(tmp_path: Path) -> None: + runner = FakeRunner( + lambda argv: _completed(argv, stdout="not-json", stderr="diagnostic warning") + ) + evidence = tmp_path / "invalid-response" + + with pytest.raises(ValueError) as error: + harness._json_call(["gh", "api", "repos/owner/repo"], _env(), runner, evidence=evidence) + + assert ( + str(error.value) + == "gh api repos/owner/repo: invalid JSON at line 1 column 1: Expecting value" + ) + assert isinstance(error.value.__cause__, json.JSONDecodeError) + assert evidence.with_suffix(".stdout.txt").read_text(encoding="utf-8") == "not-json" + assert evidence.with_suffix(".stderr.txt").read_text(encoding="utf-8") == "diagnostic warning" diff --git a/tests/infra/test_e2e_select.py b/tests/infra/test_e2e_select.py index ecec41bf95..fa4051157f 100644 --- a/tests/infra/test_e2e_select.py +++ b/tests/infra/test_e2e_select.py @@ -10,19 +10,26 @@ import pytest -from tests.infra._complexity_helpers import load_check_script +from tests._script_loader import load_script pytestmark = [pytest.mark.layer("infra"), pytest.mark.small] REPO_ROOT = Path(__file__).resolve().parents[2] _SELECT_SCRIPT = REPO_ROOT / "scripts" / "e2e" / "e2e_select.py" -select = load_check_script("_autoskillit_e2e_select", _SELECT_SCRIPT) +select = load_script("_autoskillit_e2e_select", _SELECT_SCRIPT) e2e_catalog = select.e2e_catalog REPOSITORY = "TalonT-Org/AutoSkillit" SHA_ZERO = "0" * 40 SYNTHETIC_SHAS = [hashlib.sha1(str(index).encode()).hexdigest() for index in range(2000)] +_IMPLEMENTATION_PATHS = ( + "src/autoskillit/recipes/", + "src/autoskillit/skills/", + "src/autoskillit/skills_extended/", + "src/autoskillit/server/", + "src/autoskillit/execution/", +) def _facts( @@ -188,6 +195,44 @@ def test_deleted_head_repository_is_not_selected(self) -> None: assert pr.head_repository is None assert not selection.selected + @pytest.mark.parametrize( + ("changed_files", "additions", "deletions"), + [(10, 0, 0), (1, 300, 0), (1, 100, 200)], + ) + def test_large_change_with_matching_root_selects_pipeline( + self, changed_files: int, additions: int, deletions: int + ) -> None: + pr = _facts( + head_sha=_sha_with_sample(5230, sampled=True), + changed_files=changed_files, + additions=additions, + deletions=deletions, + ) + selection = select.select_for_pull_request( + pr, + ["src/autoskillit/recipes/changed.py"], + _catalog(_pipeline_entry()), + REPOSITORY, + ) + assert selection.tests == ("implementation",) + + def test_label_does_not_make_a_small_change_pipeline_eligible(self) -> None: + pr = _facts(changed_files=1, additions=10, labels=("e2e",)) + selection = select.select_for_pull_request( + pr, + ["src/autoskillit/recipes/changed.py"], + _catalog(_pipeline_entry()), + REPOSITORY, + ) + assert not selection.selected + + def test_pipeline_is_not_an_unrelated_fallback(self) -> None: + pr = _facts(head_sha=_sha_with_sample(5230, sampled=True), changed_files=10) + selection = select.select_for_pull_request( + pr, ["README.md"], _catalog(_pipeline_entry()), REPOSITORY + ) + assert not selection.selected + class TestPickTests: _SEED = f"5230:{SHA_ZERO}" @@ -228,6 +273,25 @@ def test_star_matches_across_directories(self) -> None: catalog = _catalog(_entry("deep", ("src/*",)), _entry("other", ("docs/*",))) assert select.pick_tests(catalog, ["src/a/b/c.py"], self._SEED) == ("deep",) + @pytest.mark.parametrize("root", _IMPLEMENTATION_PATHS) + def test_pipeline_trigger_roots_match(self, root: str) -> None: + assert select.pick_tests( + _catalog(_pipeline_entry()), + [f"{root}changed.py"], + self._SEED, + large_change=True, + ) == ("implementation",) + + def test_at_most_one_pipeline_is_selected(self) -> None: + catalog = _catalog( + _pipeline_entry("pipeline-a", trigger_paths=["src/*"]), + _pipeline_entry("pipeline-b", trigger_paths=["src/*"]), + _entry("ordinary", ("src/*",)), + ) + picked = select.pick_tests(catalog, ["src/changed.py"], self._SEED, large_change=True) + assert len(picked) == min(select.MAX_TESTS, 2) + assert sum(catalog.get(name).pipeline for name in picked) == 1 + class TestSelectForEvent: def test_merge_group_is_never_selected(self) -> None: @@ -301,6 +365,18 @@ def test_matrix_includes_kind_from_each_catalog_entry() -> None: "ingredients": {"task": "Add a greeting"}, "expected_pull_request_state": "merged", } +_RECIPE_EXPECTED_FAILURE = { + "check": "implementation-check", + "message": "expected implementation failure", + "issue": "https://github.com/TalonT-Org/AutoSkillit/issues/123", +} +_IMPLEMENTATION_INGREDIENTS = { + "source_dir": "/workspace/sandbox", + "open_pr": "true", + "auto_merge": "false", + "arch_lenses": "false", + "audit_impl": "true", +} _MAX_TIMEOUT_SEC = ( e2e_catalog.MAX_JOB_MINUTES - e2e_catalog.JOB_OVERHEAD_MINUTES ) * 60 - e2e_catalog.HARNESS_GRACE_SEC @@ -311,6 +387,28 @@ def _recipe_entry_without(field: str) -> dict: return _entry("impl", kind="recipe", **fields) +def _pipeline_entry(name: str = "implementation", **overrides) -> dict: + entry = _entry( + name, + tuple(f"{path}*" for path in _IMPLEMENTATION_PATHS), + kind="recipe", + peak_sessions=8, + timeout_sec=7200, + recipe="implementation", + ingredients=dict(_IMPLEMENTATION_INGREDIENTS), + expected_pull_request_state="open", + pipeline=True, + issue_url_env="E2E_IMPLEMENTATION_ISSUE_URL", + required_changed_paths=["sandbox/", "tests/"], + test_command=["task", "test-check"], + peak_sessions_evidence=( + "configured capacity limit of eight concurrent sessions; no measured peak supplied" + ), + ) + entry.update(overrides) + return entry + + _INVALID_CATALOGS = { "duplicate-names": _raw_catalog(_entry("a"), _entry("a")), "bad-name": _raw_catalog(_entry("Bad_Name")), @@ -352,6 +450,9 @@ def _recipe_entry_without(field: str) -> dict: **_RECIPE_FIELDS, ) ), + "expected-failures-on-recipe": _raw_catalog( + _pipeline_entry(expected_failures=[_EXPECTED_FAILURE]) + ), "expected-failures-missing-field": _raw_catalog( _clean_install_entry( expected_failures=[ @@ -465,6 +566,7 @@ def test_recipe_entry_parses(self) -> None: "impl", kind="recipe", recipe_fixture="sandbox-smoke.yaml", + expected_failures=[{"severity": "error", **_RECIPE_EXPECTED_FAILURE}], **_RECIPE_FIELDS, ) ) @@ -473,6 +575,12 @@ def test_recipe_entry_parses(self) -> None: assert test.ingredients == (("task", "Add a greeting"),) assert test.expected_pull_request_state == "merged" assert test.recipe_fixture == "sandbox-smoke.yaml" + assert test.pipeline is False + assert test.issue_url_env is None + assert test.required_changed_paths == () + assert test.test_command == () + assert test.peak_sessions_evidence is None + assert test.expected_failures == ({"severity": "error", **_RECIPE_EXPECTED_FAILURE},) def test_recipe_without_fixture_defaults_to_none(self) -> None: test = _catalog(_entry("impl", kind="recipe", **_RECIPE_FIELDS)).get("impl") @@ -524,3 +632,140 @@ def test_unknown_test_name_raises_catalog_error(self) -> None: def test_invalid_catalog_raises(self, raw: dict) -> None: with pytest.raises(e2e_catalog.CatalogError): e2e_catalog.parse_catalog(raw) + + +class TestPipelineCatalog: + def test_shipped_implementation_targets_sandbox_source_and_tests_without_a_measurement( + self, + ) -> None: + test = e2e_catalog.load_catalog().get("implementation") + assert test.required_changed_paths == ("sandbox/", "tests/") + assert test.trigger_paths == tuple(f"{root}*" for root in _IMPLEMENTATION_PATHS) + assert test.peak_sessions == 8 + assert test.peak_sessions_evidence is None + assert test.timeout_sec == 7200 + assert dict(test.ingredients) == _IMPLEMENTATION_INGREDIENTS + + def test_pipeline_recipe_parses_its_contract_and_expected_failures(self) -> None: + rows = [ + _RECIPE_EXPECTED_FAILURE, + {**_RECIPE_EXPECTED_FAILURE, "check": "another-check"}, + ] + test = _catalog(_pipeline_entry(expected_failures=rows)).get("implementation") + + assert test.pipeline is True + assert test.issue_url_env == "E2E_IMPLEMENTATION_ISSUE_URL" + assert test.required_changed_paths == ("sandbox/", "tests/") + assert test.test_command == ("task", "test-check") + assert test.peak_sessions_evidence == ( + "configured capacity limit of eight concurrent sessions; no measured peak supplied" + ) + assert test.expected_failures == tuple(rows) + with pytest.raises(TypeError): + cast(MutableMapping[str, str], test.expected_failures[0])["check"] = "changed" + + @pytest.mark.parametrize("field", ["required_changed_paths", "test_command"]) + def test_pipeline_requires_changed_paths_and_test_command(self, field: str) -> None: + entry = _pipeline_entry() + entry.pop(field) + with pytest.raises(e2e_catalog.CatalogError, match="missing pipeline keys"): + _catalog(entry) + + def test_pipeline_commissioning_provenance_is_optional(self) -> None: + entry = _pipeline_entry() + entry.pop("peak_sessions_evidence") + test = _catalog(entry).get("implementation") + assert test.pipeline is True + assert test.peak_sessions_evidence is None + + def test_pipeline_rejects_unknown_keys(self) -> None: + with pytest.raises(e2e_catalog.CatalogError, match="unknown keys"): + _catalog(_pipeline_entry(unrecognized=True)) + + @pytest.mark.parametrize("ingredient", ["issue_url", "task"]) + def test_issue_url_env_conflicts_with_recipe_ingredient(self, ingredient: str) -> None: + ingredients = {**_IMPLEMENTATION_INGREDIENTS, ingredient: "value"} + with pytest.raises(e2e_catalog.CatalogError, match="cannot contain issue_url or task"): + _catalog(_pipeline_entry(ingredients=ingredients)) + + def test_issue_url_env_is_allowed_without_pipeline_true(self) -> None: + entry = _pipeline_entry(pipeline=False) + for field in ("required_changed_paths", "test_command", "peak_sessions_evidence"): + entry.pop(field) + test = _catalog(entry).get("implementation") + assert test.pipeline is False + assert test.issue_url_env == "E2E_IMPLEMENTATION_ISSUE_URL" + assert test.required_changed_paths == () + assert test.test_command == () + assert test.peak_sessions_evidence is None + + @pytest.mark.parametrize( + "field", ["required_changed_paths", "test_command", "peak_sessions_evidence"] + ) + def test_pipeline_only_fields_require_pipeline_true(self, field: str) -> None: + entry = _pipeline_entry(pipeline=False) + for other_field in ( + "required_changed_paths", + "test_command", + "peak_sessions_evidence", + ): + if other_field != field: + entry.pop(other_field) + with pytest.raises(e2e_catalog.CatalogError, match="pipeline-only keys"): + _catalog(entry) + + @pytest.mark.parametrize("value", [None, "true", 1]) + def test_pipeline_flag_must_be_boolean(self, value: object) -> None: + with pytest.raises(e2e_catalog.CatalogError, match="pipeline must be a boolean"): + _catalog(_pipeline_entry(pipeline=value)) + + @pytest.mark.parametrize("value", ["", " ", 123]) + def test_issue_url_environment_name_must_be_nonempty(self, value: object) -> None: + with pytest.raises(e2e_catalog.CatalogError, match="issue_url_env"): + _catalog(_pipeline_entry(issue_url_env=value)) + + @pytest.mark.parametrize( + "prefix", + ["", " ", 123, "/absolute/path", "../parent", "src/../other", "C:\\absolute"], + ) + def test_required_changed_paths_must_be_relative_prefixes(self, prefix: object) -> None: + with pytest.raises(e2e_catalog.CatalogError): + _catalog(_pipeline_entry(required_changed_paths=[prefix])) + + def test_required_changed_paths_error_identifies_invalid_list_index(self) -> None: + with pytest.raises(e2e_catalog.CatalogError, match=r"required_changed_paths\[1\]"): + _catalog(_pipeline_entry(required_changed_paths=["sandbox/", "../parent"])) + + @pytest.mark.parametrize("command", [[], [""], [1]]) + def test_pipeline_test_command_must_be_nonempty_argv(self, command: object) -> None: + with pytest.raises(e2e_catalog.CatalogError, match="test_command"): + _catalog(_pipeline_entry(test_command=command)) + + def test_pipeline_test_command_error_identifies_invalid_list_index(self) -> None: + with pytest.raises(e2e_catalog.CatalogError, match=r"test_command\[1\]"): + _catalog(_pipeline_entry(test_command=["task", ""])) + + @pytest.mark.parametrize("evidence", ["", " ", 123]) + def test_pipeline_peak_sessions_evidence_must_be_nonempty(self, evidence: object) -> None: + with pytest.raises(e2e_catalog.CatalogError, match="peak_sessions_evidence"): + _catalog(_pipeline_entry(peak_sessions_evidence=evidence)) + + @pytest.mark.parametrize( + "row", + [ + {key: value for key, value in _RECIPE_EXPECTED_FAILURE.items() if key != "check"}, + {**_RECIPE_EXPECTED_FAILURE, "severity": "warning"}, + {**_RECIPE_EXPECTED_FAILURE, "issue": "https://github.com/other/repo/issues/123"}, + ], + ) + def test_recipe_expected_failures_have_exact_canonical_rows(self, row: dict) -> None: + with pytest.raises(e2e_catalog.CatalogError): + _catalog(_pipeline_entry(expected_failures=[row])) + + def test_recipe_expected_failures_reject_duplicate_check_and_message(self) -> None: + duplicate = { + **_RECIPE_EXPECTED_FAILURE, + "issue": "https://github.com/TalonT-Org/AutoSkillit/issues/124", + } + with pytest.raises(e2e_catalog.CatalogError, match="duplicate diagnostic"): + _catalog(_pipeline_entry(expected_failures=[_RECIPE_EXPECTED_FAILURE, duplicate])) diff --git a/tests/infra/test_e2e_select_cli.py b/tests/infra/test_e2e_select_cli.py index 25a7f2c1e2..0f0fb0f317 100644 --- a/tests/infra/test_e2e_select_cli.py +++ b/tests/infra/test_e2e_select_cli.py @@ -24,6 +24,9 @@ _HEADLESS_SMOKE_MATRIX = ( '{"include":[{"test":"headless-smoke","kind":"recipe","timeout_minutes":47}]}' ) +_IMPLEMENTATION_MATRIX = ( + '{"include":[{"test":"implementation","kind":"recipe","timeout_minutes":152}]}' +) def _run( @@ -112,6 +115,19 @@ def test_small_unlabelled_pull_request_selects_nothing(tmp_path: Path) -> None: assert outputs["matrix"] == '{"include":[]}' +def test_large_pull_request_selects_implementation_pipeline(tmp_path: Path) -> None: + changed = tmp_path / "changed-paths.txt" + changed.write_text("src/autoskillit/recipes/changed.py\n", encoding="utf-8") + payload = _pull_request(changed_files=10, additions=0, labels=("e2e",)) + + result = _select(tmp_path, changed, "pull_request", payload) + + assert result.returncode == 0, result.stderr + outputs = _outputs(result.stdout) + assert outputs["selected"] == "true" + assert outputs["matrix"] == _IMPLEMENTATION_MATRIX + + def test_merge_group_never_reads_the_changed_paths_file(tmp_path: Path) -> None: result = _select(tmp_path, tmp_path / "missing.txt", "merge_group", {"merge_group": {}}) assert result.returncode == 0, result.stderr @@ -151,6 +167,19 @@ def test_dispatch_selects_both_named_tests_in_request_order(tmp_path: Path) -> N ) +def test_dispatch_selects_named_pipeline_without_pull_request_eligibility(tmp_path: Path) -> None: + result = _select( + tmp_path, + tmp_path / "missing.txt", + "workflow_dispatch", + {"inputs": {"tests": "implementation"}}, + ) + assert result.returncode == 0, result.stderr + outputs = _outputs(result.stdout) + assert outputs["selected"] == "true" + assert outputs["matrix"] == _IMPLEMENTATION_MATRIX + + def test_dispatch_of_an_unknown_test_fails(tmp_path: Path) -> None: payload = {"inputs": {"tests": "nope"}} result = _select(tmp_path, tmp_path / "missing.txt", "workflow_dispatch", payload) diff --git a/tests/infra/test_e2e_sessions.py b/tests/infra/test_e2e_sessions.py new file mode 100644 index 0000000000..9b2afb008e --- /dev/null +++ b/tests/infra/test_e2e_sessions.py @@ -0,0 +1,1580 @@ +"""Tests for the E2E session lifecycle and ownership evidence helper.""" + +from __future__ import annotations + +import json +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import IO, Any + +import pytest + +from autoskillit.core import DIRECT_PREFIX, PLUGIN_PREFIX +from tests._script_loader import load_script + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.medium] + +REPO_ROOT = Path(__file__).resolve().parents[2] +_SCRIPT = REPO_ROOT / "scripts" / "e2e" / "e2e_sessions.py" +sessions = load_script("_autoskillit_e2e_sessions", _SCRIPT) + +DISPATCH_ID = "dispatch-5233" +ROOT_ID = "session-root" +ISSUE_URL = "https://github.com/owner/repo/issues/5233" +REPOSITORY = "owner/repo" + + +def _jsonl(path: Path, rows: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + "".join(json.dumps(row, separators=(",", ":")) + "\n" for row in rows), + encoding="utf-8", + ) + + +def _session( + log_root: Path, + session_id: str, + *, + parent_session_id: str = "", + records: list[dict[str, Any]] | None = None, +) -> Path: + transcript = log_root / "transcripts" / f"{session_id}.jsonl" + _jsonl(transcript, records or []) + session_dir = log_root / "sessions" / session_id + session_dir.mkdir(parents=True, exist_ok=True) + (session_dir / "summary.json").write_text( + json.dumps( + { + "session_id": session_id, + "dispatch_id": DISPATCH_ID, + "execution_identity": {"parent_session_id": parent_session_id}, + } + ), + encoding="utf-8", + ) + return transcript + + +def _index(log_root: Path, entries: list[dict[str, Any]]) -> None: + _jsonl(log_root / "sessions.jsonl", entries) + + +def _index_rows(log_root: Path) -> list[dict[str, Any]]: + index_path = log_root / "sessions.jsonl" + return [json.loads(line) for line in index_path.read_text(encoding="utf-8").splitlines()] + + +def _entry( + session_id: str, + transcript: Path, + *, + parent_session_id: str = "", + caller_session_id: str = "", +) -> dict[str, Any]: + return { + "session_id": session_id, + "dir_name": session_id, + "dispatch_id": DISPATCH_ID, + "parent_session_id": parent_session_id, + "caller_session_id": caller_session_id, + "claude_code_log": str(transcript), + } + + +def _event( + name: str, + session_id: str, + timestamp: float, + *, + transcript: Path | None = None, + agent_id: str | None = None, + agent_type: str | None = None, + agent_transcript: Path | None = None, +) -> dict[str, Any]: + row: dict[str, Any] = { + "event": name, + "timestamp": timestamp, + "session_id": session_id, + } + if transcript is not None: + row["transcript_path"] = str(transcript) + if agent_id is not None: + row["agent_id"] = agent_id + if agent_type is not None: + row["agent_type"] = agent_type + if agent_transcript is not None: + row["agent_transcript_path"] = str(agent_transcript) + return row + + +def _trace(path: Path, events: list[dict[str, Any]]) -> None: + _jsonl(path, events) + + +def _envelope() -> dict[str, str]: + return {"dispatched_session_id": ROOT_ID, "dispatch_id": DISPATCH_ID} + + +def _base_sessions( + tmp_path: Path, + *, + child_id: str | None = None, + child_records: list[dict[str, Any]] | None = None, +) -> tuple[Path, Path, Path | None, Path]: + log_root = tmp_path / "logs" + root_transcript = _session(log_root, ROOT_ID) + entries = [_entry(ROOT_ID, root_transcript)] + child_transcript = None + if child_id is not None: + child_transcript = _session( + log_root, + child_id, + parent_session_id=ROOT_ID, + records=child_records, + ) + entries.append( + _entry( + child_id, + child_transcript, + parent_session_id=ROOT_ID, + caller_session_id=ROOT_ID, + ) + ) + _index(log_root, entries) + trace = tmp_path / "run" / "session-lifecycle.jsonl" + return log_root, trace, child_transcript, root_transcript + + +def _worker(log_root: Path, session_id: str) -> Path: + transcript = _session(log_root, session_id) + entries = _index_rows(log_root) + entries.append(_entry(session_id, transcript)) + _index(log_root, entries) + return transcript + + +def _trace_root_and_worker( + trace: Path, + log_root: Path, + worker_id: str, + worker_transcript: Path, +) -> None: + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event("SessionStart", worker_id, 2, transcript=worker_transcript), + _event("SessionEnd", worker_id, 3, transcript=worker_transcript), + _event("SessionEnd", ROOT_ID, 4, transcript=root_transcript), + ], + ) + + +def _tool_use( + tool_id: str, + name: str, + tool_input: dict[str, Any], + *, + prefix: str = PLUGIN_PREFIX, +) -> dict[str, Any]: + return { + "type": "assistant", + "message": { + "content": [ + { + "type": "tool_use", + "id": tool_id, + "name": f"{prefix}{name}", + "input": tool_input, + } + ] + }, + } + + +def _tool_result( + tool_id: str, + result: dict[str, Any], + *, + carrier: str = "mcp_meta", + is_error: bool = False, +) -> dict[str, Any]: + display = "tool reply\n\nsuccess: True" + if carrier == "mcp_meta" or carrier == "display_only": + content = display + elif carrier == "block_json": + content = json.dumps(result) + else: + raise ValueError(f"unsupported tool result carrier: {carrier}") + + row: dict[str, Any] = { + "type": "user", + "message": { + "content": [ + { + "type": "tool_result", + "tool_use_id": tool_id, + "content": content, + "is_error": is_error, + } + ] + }, + } + if carrier == "mcp_meta": + row["mcpMeta"] = { + "_meta": {"fastmcp": {"wrap_result": True}}, + "structuredContent": {"result": json.dumps(result)}, + } + return row + + +def _launch( + tool_id: str, + reply: dict[str, Any], + *, + tool: str = "run_skill", + is_error: bool = False, +) -> list[dict[str, Any]]: + return [ + _tool_use(tool_id, tool, {}), + _tool_result(tool_id, reply, is_error=is_error), + ] + + +def _root_ownership_proof( + tmp_path: Path, + records: list[dict[str, Any]], +) -> dict[str, Any]: + log_root, trace, _, root_transcript = _base_sessions(tmp_path) + _jsonl(root_transcript, records) + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event("SessionEnd", ROOT_ID, 2, transcript=root_transcript), + ], + ) + return sessions.collect_ownership(trace, _envelope(), log_root, ISSUE_URL, REPOSITORY) + + +def test_hook_capture_uses_event_alias_when_hook_event_name_is_null( + tmp_path: Path, +) -> None: + trace = tmp_path / "events.jsonl" + + captured = sessions.capture_hook_event( + trace, + { + "hook_event_name": None, + "event": "SessionStart", + "session_id": ROOT_ID, + }, + ) + + assert captured is True + [record] = [json.loads(line) for line in trace.read_text(encoding="utf-8").splitlines()] + assert record["event"] == "SessionStart" + + +def test_hook_capture_retries_short_appends( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + trace = tmp_path / "events.jsonl" + original_write = sessions.os.write + calls = 0 + + def short_once(fd: int, data: bytes | memoryview) -> int: + nonlocal calls + if calls == 0: + calls += 1 + return original_write(fd, data[:4]) + return original_write(fd, data) + + monkeypatch.setattr(sessions.os, "write", short_once) + + assert sessions.capture_hook_event( + trace, {"hook_event_name": "SessionStart", "session_id": ROOT_ID} + ) + [record] = [json.loads(line) for line in trace.read_text(encoding="utf-8").splitlines()] + assert record["event"] == "SessionStart" + assert record["session_id"] == ROOT_ID + + +def test_hook_capture_rolls_back_zero_write_before_next_event( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + trace = tmp_path / "events.jsonl" + original_write = sessions.os.write + calls = 0 + + def partial_then_zero(fd: int, data: bytes | memoryview) -> int: + nonlocal calls + calls += 1 + if calls == 1: + return original_write(fd, data[:4]) + return 0 + + monkeypatch.setattr(sessions.os, "write", partial_then_zero) + with pytest.raises(OSError, match="short lifecycle trace append"): + sessions.capture_hook_event( + trace, {"hook_event_name": "SessionStart", "session_id": "failed-session"} + ) + + assert trace.read_bytes() == b"" + monkeypatch.setattr(sessions.os, "write", original_write) + assert sessions.capture_hook_event( + trace, {"hook_event_name": "SessionStart", "session_id": ROOT_ID} + ) + [record] = [json.loads(line) for line in trace.read_text(encoding="utf-8").splitlines()] + assert record["session_id"] == ROOT_ID + + +def test_hook_capture_appends_concurrent_allow_listed_records(tmp_path: Path) -> None: + trace = tmp_path / "events.jsonl" + with ThreadPoolExecutor(max_workers=8) as pool: + results = list( + pool.map( + lambda index: sessions.capture_hook_event( + trace, + { + "hook_event_name": "SessionStart", + "session_id": f"session-{index}", + "source": "startup", + "transcript_path": f"/tmp/{index}.jsonl", + "prompt": "must not be persisted", + "environment": {"TOKEN": "must not be persisted"}, + }, + ), + range(32), + ) + ) + + records = [json.loads(line) for line in trace.read_text(encoding="utf-8").splitlines()] + assert all(results) + assert len(records) == 32 + assert {record["session_id"] for record in records} == { + f"session-{index}" for index in range(32) + } + assert all(isinstance(record["timestamp"], (int, float)) for record in records) + assert all( + set(record) <= {"event", "timestamp", "session_id", "source", "transcript_path"} + for record in records + ) + assert ( + sessions.capture_hook_event(trace, {"event": "OtherHook", "session_id": "ignored"}) + is False + ) + assert len(trace.read_text(encoding="utf-8").splitlines()) == 32 + + +@pytest.mark.parametrize("agent_type", ["general-purpose", ""]) +def test_hook_capture_preserves_agent_type_and_strips_stop_message( + tmp_path: Path, agent_type: str +) -> None: + trace = tmp_path / "events.jsonl" + captured = sessions.capture_hook_event( + trace, + { + "hook_event_name": "SubagentStop", + "session_id": ROOT_ID, + "agent_id": "agent-x", + "agent_type": agent_type, + "transcript_path": "/tmp/root.jsonl", + "agent_transcript_path": "/tmp/agent-x.jsonl", + "last_assistant_message": "must not be persisted", + }, + ) + + assert captured is True + [record] = [json.loads(line) for line in trace.read_text(encoding="utf-8").splitlines()] + assert record["agent_type"] == agent_type + assert set(record) == { + "event", + "timestamp", + "session_id", + "agent_id", + "agent_type", + "transcript_path", + "agent_transcript_path", + } + + +def test_summary_collapses_repeated_events_excludes_unrelated_and_sweeps_half_open( + tmp_path: Path, +) -> None: + log_root, trace, child_transcript, root_transcript = _base_sessions( + tmp_path, child_id="session-child" + ) + assert child_transcript is not None + unrelated = _session(log_root, "session-unrelated", parent_session_id="other-parent") + rows = _index_rows(log_root) + by_session_id = {row["session_id"]: row for row in rows} + entries = [by_session_id[ROOT_ID], by_session_id["session-child"]] + entries.append( + _entry( + "session-unrelated", + unrelated, + parent_session_id="other-parent", + caller_session_id="other-parent", + ) + ) + _index(log_root, entries) + _trace( + trace, + [ + _event( + "SessionStart", + ROOT_ID, + 1, + transcript=root_transcript, + ), + _event( + "SessionStart", + ROOT_ID, + 1.5, + transcript=root_transcript, + ), + _event("SessionStart", "session-child", 3, transcript=child_transcript), + _event("SessionStart", "session-unrelated", 1, transcript=unrelated), + _event("SessionEnd", ROOT_ID, 2, transcript=root_transcript), + _event("SessionEnd", ROOT_ID, 3, transcript=root_transcript), + _event("SessionEnd", "session-child", 4, transcript=child_transcript), + _event("SessionEnd", "session-unrelated", 5, transcript=unrelated), + ], + ) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["peak_sessions"] == 1 + assert summary["coverage"]["complete"] is True + assert summary["violations"] == [] + assert summary["coverage"]["managed_session_ids"] == ["session-child"] + assert [identity["session_id"] for identity in summary["identities"]] == [ + ROOT_ID, + "session-child", + ] + root_interval = summary["identities"][0] + assert (root_interval["start"], root_interval["end"]) == (1.0, 3.0) + assert (trace.parent / "session-concurrency.json").is_file() + + +@pytest.mark.parametrize( + ("child_event", "expected_violation"), + [("SessionStart", "missing_end"), ("SessionEnd", "orphan_end")], +) +def test_summary_does_not_infer_child_end( + tmp_path: Path, child_event: str, expected_violation: str +) -> None: + log_root, trace, child_transcript, root_transcript = _base_sessions( + tmp_path, child_id="session-child" + ) + assert child_transcript is not None + _trace( + trace, + [ + _event( + "SessionStart", + ROOT_ID, + 1, + transcript=root_transcript, + ), + _event(child_event, "session-child", 2, transcript=child_transcript), + _event("SessionEnd", ROOT_ID, 5, transcript=root_transcript), + ], + ) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["peak_sessions"] == 1 + assert summary["coverage"]["complete"] is False + assert f"lifecycle:session:session-child:{expected_violation}" in summary["violations"] + + +def test_summary_enumerates_native_agent_transcripts_and_requires_endpoints( + tmp_path: Path, +) -> None: + log_root, trace, _child, root_transcript = _base_sessions(tmp_path) + agent_transcript = log_root / "transcripts" / ROOT_ID / "subagents" / "agent-agent-a.jsonl" + agent_transcript.parent.mkdir(parents=True) + agent_transcript.write_text("{}\n", encoding="utf-8") + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event( + "SubagentStart", + ROOT_ID, + 2, + agent_id="agent-a", + agent_type="general-purpose", + ), + _event( + "SubagentStop", + ROOT_ID, + 4, + agent_id="agent-a", + agent_type="general-purpose", + agent_transcript=agent_transcript, + ), + _event("SessionEnd", ROOT_ID, 5, transcript=root_transcript), + ], + ) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["coverage"]["complete"] is True + assert summary["peak_sessions"] == 2 + assert summary["coverage"]["native_agents"] == [ + { + "session_id": ROOT_ID, + "agent_id": "agent-a", + "transcript_path": str(agent_transcript.resolve()), + } + ] + + +@pytest.mark.parametrize( + ("agent_type", "extra_violation"), + [ + pytest.param("custom-main", None, id="named-custom-main"), + pytest.param("", f"capture:{ROOT_ID}:agent-x:agent_type_missing", id="blank-type"), + pytest.param(None, f"capture:{ROOT_ID}:agent-x:agent_type_missing", id="missing-type"), + ], +) +def test_orphan_agent_stop_is_reported_even_without_an_admitted_type( + tmp_path: Path, + agent_type: str | None, + extra_violation: str | None, +) -> None: + log_root, trace, _, root_transcript = _base_sessions(tmp_path) + missing_agent_transcript = ( + log_root / "transcripts" / ROOT_ID / "subagents" / "agent-agent-x.jsonl" + ) + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event( + "SubagentStop", + ROOT_ID, + 2, + agent_id="agent-x", + agent_type=agent_type, + agent_transcript=missing_agent_transcript, + ), + _event("SessionEnd", ROOT_ID, 3, transcript=root_transcript), + ], + ) + + summary = sessions.summarize(trace, _envelope(), log_root) + + expected_violations = [ + f"lifecycle:agent:{ROOT_ID}:agent-x:orphan_end", + f"native:{ROOT_ID}:agent-x:lifecycle_incomplete", + f"native:{ROOT_ID}:agent-x:transcript_unenumerated", + ] + if extra_violation is not None: + expected_violations.append(extra_violation) + assert summary["coverage"]["complete"] is False + assert summary["violations"] == sorted(expected_violations) + assert summary["peak_sessions"] == 1 + + +def test_enumerated_native_transcript_without_agent_hooks_is_incomplete( + tmp_path: Path, +) -> None: + log_root, trace, _, root_transcript = _base_sessions(tmp_path) + agent_transcript = log_root / "transcripts" / ROOT_ID / "subagents" / "agent-agent-a.jsonl" + agent_transcript.parent.mkdir(parents=True) + agent_transcript.write_text("{}\n", encoding="utf-8") + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event("SessionEnd", ROOT_ID, 3, transcript=root_transcript), + ], + ) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["coverage"]["complete"] is False + assert f"native:{ROOT_ID}:agent-a:lifecycle_incomplete" in summary["violations"] + + +def test_native_transcript_symlink_loop_is_reported_missing(tmp_path: Path) -> None: + transcript = tmp_path / "agent-agent-a.jsonl" + transcript.symlink_to(transcript.name) + violations: list[str] = [] + proof = sessions._verify_native_transcript( + ROOT_ID, + "agent-a", + transcript, + [{"event": "SubagentStart"}, {"event": "SubagentStop"}], + violations, + ) + + assert proof is None + assert violations == [f"native:{ROOT_ID}:agent-a:transcript_missing"] + + +def test_unresolvable_session_transcript_is_reported_without_aborting_summary( + tmp_path: Path, +) -> None: + log_root, trace, _, _root_transcript = _base_sessions(tmp_path) + transcript_loop = tmp_path / "transcript-loop.jsonl" + transcript_loop.symlink_to(transcript_loop.name) + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=transcript_loop), + _event("SessionEnd", ROOT_ID, 2, transcript=transcript_loop), + ], + ) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["coverage"]["root_session_seen"] is True + assert summary["coverage"]["complete"] is False + assert f"lifecycle:session:{ROOT_ID}:transcript_resolution_failed" in summary["violations"] + + +def test_indexed_owned_child_without_hooks_is_incomplete(tmp_path: Path) -> None: + log_root, trace, _, root_transcript = _base_sessions(tmp_path, child_id="missing-hooks") + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event("SessionEnd", ROOT_ID, 5, transcript=root_transcript), + ], + ) + summary = sessions.summarize(trace, _envelope(), log_root) + assert not summary["coverage"]["complete"] + assert "lifecycle:missing-hooks:missing_endpoints" in summary["violations"] + + +def test_resumed_identity_requires_a_final_end(tmp_path: Path) -> None: + log_root, trace, _, root_transcript = _base_sessions(tmp_path) + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event("SessionEnd", ROOT_ID, 2, transcript=root_transcript), + _event("SessionStart", ROOT_ID, 3, transcript=root_transcript), + ], + ) + summary = sessions.summarize(trace, _envelope(), log_root) + assert not summary["coverage"]["complete"] + assert f"lifecycle:session:{ROOT_ID}:missing_final_end" in summary["violations"] + + +def test_empty_half_open_interval_does_not_reduce_concurrent_peak() -> None: + assert ( + sessions._peak_sessions( + [{"start": 1, "end": 3}, {"start": 2, "end": 2}, {"start": 2, "end": 4}] + ) + == 2 + ) + + +@pytest.mark.parametrize( + ("issue_url", "expected_violations"), + [ + (ISSUE_URL, []), + ("owner/repo#5233", []), + ("other/repo#5233", ["ownership:invalid_issue_reference"]), + ], +) +def test_issue_reference_must_match_repository( + tmp_path: Path, issue_url: str, expected_violations: list[str] +) -> None: + log_root, trace, _, root_transcript = _base_sessions(tmp_path) + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event("SessionEnd", ROOT_ID, 2, transcript=root_transcript), + ], + ) + proof = sessions.collect_ownership(trace, _envelope(), log_root, issue_url, REPOSITORY) + assert proof["branches"] == [] + assert proof["claimed"] is False + assert proof["violations"] == expected_violations + + +def test_missing_root_hooks_do_not_discard_available_branch_proof(tmp_path: Path) -> None: + log_root, trace, _, root_transcript = _base_sessions(tmp_path) + _jsonl( + root_transcript, + [ + _tool_use( + "branch", + "create_and_publish_branch", + {"issue_number": "5233", "remote_url": "https://github.com/owner/repo.git"}, + ), + _tool_result("branch", {"merge_target": "impl/5233-fix", "was_unique": True}), + ], + ) + _trace(trace, []) + proof = sessions.collect_ownership(trace, _envelope(), log_root, ISSUE_URL, REPOSITORY) + assert proof["branches"] == ["impl/5233-fix"] + assert "trace:root_session_missing" in proof["violations"] + + +def test_missing_root_and_malformed_trace_are_incomplete(tmp_path: Path) -> None: + log_root = tmp_path / "logs" + trace = tmp_path / "run" / "session-lifecycle.jsonl" + trace.parent.mkdir(parents=True) + trace.write_text("{malformed\n", encoding="utf-8") + + summary = sessions.summarize(trace, None, log_root) + + assert summary["coverage"]["complete"] is False + assert "envelope:missing" in summary["violations"] + assert "sessions_index:unavailable" in summary["violations"] + assert "trace:invalid_json:1" in summary["violations"] + + +@pytest.mark.parametrize("dir_name", [".", ".."]) +def test_summary_rejects_dot_session_directories(tmp_path: Path, dir_name: str) -> None: + log_root, trace, _, root_transcript = _base_sessions(tmp_path) + entry = _entry(ROOT_ID, root_transcript) + entry["dir_name"] = dir_name + _index(log_root, [entry]) + _trace(trace, [_event("SessionStart", ROOT_ID, 1, transcript=root_transcript)]) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["coverage"]["complete"] is False + assert f"sessions_index:{ROOT_ID}:invalid_dir_name" in summary["violations"] + + +def test_summary_reader_reports_missing_summary_reason(tmp_path: Path) -> None: + violations: list[str] = [] + + assert sessions._read_summary(tmp_path, "missing-session", violations) is None + assert violations == ["summary:missing-session:missing"] + + +def test_summary_rejects_malformed_owned_index_summary_and_transcript( + tmp_path: Path, +) -> None: + log_root, trace, _child, root_transcript = _base_sessions(tmp_path) + index_path = log_root / "sessions.jsonl" + index_path.write_text( + index_path.read_text(encoding="utf-8") + "{malformed\n", encoding="utf-8" + ) + (log_root / "sessions" / ROOT_ID / "summary.json").write_text("{malformed\n", encoding="utf-8") + root_transcript.write_text("{malformed\n", encoding="utf-8") + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event("SessionEnd", ROOT_ID, 2, transcript=root_transcript), + ], + ) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["coverage"]["complete"] is False + assert "sessions_index:invalid_json:2" in summary["violations"] + assert f"summary:{ROOT_ID}:missing_or_malformed" in summary["violations"] + assert f"summary:{ROOT_ID}:invalid_json:1:2" in summary["violations"] + assert f"transcript:{ROOT_ID}:invalid_json:1" in summary["violations"] + + +@pytest.mark.parametrize( + ("claim_input_label", "expected_claimed"), + [(None, True), ("in-progress", True), ("different-label", False)], +) +def test_ownership_pairs_root_branch_with_managed_claim_and_keeps_compact_proof( + tmp_path: Path, + claim_input_label: str | None, + expected_claimed: bool, +) -> None: + claim_input: dict[str, Any] = {"issue_url": ISSUE_URL, "allow_reentry": False} + if claim_input_label is not None: + claim_input["label"] = claim_input_label + claim_records = [ + _tool_use("claim-1", "claim_and_resolve_issue", claim_input), + _tool_result( + "claim-1", + { + "success": True, + "claimed": True, + "issue_number": 5233, + "issue_title": "private title", + }, + ), + ] + log_root, trace, child_transcript, root_transcript = _base_sessions( + tmp_path, + child_id="session-child", + child_records=claim_records, + ) + assert child_transcript is not None + _jsonl( + root_transcript, + [ + _tool_use( + "branch-1", + "create_and_publish_branch", + { + "issue_number": "5233", + "remote_url": "https://github.com/owner/repo.git", + }, + ), + _tool_result("branch-1", {"merge_target": "impl/5233-fix", "was_unique": True}), + ], + ) + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event("SessionStart", "session-child", 2, transcript=child_transcript), + _event("SessionEnd", "session-child", 4, transcript=child_transcript), + _event("SessionEnd", ROOT_ID, 5, transcript=root_transcript), + ], + ) + + proof = sessions.collect_ownership( + trace, _envelope(), log_root, ISSUE_URL, REPOSITORY, claim_label="in-progress" + ) + + assert set(proof) == {"branches", "claimed", "transcripts", "violations"} + assert all(set(item) == {"session_id", "path"} for item in proof["transcripts"]) + assert proof["branches"] == ["impl/5233-fix"] + assert proof["claimed"] is expected_claimed + if expected_claimed: + assert proof["violations"] == [] + else: + assert "ownership:claim_result_failed" in proof["violations"] + assert {item["session_id"] for item in proof["transcripts"]} == {ROOT_ID, "session-child"} + + +def test_ownership_ignores_unhashable_tool_names(tmp_path: Path) -> None: + log_root, trace, _, root_transcript = _base_sessions(tmp_path) + record = _tool_use("unrelated", "ignored", {}) + record["message"]["content"][0]["name"] = [] + _jsonl(root_transcript, [record]) + _trace(trace, [_event("SessionStart", ROOT_ID, 1, transcript=root_transcript)]) + + proof = sessions.collect_ownership(trace, _envelope(), log_root, ISSUE_URL, REPOSITORY) + + assert proof["branches"] == [] + assert proof["claimed"] is False + assert proof["violations"] == [] + + +@pytest.mark.parametrize("scenario", ["child_branch", "unpaired_root_branch", "claim_reentry"]) +def test_ownership_isolates_branch_and_claim_scenarios( + tmp_path: Path, + scenario: str, +) -> None: + child_records = [ + _tool_use( + "child-branch", + "create_and_publish_branch", + {"issue_number": "5233", "remote_url": "git@github.com:owner/repo.git"}, + ), + _tool_result("child-branch", {"merge_target": "wrong-owner/branch"}), + ] + log_root, trace, child_transcript, root_transcript = _base_sessions( + tmp_path, + child_id="session-child" if scenario == "child_branch" else None, + child_records=child_records if scenario == "child_branch" else None, + ) + if scenario == "claim_reentry": + root_records = [ + _tool_use( + "claim-1", + "claim_and_resolve_issue", + {"issue_url": ISSUE_URL, "allow_reentry": False}, + ), + _tool_result("claim-1", {"success": True, "claimed": True, "issue_number": 5233}), + _tool_use( + "claim-2", + "claim_and_resolve_issue", + {"issue_url": ISSUE_URL, "allow_reentry": False}, + ), + _tool_result("claim-2", {"success": True, "claimed": True, "issue_number": 5233}), + ] + else: + root_records = [ + _tool_use( + "root-branch", + "create_and_publish_branch", + {"issue_number": "5233", "remote_url": "git@github.com:owner/repo.git"}, + ), + _tool_result( + "root-branch" if scenario == "child_branch" else "wrong-branch-id", + {"merge_target": "impl/5233-root"}, + ), + ] + _jsonl(root_transcript, root_records) + _trace(trace, [_event("SessionStart", ROOT_ID, 1, transcript=root_transcript)]) + + proof = sessions.collect_ownership(trace, _envelope(), log_root, ISSUE_URL, REPOSITORY) + + assert proof["branches"] == (["impl/5233-root"] if scenario == "child_branch" else []) + assert proof["claimed"] is False + if scenario == "claim_reentry": + assert proof["violations"] == ["ownership:claim_reentry"] + elif scenario == "unpaired_root_branch": + assert proof["violations"] == [ + "ownership:branch_result_failed", + f"transcript:{ROOT_ID}:unpaired_tool_use:root-branch", + ] + else: + assert proof["violations"] == [] + + +@pytest.mark.parametrize("prefix", [DIRECT_PREFIX, PLUGIN_PREFIX]) +def test_ownership_accepts_both_canonical_namespaces(tmp_path: Path, prefix: str) -> None: + proof = _root_ownership_proof( + tmp_path, + [ + _tool_use( + "claim", + "claim_and_resolve_issue", + {"issue_url": ISSUE_URL, "allow_reentry": False}, + prefix=prefix, + ), + _tool_result("claim", {"success": True, "claimed": True, "issue_number": 5233}), + _tool_use( + "branch", + "create_and_publish_branch", + { + "issue_number": "5233", + "remote_url": "https://github.com/owner/repo.git", + }, + prefix=prefix, + ), + _tool_result("branch", {"merge_target": "impl/5233-fix", "was_unique": True}), + ], + ) + + assert proof["branches"] == ["impl/5233-fix"] + assert proof["claimed"] is True + assert proof["violations"] == [] + + +@pytest.mark.parametrize( + "name", + [ + "create_and_publish_branch", + "mcp__other__create_and_publish_branch", + f"{PLUGIN_PREFIX}extra__create_and_publish_branch", + ], +) +def test_tracked_tool_under_another_namespace_is_reported(tmp_path: Path, name: str) -> None: + proof = _root_ownership_proof( + tmp_path, + [ + _tool_use( + "branch", + name, + { + "issue_number": "5233", + "remote_url": "https://github.com/owner/repo.git", + }, + prefix="", + ), + _tool_result("branch", {"merge_target": "impl/5233-fix"}), + ], + ) + + assert proof["branches"] == [] + assert f"transcript:{ROOT_ID}:unrecognised_tool_namespace:1" in proof["violations"] + + +@pytest.mark.parametrize("carrier", ["mcp_meta", "block_json"]) +def test_reply_is_read_from_each_supported_carrier(tmp_path: Path, carrier: str) -> None: + proof = _root_ownership_proof( + tmp_path, + [ + _tool_use( + "branch", + "create_and_publish_branch", + { + "issue_number": "5233", + "remote_url": "https://github.com/owner/repo.git", + }, + ), + _tool_result("branch", {"merge_target": "impl/5233-fix"}, carrier=carrier), + ], + ) + + assert proof["branches"] == ["impl/5233-fix"] + assert proof["violations"] == [] + + +def test_non_json_display_text_proves_no_reply(tmp_path: Path) -> None: + result_row = _tool_result( + "branch", + {"merge_target": "impl/evil"}, + carrier="display_only", + ) + result_row["message"]["content"][0]["content"] += "\nmerge_target: impl/evil" + proof = _root_ownership_proof( + tmp_path, + [ + _tool_use( + "branch", + "create_and_publish_branch", + { + "issue_number": "5233", + "remote_url": "https://github.com/owner/repo.git", + }, + ), + result_row, + ], + ) + + assert proof["branches"] == [] + assert "ownership:branch_result_failed" in proof["violations"] + + +def test_invalid_row_metadata_does_not_fall_back_to_block_json(tmp_path: Path) -> None: + result_row = _tool_result( + "branch", + {"merge_target": "impl/5233-fix"}, + carrier="block_json", + ) + result_row["mcpMeta"] = { + "_meta": {"fastmcp": {"wrap_result": True}}, + "structuredContent": {"result": "not json"}, + } + proof = _root_ownership_proof( + tmp_path, + [ + _tool_use( + "branch", + "create_and_publish_branch", + { + "issue_number": "5233", + "remote_url": "https://github.com/owner/repo.git", + }, + ), + result_row, + ], + ) + + assert proof["branches"] == [] + assert "ownership:branch_result_failed" in proof["violations"] + + +def test_row_level_reply_is_ignored_when_a_row_holds_several_results(tmp_path: Path) -> None: + result = {"merge_target": "impl/5233-fix"} + proof = _root_ownership_proof( + tmp_path, + [ + _tool_use( + "branch", + "create_and_publish_branch", + { + "issue_number": "5233", + "remote_url": "https://github.com/owner/repo.git", + }, + ), + _tool_use("claim", "claim_and_resolve_issue", {"issue_url": "https://example.com"}), + { + "type": "user", + "message": { + "content": [ + { + "type": "tool_result", + "tool_use_id": "branch", + "content": "tool reply\n\nsuccess: True", + }, + { + "type": "tool_result", + "tool_use_id": "claim", + "content": "tool reply\n\nsuccess: True", + }, + ] + }, + "mcpMeta": { + "_meta": {"fastmcp": {"wrap_result": True}}, + "structuredContent": {"result": json.dumps(result)}, + }, + }, + ], + ) + + assert proof["branches"] == [] + assert "ownership:branch_result_failed" in proof["violations"] + + +def test_untracked_merge_target_reply_is_not_a_branch(tmp_path: Path) -> None: + proof = _root_ownership_proof( + tmp_path, + [ + _tool_use("clone", "bootstrap_clone", {}), + _tool_result("clone", {"merge_target": "main"}), + ], + ) + + assert proof["branches"] == [] + assert proof["violations"] == [] + + +@pytest.mark.parametrize( + ("tool", "success"), + [ + ("run_skill", True), + ("complete_run_skill_result", True), + ("recover_run_skill_result", True), + ("run_skill", False), + ], +) +def test_launch_receipt_adopts_a_worker_with_empty_parent_fields( + tmp_path: Path, + tool: str, + success: bool, +) -> None: + log_root, trace, _, root_transcript = _base_sessions(tmp_path) + worker_id = "session-worker" + worker_transcript = _worker(log_root, worker_id) + _jsonl( + root_transcript, + _launch( + "launch", + {"success": success, "receipt_id": "r1", "session_id": worker_id}, + tool=tool, + ), + ) + _trace_root_and_worker(trace, log_root, worker_id, worker_transcript) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["coverage"]["managed_session_ids"] == [worker_id] + assert summary["coverage"]["complete"] is True + + +def test_acknowledgement_alone_adopts_a_worker(tmp_path: Path) -> None: + log_root, trace, _, root_transcript = _base_sessions(tmp_path) + worker_id = "session-worker" + worker_transcript = _worker(log_root, worker_id) + _jsonl( + root_transcript, + [ + *_launch( + "launch", + {"success": True, "receipt_id": "r1", "audit_status": "published"}, + ), + *_launch( + "ack", + {"success": True, "receipt_id": "r1", "session_id": worker_id}, + tool="complete_run_skill_result", + ), + ], + ) + _trace_root_and_worker(trace, log_root, worker_id, worker_transcript) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["coverage"]["managed_session_ids"] == [worker_id] + assert summary["violations"] == [] + + +@pytest.mark.parametrize("worker_dispatch", [DISPATCH_ID, "another-dispatch"]) +def test_overlapping_session_without_receipt_or_parent_is_reported_not_adopted( + tmp_path: Path, + worker_dispatch: str, +) -> None: + log_root, trace, _, _ = _base_sessions(tmp_path) + worker_id = "session-worker" + worker_transcript = _worker(log_root, worker_id) + entries = _index_rows(log_root) + next(row for row in entries if row["session_id"] == worker_id)["dispatch_id"] = worker_dispatch + _index(log_root, entries) + _trace_root_and_worker(trace, log_root, worker_id, worker_transcript) + + summary = sessions.summarize(trace, _envelope(), log_root) + proof = sessions.collect_ownership(trace, _envelope(), log_root, ISSUE_URL, REPOSITORY) + + violation = f"lineage:{worker_id}:unattributed" + assert worker_id not in summary["coverage"]["managed_session_ids"] + assert violation in summary["violations"] + assert summary["coverage"]["complete"] is False + assert violation in proof["violations"] + + +def test_receipt_in_an_unowned_transcript_does_not_adopt(tmp_path: Path) -> None: + log_root, trace, _, _ = _base_sessions(tmp_path) + worker_id = "session-worker" + worker_transcript = _worker(log_root, worker_id) + foreign_id = "session-foreign" + foreign_transcript = _session( + log_root, + foreign_id, + parent_session_id="other-parent", + records=_launch( + "launch", + {"success": True, "receipt_id": "r1", "session_id": worker_id}, + ), + ) + entries = _index_rows(log_root) + entries.append( + _entry( + foreign_id, + foreign_transcript, + parent_session_id="other-parent", + caller_session_id="other-parent", + ) + ) + _index(log_root, entries) + _trace_root_and_worker(trace, log_root, worker_id, worker_transcript) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert worker_id not in summary["coverage"]["managed_session_ids"] + assert f"lineage:{worker_id}:unattributed" in summary["violations"] + + +@pytest.mark.parametrize( + "reply", + [ + { + "success": False, + "is_error": True, + "error": "permission denied", + "stage": "preflight", + "retriable": False, + }, + {"success": False, "session_id": ""}, + {"success": False, "session_id": "headless-0123456789ab"}, + ], +) +def test_reply_without_receipt_id_proves_no_launch( + tmp_path: Path, + reply: dict[str, Any], +) -> None: + log_root, trace, _, root_transcript = _base_sessions(tmp_path) + _jsonl(root_transcript, _launch("launch", reply)) + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event("SessionEnd", ROOT_ID, 2, transcript=root_transcript), + ], + ) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["coverage"]["managed_session_ids"] == [] + assert summary["violations"] == [] + + +def test_launch_reply_marked_is_error_is_not_evidence(tmp_path: Path) -> None: + log_root, trace, _, root_transcript = _base_sessions(tmp_path) + worker_id = "session-worker" + worker_transcript = _worker(log_root, worker_id) + _jsonl( + root_transcript, + _launch( + "launch", + {"success": True, "receipt_id": "r1", "session_id": worker_id}, + is_error=True, + ), + ) + _trace_root_and_worker(trace, log_root, worker_id, worker_transcript) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert worker_id not in summary["coverage"]["managed_session_ids"] + assert f"lineage:{worker_id}:unattributed" in summary["violations"] + + +def test_launched_session_absent_from_trace_and_index_is_reported(tmp_path: Path) -> None: + log_root, trace, _, root_transcript = _base_sessions(tmp_path) + _jsonl( + root_transcript, + _launch( + "launch", + {"success": True, "receipt_id": "r1", "session_id": "session-ghost"}, + ), + ) + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event("SessionEnd", ROOT_ID, 2, transcript=root_transcript), + ], + ) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert "lineage:session-ghost:launched_session_unobserved" in summary["violations"] + + +@pytest.mark.parametrize( + ("index_count", "receipt_names_worker"), + [(0, False), (0, True), (2, False), (2, True)], +) +def test_candidate_without_exactly_one_index_row_is_reported( + tmp_path: Path, + index_count: int, + receipt_names_worker: bool, +) -> None: + log_root, trace, _, root_transcript = _base_sessions(tmp_path) + worker_id = "session-worker" + worker_transcript = _worker(log_root, worker_id) + entries = _index_rows(log_root) + worker_entry = next(row for row in entries if row["session_id"] == worker_id) + entries = [row for row in entries if row["session_id"] != worker_id] + if index_count == 2: + entries.extend([worker_entry, dict(worker_entry)]) + _index(log_root, entries) + _jsonl( + root_transcript, + _launch( + "launch", + {"success": True, "receipt_id": "r1", "session_id": worker_id}, + ) + if receipt_names_worker + else [], + ) + _trace_root_and_worker(trace, log_root, worker_id, worker_transcript) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert worker_id not in summary["coverage"]["managed_session_ids"] + assert f"sessions_index:{worker_id}:missing_or_ambiguous" in summary["violations"] + + +def test_receipt_named_worker_from_another_dispatch_is_not_adopted(tmp_path: Path) -> None: + log_root, trace, _, root_transcript = _base_sessions(tmp_path) + worker_id = "session-worker" + worker_transcript = _worker(log_root, worker_id) + entries = _index_rows(log_root) + next(row for row in entries if row["session_id"] == worker_id)["dispatch_id"] = ( + "another-dispatch" + ) + _index(log_root, entries) + _jsonl( + root_transcript, + _launch( + "launch", + {"success": True, "receipt_id": "r1", "session_id": worker_id}, + ), + ) + _trace_root_and_worker(trace, log_root, worker_id, worker_transcript) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert worker_id not in summary["coverage"]["managed_session_ids"] + assert f"sessions_index:{worker_id}:dispatch_mismatch" in summary["violations"] + + +def test_receipt_adopted_worker_with_a_contradicting_declared_parent_is_reported( + tmp_path: Path, +) -> None: + log_root, trace, _, root_transcript = _base_sessions(tmp_path) + worker_id = "session-worker" + worker_transcript = _worker(log_root, worker_id) + summary_path = log_root / "sessions" / worker_id / "summary.json" + worker_summary = json.loads(summary_path.read_text(encoding="utf-8")) + worker_summary["execution_identity"]["parent_session_id"] = "other-parent" + summary_path.write_text(json.dumps(worker_summary), encoding="utf-8") + _jsonl( + root_transcript, + _launch( + "launch", + {"success": True, "receipt_id": "r1", "session_id": worker_id}, + ), + ) + _trace_root_and_worker(trace, log_root, worker_id, worker_transcript) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["coverage"]["managed_session_ids"] == [worker_id] + assert f"summary:{worker_id}:parent_mismatch" in summary["violations"] + + +def test_receipt_adopted_worker_without_lifecycle_endpoints_is_incomplete(tmp_path: Path) -> None: + log_root, trace, _, root_transcript = _base_sessions(tmp_path) + worker_id = "session-worker" + _worker(log_root, worker_id) + _jsonl( + root_transcript, + _launch( + "launch", + {"success": True, "receipt_id": "r1", "session_id": worker_id}, + ), + ) + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event("SessionEnd", ROOT_ID, 2, transcript=root_transcript), + ], + ) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["coverage"]["managed_session_ids"] == [worker_id] + assert summary["coverage"]["complete"] is False + assert f"lifecycle:{worker_id}:missing_endpoints" in summary["violations"] + + +def test_missing_native_transcript_is_reported_once(tmp_path: Path) -> None: + root_transcript = tmp_path / "transcripts" / f"{ROOT_ID}.jsonl" + root_transcript.parent.mkdir(parents=True) + root_transcript.write_text("{}\n", encoding="utf-8") + subagents_dir = root_transcript.parent / ROOT_ID / "subagents" + subagents_dir.mkdir(parents=True) + missing_transcript = subagents_dir / "agent-agent-a.jsonl" + missing_transcript.symlink_to(subagents_dir / "missing-agent.jsonl") + events = [ + _event( + "SubagentStart", + ROOT_ID, + 1, + agent_id="agent-a", + agent_type="general-purpose", + ), + _event( + "SubagentStop", + ROOT_ID, + 2, + agent_id="agent-a", + agent_type="general-purpose", + agent_transcript=missing_transcript, + ), + ] + violations: list[str] = [] + + native_agents = sessions._check_transcript_coverage( + events, + {ROOT_ID: {"transcript": root_transcript}}, + violations, + ) + + missing = f"native:{ROOT_ID}:agent-a:transcript_missing" + assert native_agents == [] + assert violations.count(missing) == 1 + + +def test_native_transcript_without_observed_path_reports_one_diagnostic( + tmp_path: Path, +) -> None: + transcript = tmp_path / "agent-agent-a.jsonl" + transcript.write_text("{}\n", encoding="utf-8") + violations: list[str] = [] + + proof = sessions._verify_native_transcript( + ROOT_ID, + "agent-a", + transcript, + [{"event": "SubagentStart"}, {"event": "SubagentStop"}], + violations, + ) + + assert proof is not None + assert violations == [f"native:{ROOT_ID}:agent-a:transcript_path_missing"] + + +@pytest.mark.parametrize( + ("session_id", "symlink_root"), + [("../outside", False), (ROOT_ID, True)], +) +def test_native_transcript_enumeration_rejects_escaping_subagent_roots( + tmp_path: Path, session_id: str, symlink_root: bool +) -> None: + transcript_root = tmp_path / "transcripts" + transcript_root.mkdir() + transcript = transcript_root / "root.jsonl" + transcript.write_text("{}\n", encoding="utf-8") + outside = tmp_path / "outside" + outside_subagents = outside / "subagents" + outside_subagents.mkdir(parents=True) + (outside_subagents / "agent-agent-a.jsonl").write_text("{}\n", encoding="utf-8") + if symlink_root: + (transcript_root / session_id).symlink_to(outside, target_is_directory=True) + violations: list[str] = [] + + enumerated = sessions._enumerate_native_transcripts( + {session_id: {"transcript": transcript}}, + {session_id: {str(transcript.resolve())}}, + violations, + ) + + assert enumerated[session_id] == {} + assert f"native:{session_id}:unsafe_subagent_root" in violations + + +def test_native_transcript_enumeration_rejects_file_symlink_escape( + tmp_path: Path, +) -> None: + transcript_root = tmp_path / "transcripts" + transcript_root.mkdir() + transcript = transcript_root / f"{ROOT_ID}.jsonl" + transcript.write_text("{}\n", encoding="utf-8") + subagents_dir = transcript_root / ROOT_ID / "subagents" + subagents_dir.mkdir(parents=True) + outside = tmp_path / "outside-agent.jsonl" + outside.write_text("{}\n", encoding="utf-8") + (subagents_dir / "agent-agent-a.jsonl").symlink_to(outside) + violations: list[str] = [] + + enumerated = sessions._enumerate_native_transcripts( + {ROOT_ID: {"transcript": transcript}}, + {ROOT_ID: {str(transcript.resolve())}}, + violations, + ) + + assert enumerated[ROOT_ID] == {} + assert f"native:{ROOT_ID}:agent-a:transcript_outside_root" in violations + + +def test_owned_session_transcript_is_validated_and_paired_in_one_read( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + transcript = tmp_path / "session-root.jsonl" + records = _launch( + "launch", + {"success": True, "receipt_id": "r1", "session_id": "session-worker"}, + tool="run_skill", + ) + transcript.write_bytes( + b'\nnot-json\nnull\n{"invalid":"bad\xff"}\n' + + "".join(json.dumps(record) + "\n" for record in records).encode("utf-8") + ) + opened: list[Path] = [] + original_open = Path.open + + def count_transcript_open(path: Path, *args: Any, **kwargs: Any) -> IO[Any]: + if path == transcript: + opened.append(path) + return original_open(path, *args, **kwargs) + + monkeypatch.setattr(Path, "open", count_transcript_open) + violations: list[str] = [] + owned: dict[str, sessions._OwnedSession] = {} + summary = { + "session_id": ROOT_ID, + "dispatch_id": DISPATCH_ID, + "execution_identity": {"parent_session_id": ""}, + } + + sessions._add_owned_session( + tmp_path, + _entry(ROOT_ID, transcript), + ROOT_ID, + violations, + owned, + summary=summary, + ) + + assert opened == [transcript] + assert violations == [ + f"transcript:{ROOT_ID}:empty_line:1", + f"transcript:{ROOT_ID}:invalid_json:2", + f"transcript:{ROOT_ID}:invalid_record:3", + f"transcript:{ROOT_ID}:invalid_json:4", + ] + assert len(owned[ROOT_ID]["calls"]) == 1 + assert owned[ROOT_ID]["calls"][0][0] == "run_skill" diff --git a/tests/infra/test_e2e_sessions_captured.py b/tests/infra/test_e2e_sessions_captured.py new file mode 100644 index 0000000000..43cb38757b --- /dev/null +++ b/tests/infra/test_e2e_sessions_captured.py @@ -0,0 +1,270 @@ +"""Tests for sanitized, captured E2E session evidence.""" + +from __future__ import annotations + +import json +import re +from pathlib import Path +from typing import Any, NamedTuple + +import pytest + +from autoskillit.core import PLUGIN_PREFIX +from tests._hook_protocol_oracle import claude_matcher_admits +from tests._script_loader import load_script +from tests.infra._e2e_evidence import fixture_dir, load_provenance, materialize_run + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.medium] + +REPO_ROOT = Path(__file__).resolve().parents[2] +FIXTURE_NAME = "run_38170521660_attempt_1" +FIXTURE_DIR = fixture_dir(FIXTURE_NAME) +_SCRIPT = REPO_ROOT / "scripts" / "e2e" / "e2e_sessions.py" +sessions = load_script("_autoskillit_e2e_sessions_captured", _SCRIPT) +ROOT_SESSION_ID = "2292658a-34cf-44a0-a0b1-f7a20218874b" +WORKER_SESSION_IDS = ( + "8ef8f993-8caa-47dd-83b0-26f9e5b51139", + "54f18144-1188-45d7-a392-17b4529f721e", +) +UNMATCHED_AGENT_ID = "a610a4bda1f28ca35" +NATIVE_AGENT_IDS = { + "a516cbae85151dc7c", + "a9ed51b7c02b828dd", + "aeb96cc0ddbdab294", + "a7e613554b849d6b2", +} + +_INPUTS_TO_PRESERVE = { + PLUGIN_PREFIX + "claim_and_resolve_issue", + PLUGIN_PREFIX + "create_and_publish_branch", +} +_TRUNCATION = re.compile(r"\.\.\.\[fixture-truncated (\d+) chars\]$") + + +class _MaterializedRun(NamedTuple): + provenance: dict[str, Any] + trace: Path + log_root: Path + projects: Path + envelope: dict[str, Any] + + +def _rows(path: Path) -> list[dict[str, Any]]: + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] + + +def _tool_reply(rows: list[dict[str, Any]], suffix: str) -> tuple[dict[str, Any], dict[str, Any]]: + matches = [ + (row, block) + for row in rows + for block in row.get("message", {}).get("content", []) + if isinstance(block, dict) + and block.get("type") == "tool_use" + and isinstance(block.get("name"), str) + and block["name"].endswith(suffix) + ] + assert len(matches) == 1 + use_row, use_block = matches[0] + reply = next( + (row for row in rows if row.get("sourceToolAssistantUUID") == use_row["uuid"]), + None, + ) + assert reply is not None, f"missing tool reply for {suffix}" + result = next( + block + for block in reply.get("message", {}).get("content", []) + if isinstance(block, dict) + and block.get("type") == "tool_result" + and block.get("tool_use_id") == use_block.get("id") + ) + return reply, result + + +def _assert_truncated_prose(value: Any, *, preserve_input: bool = False) -> None: + if isinstance(value, str): + stripped = value.lstrip() + if not preserve_input and stripped.startswith(("{", "[")): + try: + parsed = json.loads(value) + except json.JSONDecodeError: + parsed = None + if isinstance(parsed, (dict, list)): + _assert_truncated_prose(parsed) + return + if not preserve_input and len(value) > 400: + marker = _TRUNCATION.search(value) + assert marker is not None + assert marker.start() == 400 + assert int(marker.group(1)) > 0 + return + if isinstance(value, list): + for item in value: + _assert_truncated_prose(item, preserve_input=preserve_input) + return + if isinstance(value, dict): + keep_full_input = ( + value.get("type") == "tool_use" and value.get("name") in _INPUTS_TO_PRESERVE + ) + for key, item in value.items(): + _assert_truncated_prose( + item, + preserve_input=preserve_input or (key == "input" and keep_full_input), + ) + + +@pytest.fixture +def materialized_run(tmp_path: Path) -> _MaterializedRun: + provenance = load_provenance(FIXTURE_NAME) + trace = tmp_path / "trace" / "nested" / "session-events.jsonl" + log_root = tmp_path / "log-root" + projects = tmp_path / "projects" + envelope = materialize_run( + FIXTURE_NAME, + trace=trace, + log_root=log_root, + projects=projects, + ) + return _MaterializedRun(provenance, trace, log_root, projects, envelope) + + +def test_capture_payload_files_have_no_raw_home_paths_and_keep_wire_shape() -> None: + provenance = load_provenance(FIXTURE_NAME) + # PROVENANCE records the source prefix; payload files must not retain it. + for path in FIXTURE_DIR.rglob("*"): + if path.is_file() and path.name != "PROVENANCE.json": + assert b"/home/" not in path.read_bytes(), path.relative_to(FIXTURE_DIR) + + envelope = json.loads((FIXTURE_DIR / "out/envelope.json").read_text(encoding="utf-8")) + root_transcript = ( + FIXTURE_DIR + / "projects" + / "-workspace-sandbox" + / f"{envelope['dispatched_session_id']}.jsonl" + ) + rows = _rows(root_transcript) + version_rows = [row["version"] for row in rows if "version" in row] + assert version_rows + assert all(version == provenance["cli_version"] for version in version_rows) + + tool_uses = [ + block + for row in rows + for block in row.get("message", {}).get("content", []) + if isinstance(block, dict) and block.get("type") == "tool_use" + ] + autoskillit_names = [ + block["name"] + for block in tool_uses + if isinstance(block.get("name"), str) and "autoskillit" in block["name"] + ] + assert autoskillit_names + assert all(name.startswith(PLUGIN_PREFIX) for name in autoskillit_names) + + for suffix in ("claim_and_resolve_issue", "create_and_publish_branch"): + reply, result_block = _tool_reply(rows, suffix) + structured = reply["mcpMeta"]["structuredContent"]["result"] + assert isinstance(json.loads(structured), dict) + with pytest.raises(json.JSONDecodeError): + json.loads(result_block["content"]) + + bootstrap_reply, _ = _tool_reply(rows, "bootstrap_clone") + bootstrap_result = json.loads(bootstrap_reply["mcpMeta"]["structuredContent"]["result"]) + assert bootstrap_result["merge_target"] == "main" + for transcript_path in sorted((FIXTURE_DIR / "projects").rglob("*.jsonl")): + _assert_truncated_prose(_rows(transcript_path)) + + +def test_captured_run_proves_claim_and_published_branch( + materialized_run: _MaterializedRun, +) -> None: + provenance = materialized_run.provenance + trace = materialized_run.trace + log_root = materialized_run.log_root + envelope = materialized_run.envelope + + proof = sessions.collect_ownership( + trace, + envelope, + log_root, + provenance["issue_url"], + provenance["repository"], + claim_label=provenance["claim_label"], + ) + assert proof["branches"] == ["e2e-implementation-fixture-normalize-whitespace/1-3"] + assert proof["claimed"] is True + assert proof["violations"] == [] + assert envelope["l3_payload"] is None + + +def test_captured_run_accounts_for_successful_and_failed_workers( + materialized_run: _MaterializedRun, +) -> None: + provenance = materialized_run.provenance + trace = materialized_run.trace + log_root = materialized_run.log_root + envelope = materialized_run.envelope + + summary = sessions.summarize(trace, envelope, log_root) + coverage = summary["coverage"] + assert coverage["managed_session_ids"] == sorted(WORKER_SESSION_IDS) + assert coverage["owned_session_ids"] == sorted((ROOT_SESSION_ID, *WORKER_SESSION_IDS)) + assert summary["peak_sessions"] == 5 + assert sorted(agent["agent_id"] for agent in coverage["native_agents"]) == sorted( + NATIVE_AGENT_IDS + ) + assert coverage["complete"] is False + assert summary["violations"] == sorted( + [ + f"lifecycle:agent:{ROOT_SESSION_ID}:{UNMATCHED_AGENT_ID}:orphan_end", + f"native:{ROOT_SESSION_ID}:{UNMATCHED_AGENT_ID}:lifecycle_incomplete", + f"native:{ROOT_SESSION_ID}:{UNMATCHED_AGENT_ID}:transcript_unenumerated", + f"capture:{ROOT_SESSION_ID}:{UNMATCHED_AGENT_ID}:agent_type_missing", + ] + ) + assert envelope["l3_payload"] is None + + ownership = sessions.collect_ownership( + trace, + envelope, + log_root, + provenance["issue_url"], + provenance["repository"], + claim_label=provenance["claim_label"], + ) + assert [item["session_id"] for item in ownership["transcripts"]] == sorted( + (ROOT_SESSION_ID, *WORKER_SESSION_IDS) + ) + + +def test_captured_run_is_complete_under_the_declared_capture_policy( + materialized_run: _MaterializedRun, +) -> None: + trace = materialized_run.trace + log_root = materialized_run.log_root + envelope = materialized_run.envelope + events = _rows(trace) + admitted = [] + excluded: list[tuple[str, str, str]] = [] + for event in events: + matcher = sessions.LIFECYCLE_HOOK_MATCHERS[event["event"]] + agent_type = event.get("agent_type", "") + if claude_matcher_admits(matcher, agent_type): + admitted.append(event) + else: + excluded.append((event["event"], event.get("agent_id", ""), agent_type)) + assert excluded == [("SubagentStop", UNMATCHED_AGENT_ID, "")] + trace.write_text( + "".join( + json.dumps(event, separators=(",", ":"), ensure_ascii=False) + "\n" + for event in admitted + ), + encoding="utf-8", + ) + + summary = sessions.summarize(trace, envelope, log_root) + assert summary["coverage"]["complete"] is True + assert summary["violations"] == [] + assert summary["peak_sessions"] == 5 + assert sorted(agent["agent_id"] for agent in summary["coverage"]["native_agents"]) == sorted( + NATIVE_AGENT_IDS + ) diff --git a/tests/infra/test_e2e_workflow.py b/tests/infra/test_e2e_workflow.py index ede505cd35..896a963343 100644 --- a/tests/infra/test_e2e_workflow.py +++ b/tests/infra/test_e2e_workflow.py @@ -382,9 +382,16 @@ def test_docker_run_forwards_secrets_by_name_only(self, workflow: dict, step_id: forwarded = re.findall(r"--env ([A-Z0-9_]+)", run) expected = ["MINIMAX_API_KEY", "E2E_SANDBOX_TOKEN"] if step_id == "e2e": - expected.append("E2E_SANDBOX_TOKEN") + expected.extend(["E2E_SANDBOX_TOKEN", "E2E_IMPLEMENTATION_ISSUE_URL"]) assert Counter(forwarded) == Counter(expected) + def test_seed_repository_variable_reaches_test_container(self, workflow: dict) -> None: + step = _step(workflow, "run", step_id="e2e") + assert ( + step["env"]["E2E_IMPLEMENTATION_ISSUE_URL"] + == "${{ vars.E2E_IMPLEMENTATION_ISSUE_URL }}" + ) + def test_redaction_keeps_required_secret_names(self, workflow: dict) -> None: run = _step(workflow, "run", step_id="redact")["run"] branch, outside = _model_branch(run) @@ -433,10 +440,10 @@ def test_clean_install_uses_the_common_user_image_and_artifacts( ] assert len(credential_lines) == 1 assert credential_lines[0].startswith("docker_args+=(") - assert re.findall(r"--env ([A-Z0-9_]+)", credential_lines[0]) == [ - "MINIMAX_API_KEY", - "E2E_SANDBOX_TOKEN", - ] + forwarded = ["MINIMAX_API_KEY", "E2E_SANDBOX_TOKEN"] + if step_id == "e2e": + forwarded.append("E2E_IMPLEMENTATION_ISSUE_URL") + assert re.findall(r"--env ([A-Z0-9_]+)", credential_lines[0]) == forwarded if step_id == "e2e": assert not re.search(r"--env MINIMAX_API_KEY\b", outside) assert re.findall(r"--env ([A-Z0-9_]+)", outside) == ["E2E_SANDBOX_TOKEN"] diff --git a/tests/server/test_e2e_sessions_contract.py b/tests/server/test_e2e_sessions_contract.py new file mode 100644 index 0000000000..f68ceff369 --- /dev/null +++ b/tests/server/test_e2e_sessions_contract.py @@ -0,0 +1,221 @@ +"""Check the standalone ownership parser against live MCP tool response schemas.""" + +from pathlib import Path +from typing import Any + +import pytest +from mcp.types import CallToolResult + +from tests._script_loader import load_script + +pytestmark = [pytest.mark.layer("server"), pytest.mark.medium] + +sessions = load_script( + "_autoskillit_e2e_sessions_mcp_contract", + Path(__file__).resolve().parents[2] / "scripts" / "e2e" / "e2e_sessions.py", +) + + +def _parsed_response(response: Any) -> dict[str, Any] | None: + wire_result = CallToolResult( + content=response.content, structuredContent=response.structured_content + ).model_dump(mode="json", by_alias=True, exclude_none=True) + return sessions._result_object(wire_result) + + +@pytest.mark.anyio +async def test_ownership_parser_accepts_actual_product_tool_envelopes( + tool_ctx_kitchen_open: Any, + kitchen_enabled: Any, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + from unittest.mock import AsyncMock + + import autoskillit.server as server + import autoskillit.server._misc as server_misc + from autoskillit.core import GitHubFetcher + from autoskillit.server.tools import tools_git, tools_issue_composite + + async def resolve_remote(_work_dir: str) -> str: + return "origin" + + async def create_branch(_base_name: str, _remote: str, _work_dir: str) -> dict[str, Any]: + return {"branch_name": "impl-contract-42", "was_unique": True, "base_ref": "main"} + + class CloneManager: + push_result: dict[str, Any] = {"success": True} + + def push_to_remote(self, *_args: object, **_kwargs: object) -> dict[str, Any]: + return self.push_result + + clone_manager = CloneManager() + monkeypatch.setattr(tool_ctx_kitchen_open, "clone_mgr", clone_manager) + monkeypatch.setattr(server_misc, "resolve_remote_name", resolve_remote) + monkeypatch.setattr(tools_git, "_resolve_and_create_branch", create_branch) + + branch_args: dict[str, Any] = { + "issue_slug": "contract", + "run_name": "impl", + "issue_number": "42", + "work_dir": str(tmp_path), + "remote_url": "https://github.com/owner/repo.git", + } + branch_response = await server.mcp.call_tool( + "create_and_publish_branch", branch_args, run_middleware=False + ) + branch_result = _parsed_response(branch_response) + assert branch_result is not None + branch_violations: list[str] = [] + assert ( + sessions._branch_target( + "root-session", + "root-session", + "create_and_publish_branch", + branch_args, + branch_result, + False, + True, + "42", + "owner/repo", + branch_violations, + ) + == "impl-contract-42" + ) + assert not branch_violations + + clone_manager.push_result = { + "success": False, + "stderr": "push failed", + "error_type": "PushError", + } + failed_branch_response = await server.mcp.call_tool( + "create_and_publish_branch", branch_args, run_middleware=False + ) + failed_branch_result = _parsed_response(failed_branch_response) + assert failed_branch_result is not None + branch_violations = [] + assert ( + sessions._branch_target( + "root-session", + "root-session", + "create_and_publish_branch", + branch_args, + failed_branch_result, + False, + True, + "42", + "owner/repo", + branch_violations, + ) + is None + ) + assert branch_violations == ["ownership:branch_result_failed"] + + issue_url = "https://github.com/owner/repo/issues/42" + claim_label = tool_ctx_kitchen_open.config.github.in_progress_label + github_client = AsyncMock(spec_set=GitHubFetcher) + github_client.fetch_title = AsyncMock( + return_value={"success": True, "number": 42, "title": "Contract", "slug": "contract"} + ) + github_client.fetch_issue = AsyncMock( + side_effect=[ + {"success": True, "state": "open", "labels": [], "body": ""}, + { + "success": True, + "state": "open", + "labels": [{"name": claim_label}], + "body": "", + }, + ] + ) + github_client.ensure_label = AsyncMock(return_value={"success": True}) + github_client.swap_labels = AsyncMock(return_value={"success": True}) + monkeypatch.setattr(tool_ctx_kitchen_open, "github_client", github_client) + monkeypatch.setattr(tools_issue_composite, "_get_campaign_state_paths", lambda _ctx: []) + + claim_args: dict[str, Any] = { + "issue_url": issue_url, + "label": claim_label, + "allow_reentry": False, + } + claim_response = await server.mcp.call_tool( + "claim_and_resolve_issue", claim_args, run_middleware=False + ) + claim_result = _parsed_response(claim_response) + assert claim_result is not None + claim_violations: list[str] = [] + assert sessions._claim_succeeded( + [(claim_args, claim_result, False)], "42", claim_label, claim_violations + ) + assert not claim_violations + + reentry_args: dict[str, Any] = { + "issue_url": issue_url, + "label": claim_label, + "allow_reentry": True, + } + reentry_response = await server.mcp.call_tool( + "claim_and_resolve_issue", reentry_args, run_middleware=False + ) + reentry_result = _parsed_response(reentry_response) + assert reentry_result is not None + assert reentry_result.get("reentry") is True + claim_violations = [] + assert not sessions._claim_succeeded( + [(reentry_args, reentry_result, False)], "42", claim_label, claim_violations + ) + assert claim_violations == ["ownership:claim_result_failed"] + + +@pytest.mark.anyio +async def test_completion_receipt_envelope_matches_collector_launch_contract( + tool_ctx_kitchen_open: Any, + kitchen_enabled: Any, + monkeypatch: pytest.MonkeyPatch, +) -> None: + import autoskillit.server as server + from autoskillit.server.tools.tools_pipeline_tracker import _handlers + + authority = tool_ctx_kitchen_open.run_skill_completion + assert authority is not None + monkeypatch.setenv("AUTOSKILLIT_SESSION_TYPE", "orchestrator") + monkeypatch.setenv("AUTOSKILLIT_HEADLESS", "0") + request_session_id = "contract-request-session" + monkeypatch.setattr( + _handlers, + "_request_session_identity", + lambda _context: request_session_id, + ) + invocation_id = authority.begin( + kitchen_id=tool_ctx_kitchen_open.kitchen_id, + request_session_id=request_session_id, + tracker_order_id="", + tracker_path="", + tracker_kitchen_id="", + tracker_incarnation_id="", + step_name="contract", + ) + receipt = authority.draft( + invocation_id, + classification="success", + success=True, + result_digest="contract-result", + child_session_id="child-contract-session", + ) + authority.publish(receipt.receipt_id) + + response = await server.mcp.call_tool( + "complete_run_skill_result", + {"receipt_id": receipt.receipt_id}, + run_middleware=False, + ) + reply = _parsed_response(response) + + assert reply is not None + assert reply["success"] is True + assert reply["receipt_id"] == receipt.receipt_id + assert reply["session_id"] == "child-contract-session" + assert sessions._launched_session_ids( + {"root-session": {"calls": [("complete_run_skill_result", {}, reply, False)]}} + ) == {"child-contract-session"} diff --git a/tests/server/test_server_version_telemetry.py b/tests/server/test_server_version_telemetry.py index e80e1a4df5..eb7c33ec5d 100644 --- a/tests/server/test_server_version_telemetry.py +++ b/tests/server/test_server_version_telemetry.py @@ -160,7 +160,7 @@ def test_initialize_uses_clear_marker_as_since_bound(self, tool_ctx, tmp_path, m from datetime import UTC, datetime, timedelta from autoskillit.server.lifecycle import _state - from tests.execution.conftest import _flush + from tests._session_log_helpers import _flush log_dir = tmp_path / "logs" log_dir.mkdir() diff --git a/tests/server/test_tools_github_api_tracking.py b/tests/server/test_tools_github_api_tracking.py index 26db7df056..1ac2d06f77 100644 --- a/tests/server/test_tools_github_api_tracking.py +++ b/tests/server/test_tools_github_api_tracking.py @@ -105,7 +105,7 @@ async def test_flush_session_log_writes_github_api_usage(tmp_path): import json from autoskillit.pipeline.github_api_log import DefaultGitHubApiLog - from tests.execution.conftest import _flush + from tests._session_log_helpers import _flush log = DefaultGitHubApiLog() await log.record_httpx(