diff --git a/.github/workflows/conformance-probes.yml b/.github/workflows/conformance-probes.yml index fcd5fff142..ed1bfde194 100644 --- a/.github/workflows/conformance-probes.yml +++ b/.github/workflows/conformance-probes.yml @@ -194,10 +194,10 @@ jobs: matrix: include: - os: ubuntu-22.04 - claude-version: "2.1.280" + claude-version: "2.1.296" probe-tmpdir: /dev/shm/pytest-probes - os: macos-14 - claude-version: "2.1.280" + claude-version: "2.1.296" probe-tmpdir: /tmp/pytest-probes env: CLAUDE_CODE_SMOKE_TEST: "1" diff --git a/docs/README.md b/docs/README.md index b5dc600555..83d15cc892 100644 --- a/docs/README.md +++ b/docs/README.md @@ -37,4 +37,5 @@ multi-level orchestrator. The bundled recipes implement issue → plan → workt - [audit/surface-freeze-checklist.md](audit/surface-freeze-checklist.md) — commands.py public import surface freeze checklist - [verification/review-pr-immunity.md](verification/review-pr-immunity.md) — deterministic review-pr projection matrix and live provider-attempt record - [verification/child-terminal-reasons-4623.md](verification/child-terminal-reasons-4623.md) — child-terminal-reason taxonomy implementation and test-coverage verification record +- [verification/model-pins-5276.md](verification/model-pins-5276.md) — Claude 5.5 model pins and the Claude Code 2.1.296 floor: launch, effort, canary and resume record - [phoropter/](phoropter/README.md) — phoropter lens framework: execution contracts, recipe blocks, synthesis strategies, and authoring guide diff --git a/docs/configuration.md b/docs/configuration.md index 945697de3e..5ca68050af 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -32,7 +32,7 @@ test_check: command: ["task", "test-check"] model: - default: "claude-sonnet-5" + default: "claude-sonnet-5-5" worktree_setup: command: ["task", "install-worktree"] @@ -70,7 +70,7 @@ AutoSkillit uses [dynaconf](https://www.dynaconf.com/) for configuration. Enviro ```bash AUTOSKILLIT_TEST_CHECK__TIMEOUT=300 # sets test_check.timeout to 300 -AUTOSKILLIT_MODEL__DEFAULT=claude-sonnet-5 # sets model.default +AUTOSKILLIT_MODEL__DEFAULT=claude-sonnet-5-5 # sets model.default AUTOSKILLIT_QUOTA_GUARD__ENABLED=false # disables quota guard ``` @@ -102,13 +102,31 @@ model: Recipe steps can also specify a `model` field per-step in their YAML definition. -The logical `opus` class resolves to the canonical `claude-opus-5-5` model ID by -default. Projects that route Claude through a provider whose deployment does -not accept that raw ID (Amazon Bedrock, Microsoft Foundry, or a custom -gateway) should supply the provider's own accepted deployment/model -identifier via a `providers.profiles` entry and a matching `step_overrides` / -`recipe_overrides` / `model_overrides` binding — those overrides are resolved -before this alias and always take precedence. +AutoSkillit pins the model classes to canonical Anthropic IDs: `sonnet` resolves to +`claude-sonnet-5-5`, `opus` to `claude-opus-5-5`, and `haiku` to `claude-haiku-5-5`. +Claude Code's own aliases can select older versions on Claude Platform on AWS, +Amazon Bedrock, Google Cloud's Agent Platform (Vertex AI), and Microsoft Foundry. +The supported minimum is Claude Code 2.1.296: interactive launches below it fail, +and `autoskillit doctor` warns. + +[Claude Code's model configuration](https://code.claude.com/docs/en/model-config) +describes provider mapping. On Amazon Bedrock and Google Cloud's Agent Platform, +Claude Code maps an Anthropic ID passed with `--model` to the provider's ID when +that provider serves the version. For a different deployment name, its +`modelOverrides` setting maps each Anthropic ID to a Microsoft Foundry deployment, +Bedrock inference profile ARN, or custom gateway model name. This applies to every +`--model` AutoSkillit passes, including triage. + +AutoSkillit can select a step's model through `model.step_overrides`, +`model.recipe_overrides`, or `providers.model_overrides`. Values can be literal IDs +or names in `providers.profiles`; profiles resolve only with the experimental +`providers` feature enabled. See [Model precedence](#model-precedence): +`model.override` outranks these sources, and each ladder ignores the other's sources. + +Contract-staleness triage always uses the `haiku` class ID and ignores AutoSkillit's +model overrides. The pre-launch version check does not cover triage. If the provider +rejects the ID without a matching `modelOverrides` entry, or `claude` on `PATH` is +older than 2.1.296, triage treats every changed skill as meaningful. ### Operator escape hatches @@ -548,7 +566,7 @@ test_check: command: ["pytest", "-v", "--tb=short"] model: - default: "claude-sonnet-5" + default: "claude-sonnet-5-5" worktree_setup: command: ["task", "install-worktree"] diff --git a/docs/design/acp-session-contract.md b/docs/design/acp-session-contract.md index d36b7e0b95..bf0d8ecefb 100644 --- a/docs/design/acp-session-contract.md +++ b/docs/design/acp-session-contract.md @@ -374,7 +374,7 @@ future use and have no current production consumer outside the exemption set. | `required_session_files` | Session directory contract enforcement (Codex: `frozenset({"config.toml"})`; Claude: `frozenset()`) | | `session_dir_symlinks` | Session directory layout (Codex: `frozenset({"sessions", "archived_sessions"})`; Claude: `frozenset()`) | | `patch_format` | Write-guard path extraction (Claude: `"unified_diff"`; Codex: `"codex_star_update"`) | -| `min_version` | Version validation in doctor (Codex: `"0.156.1"`; Claude: `"2.1.280"`) | +| `min_version` | Version validation in doctor (Codex: `"0.156.1"`; Claude: `"2.1.296"`) | | `mcp_env_forward_vars` | MCP env forwarding (Codex: `CODEX_MCP_ENV_FORWARD_VARS = AUTOSKILLIT_PRIVATE_ENV_VARS - CODEX_MCP_ENV_SERVER_EXCLUDED_VARS`; Claude Code inherits the full environment and declares `frozenset()`). MCP registration forwards each listed variable present in the agent process; the Codex MCP server env and builder coverage architecture tests enforce the contract. | | `github_api_callable` | Future network-capability gate for outbound GitHub API writes | diff --git a/docs/design/forward-obligations/triage-portability-impact.md b/docs/design/forward-obligations/triage-portability-impact.md index fb1f0b6b5f..0b4e83dac2 100644 --- a/docs/design/forward-obligations/triage-portability-impact.md +++ b/docs/design/forward-obligations/triage-portability-impact.md @@ -14,13 +14,14 @@ an LLM subprocess. ## Current State -- `ClaudeCodeBackend`: `triage_capable=True` (set in `CLAUDE_CODE_CAPABILITIES`, - `_type_backend.py:229`). Uses a `claude -p` subprocess with Haiku model. -- `CodexBackend`: `triage_capable=False` (`codex.py:559`). Triage is skipped entirely. +- `ClaudeCodeBackend`: `triage_capable=True` (set in `CLAUDE_CODE_CAPABILITIES`). Uses + a `claude -p` subprocess with the `haiku` model class resolved through + `backend.translate_model("haiku")` (`claude-haiku-5-5`). +- `CodexBackend`: `triage_capable=False` (set in `CodexBackend.capabilities`). Triage is skipped entirely. ## Contract Gaps -`_triage_batch()` (`_llm_triage.py:80`) constructs the triage command at lines 123–131: +`_triage_batch()` constructs the triage command directly: ```python triage_cmd: list[str] = [ @@ -28,7 +29,7 @@ triage_cmd: list[str] = [ ClaudeFlags.PRINT, prompt, ClaudeFlags.MODEL, - "claude-haiku-4-5-20251001", + backend.translate_model("haiku"), ClaudeFlags.OUTPUT_FORMAT, fmt.value, ] @@ -36,8 +37,7 @@ triage_cmd: list[str] = [ The command uses `backend.binary_name()` (not a hardcoded `"claude"` literal) but depends on `ClaudeFlags` constants (`PRINT`, `MODEL`, `OUTPUT_FORMAT`) which are Claude-CLI-specific. -The model `"claude-haiku-4-5-20251001"` is hardcoded. Portability to a non-Claude backend -requires either: +The backend owns the model ID. Portability to a non-Claude backend still requires either: - A backend-agnostic `triage_cmd()` protocol method on `CodingAgentBackend`, or - A dedicated lightweight triage backend decoupled from the coding-agent backend. @@ -45,21 +45,21 @@ requires either: ## Required Steps for Portable Triage 1. Add an optional `triage_cmd(prompt: str, model: str) -> list[str]` method to - `CodingAgentBackend` (or add a `BackendCapabilities.triage_model` field). + `CodingAgentBackend`. 2. Update `_triage_batch` to call `backend.triage_cmd()` instead of constructing the command inline with `ClaudeFlags`. -3. Gate on `triage_capable` (already done at `_llm_triage.py:107`). +3. Gate on `triage_capable` (already done in `_triage_batch`). ## Blast Radius `_llm_triage.py` is a standalone module at the package root. Changes are isolated and do not touch `BackendCapabilities` layout or the sub-protocol set. The `triage_capable` field on -`BackendCapabilities` (`_type_backend.py:86`) already has production consumers (the guard at -`_llm_triage.py:107`), so no `_FORWARD_DECLARED` exemption is needed. +`BackendCapabilities` already has a production consumer in `_triage_batch`, so no +`_FORWARD_DECLARED` exemption is needed. ## Key References -- `_llm_triage.py:33` — `triage_staleness()` entry point -- `_llm_triage.py:80` — `_triage_batch()` with command construction -- `_llm_triage.py:107` — `triage_capable` guard -- `core/types/execution/_type_backend.py:86` — `triage_capable` field definition +- `triage_staleness()` — entry point in `_llm_triage.py` +- `_triage_batch()` — command construction in `_llm_triage.py` +- `_triage_batch()`'s `triage_capable` guard — backend admission in `_llm_triage.py` +- `BackendCapabilities.triage_capable` — field definition in `core/types/execution/_type_backend.py` diff --git a/docs/execution/claude-startup-readiness.md b/docs/execution/claude-startup-readiness.md index 6836eed870..90ff0c860e 100644 --- a/docs/execution/claude-startup-readiness.md +++ b/docs/execution/claude-startup-readiness.md @@ -18,7 +18,7 @@ Claude interactive launches seal these values after caller environment overlays: |---|---:|---| | `MCP_CONNECTION_NONBLOCKING` | `0` | Ask Claude to wait for the connection batch | | `MCP_CONNECT_TIMEOUT_MS` | `30000` | Bound the connection batch and tool-list snapshot | -| minimum Claude Code version | `2.1.280` | [Anthropic's 2.1.280 release](https://github.com/anthropics/claude-code/releases/tag/v2.1.280) adds Claude Opus 5.5, the pinned `opus` target | +| minimum Claude Code version | `2.1.296` | The lowest release that serves every pinned model: 2.1.280 adds Opus 5.5, 2.1.284 Sonnet 5.5 and 2.1.293 Haiku 5.5 ([model configuration](https://code.claude.com/docs/en/model-config)), and [2.1.296](https://github.com/anthropics/claude-code/releases/tag/v2.1.296) fixes Haiku 5.5 token counts behind gateways | | fresh prompt attempts | `3` | Defense-in-depth dispatch cap after a pre-dispatch failure | `MCP_CONNECT_TIMEOUT_MS` is the connection batch/list-snapshot deadline. It is @@ -73,8 +73,8 @@ The weekly `conformance-probes.yml` workflow runs these pinned rows: | Platform | Claude Code | |---|---| -| Ubuntu 22.04 | 2.1.280 | -| macOS 14 | 2.1.280 | +| Ubuntu 22.04 | 2.1.296 | +| macOS 14 | 2.1.296 | Each row installs the named version, exports its canonical path as `CLAUDE_CODE_EXECPATH`, and runs the exact restricted interactive surface with diff --git a/docs/skills/overrides.md b/docs/skills/overrides.md index 2594101763..fd5231143d 100644 --- a/docs/skills/overrides.md +++ b/docs/skills/overrides.md @@ -66,7 +66,8 @@ When an override is rejected, every AutoSkillit-mediated surface — `run_skill` `/autoskillit:`, and the session add-dir — falls back to serving the bundled twin. Claude Code, however, still discovers the project's `.claude/skills/` directory natively as `/`, independent of AutoSkillit's own admission decision. Claude Code -2.1.280 was observed to prefer that native copy over a same-named add-dir copy. This is +2.1.280, an earlier AutoSkillit support floor, was observed to prefer that native copy over a +same-named add-dir copy; the observation has not been repeated on later releases. This is observed behavior, not a documented contract, and it was verified for Claude Code only. An admitted override is also published into the add-dir, so both the native diff --git a/docs/verification/model-pins-5276.md b/docs/verification/model-pins-5276.md new file mode 100644 index 0000000000..a8328927ed --- /dev/null +++ b/docs/verification/model-pins-5276.md @@ -0,0 +1,227 @@ +# Claude 5.5 Model Pins Verification (#5276) + +Date: 2026-10-25 +Base: `f9d103002` (`develop`) +Client: `2.1.296 (Claude Code)` + +The evidence client was installed with +`npm install --prefix .autoskillit/temp/5276/cc @anthropic-ai/claude-code@2.1.296`. +Raw stdout, stderr, and stub requests are retained under `.autoskillit/temp/5276/` +in the original checkout. They are local artifacts, excluded from the commit. + +## Isolation and authentication + +Every probe used `env -i`, preserving only `PATH`, an explicit fresh `HOME`, and +the credentials or stub routing needed by that probe. Working directories were +`/`, outside either checkout. For tool checks, notes and an empty extra directory +lived under the evidence directory and prompts used the absolute notes path; +`--allowedTools Read` enabled the read without an interactive permission prompt. +Each start/resume pair shared its isolated home and working directory. + +The first-party credential was the existing Claude login's access token, passed +as `CLAUDE_CODE_OAUTH_TOKEN`. No provider endpoint override was supplied. MiniMax's +token came from AutoSkillit's merged provider configuration, using +`load_config(project_root).providers.profiles["minimax"]["ANTHROPIC_AUTH_TOKEN"]`. +The E2E harness's `write_claude_settings` wrote its exact provider settings into a +separate home, with no first-party credential in that process's environment. +That home was deleted after the canaries. Credential values are not in this record. + +No probe supplied `--effort`, `CLAUDE_CODE_EFFORT_LEVEL`, `CLAUDE_CONFIG_DIR`, or a +saved `effortLevel`. User and project settings did not participate. + +## Decisions + +- Pin all classes: `sonnet` → `claude-sonnet-5-5`, `opus` → `claude-opus-5-5`, + `haiku` → `claude-haiku-5-5`. Triage uses the backend's translation of `haiku`. + The [provider alias table](https://code.claude.com/docs/en/model-config#model-aliases) + documents provider-dependent aliases, which can select older versions. +- Set the support floor to 2.1.296. The documented per-model minimums are + 2.1.280 for Opus 5.5, 2.1.284 for Sonnet 5.5, and 2.1.293 for Haiku 5.5. + [Release 2.1.296](https://github.com/anthropics/claude-code/releases/tag/v2.1.296) + fixes adaptive-thinking token counts, including Haiku behind gateways. + The below-floor fixture is 2.1.295. The registry test enforces #5157's rule + that the client floor must support every pinned model; image and CI tests + tie their pins to that floor. +- Keep effort unset. The measurements below establish each class's default, + rather than choosing effort for AutoSkillit. Whether `medium` is suitable + remains a per-class judgement; an unsuitable class needs a follow-up issue. + These short probes do not establish coding-task quality. +- Remove `[1m]` from the E2E Claude model. The successful canaries below reported + native 1M windows. Configured suffixes remain supported by model translation; + `sonnet[1m]` would still become `claude-sonnet-5-5[1m]`. The unrecognized + `MiniMax-M3[1m]` provider ID keeps its explicit window declaration. +- Couple pins to minimum client releases in + `CLAUDE_MODEL_MIN_CLIENT_VERSIONS` in the alias-registry tests. The Haiku entry + requires the token-count correction release, beyond its launch minimum. +- Keep #4412 open: this change pins Haiku but adds neither an alias-freshness + marker nor a doctor freshness check. #5274's suffix removal accompanies the pins. + +## Authenticated first-party launches + +Each command was `claude -p "Reply with exactly OK" --output-format stream-json +--verbose --model `. All replies were `OK`. The init record, command argument, +and sole usage key agreed on the requested ID; assistant records independently +reported the served model. + +| Class | Requested ID and usage key | Served assistant model | Context window | Maximum output | is_error | Credential | +|---|---|---|---:|---:|---|---| +| sonnet | `claude-sonnet-5-5` | `claude-sonnet-5-5` | 1000000 | 128000 | false | Claude OAuth | +| opus | `claude-opus-5-5` | `claude-opus-5-5` | 1000000 | 128000 | false | Claude OAuth | +| haiku | `claude-haiku-5-5` | `claude-haiku-5-5` | 1000000 | 128000 | false | Claude OAuth | + +## Effort with no user-level setting + +A stdlib HTTP server on `127.0.0.1` accepted Messages requests, ignoring the query +string when matching paths. It returned a token count or a minimal streaming +text reply. Each client received only its isolated home, stub endpoint, and +dummy API key. The recorded requests contained the prompt `hi` and the requested +model. Only the beta, API-version, and user-agent headers were retained. + +| Class | Requested model | output_config.effort | anthropic-beta | +|---|---|---|---| +| sonnet | `claude-sonnet-5-5` | `medium` | Sonnet/Opus value below | +| opus | `claude-opus-5-5` | `medium` | Sonnet/Opus value below | +| haiku | `claude-haiku-5-5` | `medium` | Haiku value below | + +Sonnet/Opus beta header, in observed order: + +```text +claude-code-20250219,interleaved-thinking-2025-05-14,thinking-token-count-2026-05-13,context-management-2025-06-27,prompt-caching-scope-2026-01-05,mid-conversation-system-2026-04-07,per-turn-control-2026-07-01,mid-conversation-tool-changes-2026-07-01,inline-tools-2026-09-15,advisor-tool-2026-03-01,effort-2025-11-24 +``` + +Haiku beta header, in observed order: + +```text +interleaved-thinking-2025-05-14,thinking-token-count-2026-05-13,context-management-2025-06-27,prompt-caching-scope-2026-01-05,mid-conversation-system-2026-04-07,claude-code-20250219,per-turn-control-2026-07-01,mid-conversation-tool-changes-2026-07-01,inline-tools-2026-09-15,advisor-tool-2026-03-01,effort-2025-11-24 +``` + +No context-1m beta was present. The stub measures what Claude Code sends; it does +not measure how a provider executes that effort setting. + +## MiniMax canary + +The harness settings supplied `https://api.minimax.io/anthropic` and its auth token. +Commands used the harness canary prompt, JSON output, and the explicit IDs below. +Both returned `AUTOSKILLIT-E2E-CANARY-OK` with is_error literally false. + +| Requested ID | Token present | All usage keys | Reported context window | +|---|---|---|---:| +| `claude-sonnet-5-5` | yes | `claude-sonnet-5-5` | 1000000 | +| `claude-haiku-5-5` | yes | `claude-haiku-5-5` | 1000000 | + +These replies establish the configured MiniMax route accepts the requests. The +usage metadata is Claude Code's assigned window, not proof of a million-token +request succeeding at the gateway. The harness retains its observed-window checks. +JSON result envelopes did not expose an assistant model for these canaries. + +## Completion marker and resume + +Each marker probe read the notes with the Read tool and emitted `%%ORDER_UP%%` +alone on the final line. Both the final result text and an assistant text block +contained that standalone line, covering the two completion-detection channels. + +| Model | Read tool used | Result marker | Assistant text marker | +|---|---|---|---| +| `claude-sonnet-5-5` | yes | pass | pass | +| `claude-opus-5-5` | yes | pass | pass | +| `claude-haiku-5-5` | yes | pass | pass | + +Sonnet 5.5 and Haiku 5.5 each started with a notes-read request and `--add-dir`. +Each then resumed with `--resume`, `--dangerously-skip-permissions`, JSON output, +and neither `--model` nor `--add-dir`. Both succeeded and returned `RESUME-OK`. +Haiku's resume usage included 338 thinking tokens; no thinking-block rejection +occurred. This is a short-session check, not exhaustive transcript compatibility. + +A session started with `--model claude-sonnet-5` then resumed with +`--model claude-sonnet-5-5`. It returned `UPGRADE-OK`, is_error false. Its init +model and new usage entry were `claude-sonnet-5-5`, and the served assistant model +was also `claude-sonnet-5-5`. Cumulative usage retained the earlier Sonnet 5 entry. + +### AutoSkillit resume refusals + +The managed adapter stores the configured class in `physical_model`. Claude's +headless skill launch selects `PluginLoadMode.EXPLICIT_PLUGIN_DIR` in +`execution/headless/_managed/_attempt.py::_headless_plugin_load_mode`: both +skill injection and plugin installation are supported. That mode consumes an +artifact, so `core/runtime/artifact_lease.py::plugin_launch_binding_scope` +acquires a real binding. `_headless_launch.py::_run_headless_attempt` replaces +the preliminary launch preparation's plugin identity with `_binding_identity(binding)` +before finalization and resume validation. The resulting identity includes +`incarnation_id` and `artifact_digest`. + +An existing persisted Claude headless-skill manifest corroborates the binding: +`~/.local/share/autoskillit/logs/skill-session-contracts/sessions/7c392e2dec109c442d5b39eccd1bbab78a2166a7b858d99102b28bb36b03c860/manifest.json` +records `explicit_plugin_dir`, incarnation `fd2d6ec8d0484031bea2b8f450e3be2b`, and +artifact digest `88ca7e7b590bdc2fbdbcc2063ad29989932eab10b27a25faed78ed3cac2eef3c`. +The selected metadata is retained as `.autoskillit/temp/5276/plugin-binding-evidence.json`. +This historical manifest uses launch schema 3; it was inspected as binding evidence, +not migrated into the current schema 5 validator. + +`execution/runtime/launch_resolution.py::DefaultLaunchResolver.validate_resume` +establishes these independent fail-closed conditions. `_run_headless_attempt` +calls it before dispatching the resumed process: + +| Resume change | Establishing evidence | Refusal diagnostic | +|---|---|---| +| Plugin identity across an installed update | `validate_resume` compares the persisted and current plugin identities. `workspace/_installed/_artifact.py::write_installed_plugin_artifact_manifest_locked` mints a fresh incarnation when none is supplied. The acquired binding carries that incarnation and its artifact digest. | `resume launch plugin identity drifted from persisted contract` | +| Projection for a skill declaring `child_model_policies` | `ClaudeCodeBackend.adapt_skill_semantics` translates the child class into `model_effort_policy`; `SkillSemanticAdaptationResult.digest` includes that policy, and `SkillProjectionBinding.projection_digest` includes its adaptation digests. `validate_resume` compares the persisted and current projection digests. | `resume launch projection digest drifted from persisted contract` | + +This pin change moves both axes: the translated child model changes from Sonnet 5 +to Sonnet 5.5, changing the adaptation and projection digests; publishing the changed +projection under its new key also mints a new plugin incarnation in +`workspace/_projected_artifact/authority.py`. If both identities +drift, the projection comparison precedes the plugin comparison, so the first refusal +can report projection drift. Existing deterministic coverage is in +`tests/execution/test_launch_resolution.py`. + +**Evidence scope and deviation:** these refusals are established by the launch-path +trace and persisted binding metadata. The direct CLI probes above bypass AutoSkillit's +launch machinery. No live installed-update or cross-pin `run_skill` resume was +performed: replacing the shared installation or resuming another job's persisted +session would affect work outside this verification. This documentation correction +preserves runtime behavior and records that limit explicitly; it does not claim +either refusal was observed during the direct CLI probes. + +## Haiku launches + +Triage, haiku-class `run_skill` calls, and recipe-step models now pass +`claude-haiku-5-5`. In-session `Agent(..., model='haiku')` subagents and +`model: haiku` agent definitions still use Claude Code's alias. Triage ignores +AutoSkillit model overrides; Claude Code's own `modelOverrides` can map its ID. +Triage is outside the pre-launch version check. A rejected call, refusal, or +too-old client can produce an error or empty result; parsing then marks all items +meaningful with the summary "Batch triage parse failed." It does not identify the +cause, and no server-side refusal fallback was added. + +The [Haiku context and pricing documentation](https://code.claude.com/docs/en/model-config#haiku-55-context-window-and-pricing) +describes higher per-token rates for prompts above 100K tokens. This session did +not exercise that threshold or refusal classifiers. Existing consumers now select +Haiku 5.5 rather than retaining Haiku 4.5 behavior. + +## Quota guard + +Quota `long_window_patterns` classify usage-API window names in +`execution/quota/_quota_gate.py::_is_long_window`, not model IDs. No quota-pattern +change is needed. + +## Local gate evidence + +For the initial implementation at `05b4e0f4f`, the session completed +`pre-commit run --all-files`, then the configured `task test-local-gate` through +`test_check`, before creating the commit. The gate returned 41,389 passed, +508 skipped, and 13 xfailed. Retained local records are +`.autoskillit/temp/5276/pre-commit-staged.log`, +`.autoskillit/temp/5276/test-check-result.json`, and +`.autoskillit/temp/5276/commit.log`. The ordering is in the original session's +tool-call transcript; these files alone do not independently attest that ordering. + +The audit corrections use a new documentation commit under the repository's +new-commit rule. The original model pins and client floor remain together in +their original commit. + +## Remaining external verification + +The conformance workflow's Ubuntu and macOS rows now select 2.1.296. Their CI runs, +including the live output-budget smoke probe, and the E2E canary workflow need +post-push run URLs and outcomes in the PR description. Local direct CLI probes +above do not substitute for those workflow runs. diff --git a/scripts/docker/Dockerfile b/scripts/docker/Dockerfile index 53ed23a962..5f2bbcd483 100644 --- a/scripts/docker/Dockerfile +++ b/scripts/docker/Dockerfile @@ -12,7 +12,7 @@ FROM node:${NODE_VERSION}-bookworm AS node FROM rust:${RUST_VERSION}-bookworm AS toolchain ARG CODEX_VERSION=0.156.1 -ARG CLAUDE_VERSION=2.1.280 +ARG CLAUDE_VERSION=2.1.296 ARG GO_TASK_VERSION=3.43.3 ARG PRE_COMMIT_VERSION=4.6.2 ARG RIPGREP_VERSION=15.2.0 diff --git a/scripts/e2e/README.md b/scripts/e2e/README.md index cc157b6fa5..79873bf682 100644 --- a/scripts/e2e/README.md +++ b/scripts/e2e/README.md @@ -79,29 +79,35 @@ limit fails without retry. | Setting | Sessions it governs in E2E | |---|---| -| `--model claude-sonnet-5[1m]` (from `model.default: sonnet[1m]`) | Fleet food truck and `run_skill` workers, fresh and resumed; also the canary and in-session `sonnet` subagents, which inherit the parent's id. A recipe step's own `model:` replaces this default. Subagent windows are not in turn rows, so the gate does not observe them | +| `--model claude-sonnet-5-5` (the packaged `model.default: sonnet`) | Fleet food truck and `run_skill` workers, fresh and resumed; also the canary and in-session `sonnet` subagents, which inherit the parent's id. A recipe step's own `model:` replaces this default. Subagent windows are not in turn rows, so the gate does not observe them | | `ANTHROPIC_MODEL` | Contract-nudge resumes, which pass no `--model` and resolve to `MiniMax-M3[1m]`. Their single-envelope output records no turn row, so the gate does not observe it | -| `ANTHROPIC_DEFAULT_HAIKU_MODEL` | A `haiku`-class step (`--model haiku`) and in-session `haiku` subagents | +| `ANTHROPIC_DEFAULT_HAIKU_MODEL` | In-session `haiku` subagents. AutoSkillit's `haiku` class passes `claude-haiku-5-5`, not the alias | | `ANTHROPIC_DEFAULT_OPUS_MODEL` | In-session `opus` subagents. AutoSkillit's `opus` class passes `claude-opus-5-5`, not the alias | | `ANTHROPIC_DEFAULT_SONNET_MODEL` | Provider fallback: AutoSkillit passes no bare `sonnet`, and `sonnet` subagents inherit the parent's id | | `CLAUDE_CODE_AUTO_COMPACT_WINDOW` | Every session, capped at that session's model window | -The live probe confirms the explicit `[1m]` declaration yields 1M on the pinned -2.1.280/custom-endpoint path. [Claude Code 2.1.285](https://github.com/anthropics/claude-code/releases/tag/v2.1.285) -changed custom-endpoint sessions to use Sonnet 5's native 1M window; the -[current model-config docs](https://code.claude.com/docs/en/model-config) describe -`[1m]` as having no effect when Sonnet already has a native 1M window. The harness -retains the suffix while supporting the pinned path. +From [Claude Code 2.1.285](https://github.com/anthropics/claude-code/releases/tag/v2.1.285), +a recognized model with a native 1M window keeps that window behind a custom +`ANTHROPIC_BASE_URL`. At the 2.1.296 floor, the canary measured `claude-sonnet-5-5` +and `claude-haiku-5-5` at 1M without `[1m]`. Adding the suffix would only add the +context-1m beta header. The pin preserves a configured suffix: `sonnet[1m]` would +pass `claude-sonnet-5-5[1m]`, so the harness config drops the suffix itself. + +Claude Code does not recognize `MiniMax-M3`. For an unrecognized ID containing +`[1m]`, it assumes a 1M window, as the +[model-config docs](https://code.claude.com/docs/en/model-config) describe. The +provider env values therefore keep `[1m]`. [Claude Code 2.1.295](https://github.com/anthropics/claude-code/releases/tag/v2.1.295) -documents retrying without a rejected context-1m beta. Do not assume that fallback -exists at the pinned floor or that every older release had the same defect; -attribute failures from the probe's actual responses. The floor change belongs to #5276. +documents retrying without a rejected context-1m beta. Attribute failures from the +probe's actual responses. The harness relies on this provider contract: the Anthropic Messages API at `ANTHROPIC_BASE_URL`, Bearer authentication from `ANTHROPIC_AUTH_TOKEN`, tool use, and -successful calls using the exact `claude-sonnet-5[1m]` id that fleet sessions and the +successful calls using the exact `claude-sonnet-5-5` id that fleet sessions and the canary pass with `--model`, including any beta-header behavior of the pinned client. +Haiku-class steps pass `claude-haiku-5-5`, and opus-class steps pass `claude-opus-5-5`. +A step's own `opus[1m]` keeps its suffix. The canary passes `--model` `FLEET_MODEL` and fails unless `modelUsage[FLEET_MODEL].contextWindow` is 1,000,000. @@ -160,8 +166,6 @@ executes it explicitly. For model tests, `autoskillit-config.yaml` becomes `~/.autoskillit/config.yaml` in the container: -- `model.default: sonnet[1m]` — puts the window declaration on the id AutoSkillit - passes with `--model`; `tests/infra/test_e2e_harness.py` derives `FLEET_MODEL` from it; - `quota_guard.enabled: false` — the guard measures Anthropic account quota through Claude credentials (`~/.claude/.credentials.json`) the container does not have; `fleet run` also gets `--disable-quota-guard`; diff --git a/scripts/e2e/autoskillit-config.yaml b/scripts/e2e/autoskillit-config.yaml index 511b455990..577e8df598 100644 --- a/scripts/e2e/autoskillit-config.yaml +++ b/scripts/e2e/autoskillit-config.yaml @@ -3,9 +3,6 @@ quota_guard: enabled: false -model: - default: sonnet[1m] - features: fleet: true fleet_headless_run: true diff --git a/scripts/e2e/e2e_harness.py b/scripts/e2e/e2e_harness.py index c3cdff3624..ad5ff3c8f3 100644 --- a/scripts/e2e/e2e_harness.py +++ b/scripts/e2e/e2e_harness.py @@ -25,8 +25,8 @@ MINIMAX_BASE_URL = "https://api.minimax.io/anthropic" MINIMAX_MODEL = "MiniMax-M3[1m]" MINIMAX_COMPACT_WINDOW_TOKENS = "1000000" -# tests/infra/test_e2e_harness.py derives this --model value from the shipped config. -FLEET_MODEL = "claude-sonnet-5[1m]" +# tests/infra/test_e2e_harness.py asserts this --model value matches the effective config. +FLEET_MODEL = "claude-sonnet-5-5" REQUIRED_CONTEXT_WINDOW_TOKENS = 1_000_000 # implement runs four concurrent children and resolve-review three to six; a spawn past the # limit fails without retry. diff --git a/src/autoskillit/_llm_triage.py b/src/autoskillit/_llm_triage.py index 28bb3f3654..320b910f71 100644 --- a/src/autoskillit/_llm_triage.py +++ b/src/autoskillit/_llm_triage.py @@ -195,7 +195,7 @@ async def _triage_batch( ClaudeFlags.PRINT, prompt, ClaudeFlags.MODEL, - "claude-haiku-4-5-20251001", + backend.translate_model("haiku"), ClaudeFlags.OUTPUT_FORMAT, fmt.value, ] @@ -219,6 +219,9 @@ async def _triage_batch( logger.warning( "triage_staleness batch parse failed; treating all as meaningful", batch=[i.skill for i in triageable], + returncode=result.returncode, + stderr=result.stderr[:1000], + session_result=session.result[:1000], ) triageable_results = [ {"skill": i.skill, "meaningful": True, "summary": "Batch triage parse failed."} diff --git a/src/autoskillit/core/types/execution/_type_backend.py b/src/autoskillit/core/types/execution/_type_backend.py index f296410c05..ec12155aae 100644 --- a/src/autoskillit/core/types/execution/_type_backend.py +++ b/src/autoskillit/core/types/execution/_type_backend.py @@ -345,11 +345,13 @@ class BackendCapabilities: _CONTEXT_WINDOW_SUFFIX_RE: _re.Pattern[str] = _re.compile(r"\[\d+[mk]?\]$", _re.IGNORECASE) -CLAUDE_MODEL_ALIASES: dict[str, str] = { - "sonnet": "claude-sonnet-5", - "opus": "claude-opus-5-5", - "haiku": "haiku", -} +CLAUDE_MODEL_ALIASES: Mapping[str, str] = MappingProxyType( + { + "sonnet": "claude-sonnet-5-5", + "opus": "claude-opus-5-5", + "haiku": "claude-haiku-5-5", + } +) CODEX_MODEL_ALIASES: Mapping[str, str] = MappingProxyType( { @@ -468,7 +470,7 @@ def model_class(model: str) -> str: ), write_guard_tool_names=frozenset({"Write", "Edit", "Bash", "apply_patch"}), env_denylist_prefixes=(), - min_version="2.1.280", + min_version="2.1.296", version_check_command="claude --version", process_name="claude", process_name_aliases=frozenset({"claude"}), diff --git a/src/autoskillit/execution/evidence/anomaly_detection.py b/src/autoskillit/execution/evidence/anomaly_detection.py index 179e297290..57895a5ea6 100644 --- a/src/autoskillit/execution/evidence/anomaly_detection.py +++ b/src/autoskillit/execution/evidence/anomaly_detection.py @@ -76,7 +76,7 @@ def normalize_model_id(model: str) -> str: Handles these forms: 1. Short aliases: "sonnet" → "claude-sonnet" - 2. Full IDs with date suffixes: "claude-haiku-4-5-20251001" → "claude-haiku-4-5" + 2. Full IDs with date suffixes: "claude-sonnet-4-5-20250929" → "claude-sonnet-4-5" 3. Full IDs without dates: "claude-sonnet-4-6" → "claude-sonnet-4-6" (unchanged) 4. Context-window suffixes stripped before alias lookup: "opus[1m]" → "claude-opus", "claude-opus-4-6[1m]" → "claude-opus-4-6" diff --git a/tests/_realistic_project.py b/tests/_realistic_project.py index 7e29df40bc..b9c6db4425 100644 --- a/tests/_realistic_project.py +++ b/tests/_realistic_project.py @@ -24,8 +24,8 @@ # `CLAUDE_CODE_CAPABILITIES.min_version` in src/autoskillit/core/types/execution/_type_backend.py. # Bumping the product floor requires bumping this constant and any inline shim # scripts that still embed the literal directly. -PINNED_CLAUDE_SHIM_VERSION_OUTPUT = "2.1.280 (Claude Code)" -PINNED_CLAUDE_SHIM_VERSION = "2.1.280" +PINNED_CLAUDE_SHIM_VERSION_OUTPUT = "2.1.296 (Claude Code)" +PINNED_CLAUDE_SHIM_VERSION = "2.1.296" _CONFIG_YAML = """agent_backend: backend: claude-code diff --git a/tests/cli/test_doctor_backend_guards.py b/tests/cli/test_doctor_backend_guards.py index 310fbef1dd..6723d7dbae 100644 --- a/tests/cli/test_doctor_backend_guards.py +++ b/tests/cli/test_doctor_backend_guards.py @@ -726,15 +726,15 @@ def test_claude_uses_its_own_floor_and_display_name( monkeypatch.setattr( subprocess, "run", - lambda *a, **kw: self._codex_result("2.1.279 (Claude Code)\n"), + lambda *a, **kw: self._codex_result("2.1.295 (Claude Code)\n"), ) result = _check_backend_version(backend=ClaudeCodeBackend()) assert result.severity == Severity.WARNING - assert "Claude Code 2.1.279" in result.message + assert "Claude Code 2.1.295" in result.message assert "below minimum" in result.message - assert "2.1.280" in result.message + assert "2.1.296" in result.message assert "Codex CLI" not in result.message def test_claude_at_minimum_returns_ok(self, monkeypatch: pytest.MonkeyPatch) -> None: @@ -751,7 +751,7 @@ def test_claude_at_minimum_returns_ok(self, monkeypatch: pytest.MonkeyPatch) -> ) result = _check_backend_version(backend=ClaudeCodeBackend()) assert result.severity == Severity.OK - assert "Claude Code 2.1.280" in result.message + assert "Claude Code 2.1.296" in result.message assert "below minimum" not in result.message diff --git a/tests/core/test_backend_capabilities.py b/tests/core/test_backend_capabilities.py index 0684f0ddb8..489a17e789 100644 --- a/tests/core/test_backend_capabilities.py +++ b/tests/core/test_backend_capabilities.py @@ -247,7 +247,7 @@ def test_claude_code_capabilities_field_values(): {"Write", "Edit", "Bash", "apply_patch"} ) assert CLAUDE_CODE_CAPABILITIES.env_denylist_prefixes == () - assert CLAUDE_CODE_CAPABILITIES.min_version == "2.1.280" + assert CLAUDE_CODE_CAPABILITIES.min_version == "2.1.296" assert CLAUDE_CODE_CAPABILITIES.version_check_command == "claude --version" assert CLAUDE_CODE_CAPABILITIES.process_name == "claude" assert CLAUDE_CODE_CAPABILITIES.process_name_aliases == frozenset({"claude"}) diff --git a/tests/execution/backends/test_claude_startup_readiness.py b/tests/execution/backends/test_claude_startup_readiness.py index 68fd28e3fa..c40ea3839a 100644 --- a/tests/execution/backends/test_claude_startup_readiness.py +++ b/tests/execution/backends/test_claude_startup_readiness.py @@ -139,7 +139,7 @@ def fake_run(cmd, **kwargs): # type: ignore[no-untyped-def] (1, "", "failed"), (0, "", "empty"), (0, "Claude Code unknown", "unparseable"), - (0, "2.1.279 (Claude Code)", "requires Claude Code 2.1.280 or newer"), + (0, "2.1.295 (Claude Code)", "requires Claude Code 2.1.296 or newer"), ], ) def test_pre_launch_fails_closed_for_unusable_capability_probe( diff --git a/tests/execution/backends/test_model_translation.py b/tests/execution/backends/test_model_translation.py index 1323a1ffed..352fc121ba 100644 --- a/tests/execution/backends/test_model_translation.py +++ b/tests/execution/backends/test_model_translation.py @@ -89,8 +89,8 @@ def test_unknown_passthrough(self, model_id: str) -> None: """Unrecognized or backend-foreign model ids pass through untranslated.""" assert ClaudeCodeBackend().translate_model(model_id) == model_id - def test_haiku_identity(self) -> None: - assert ClaudeCodeBackend().translate_model("haiku") == "haiku" + def test_haiku_alias(self) -> None: + assert ClaudeCodeBackend().translate_model("haiku") == CLAUDE_MODEL_ALIASES["haiku"] def test_suffix_case_insensitive(self) -> None: assert ( diff --git a/tests/execution/test_model_alias_registry.py b/tests/execution/test_model_alias_registry.py index 86c340dd7e..f1c0c42630 100644 --- a/tests/execution/test_model_alias_registry.py +++ b/tests/execution/test_model_alias_registry.py @@ -6,7 +6,16 @@ pytestmark = [pytest.mark.layer("execution"), pytest.mark.small, pytest.mark.model_contract] -VALID_CLAUDE_MODEL_IDS: frozenset[str] = frozenset({"claude-sonnet-5", "claude-opus-5-5", "haiku"}) +# Test-only evidence for the global product floor; runtime checks use +# CLAUDE_CODE_CAPABILITIES.min_version rather than per-model thresholds. +# Lowest Claude Code release that serves each pinned id correctly: model-config requires +# 2.1.280 for Opus 5.5, 2.1.284 for Sonnet 5.5 and 2.1.293 for Haiku 5.5; 2.1.296 fixes +# Haiku 5.5 token counts behind gateways. +CLAUDE_MODEL_MIN_CLIENT_VERSIONS: dict[str, str] = { + "claude-sonnet-5-5": "2.1.284", + "claude-opus-5-5": "2.1.280", + "claude-haiku-5-5": "2.1.296", +} def test_anomaly_detection_aliases_keys_match_shared() -> None: @@ -75,22 +84,46 @@ def test_claude_alias_values_in_allowlist() -> None: from autoskillit.core.types.execution._type_backend import CLAUDE_MODEL_ALIASES for key, value in CLAUDE_MODEL_ALIASES.items(): - assert value in VALID_CLAUDE_MODEL_IDS, ( - f"CLAUDE_MODEL_ALIASES[{key!r}] = {value!r} is not in VALID_CLAUDE_MODEL_IDS. " - "Update VALID_CLAUDE_MODEL_IDS if the intended target model changed." + assert value in CLAUDE_MODEL_MIN_CLIENT_VERSIONS, ( + f"CLAUDE_MODEL_ALIASES[{key!r}] = {value!r} is not in " + "CLAUDE_MODEL_MIN_CLIENT_VERSIONS. Add the pinned id with its minimum release." ) -def test_claude_sonnet_alias_uses_sonnet_5() -> None: +def test_claude_aliases_map_classes_to_5_5_models() -> None: from autoskillit.core.types.execution._type_backend import CLAUDE_MODEL_ALIASES - assert CLAUDE_MODEL_ALIASES["sonnet"] == "claude-sonnet-5" + assert CLAUDE_MODEL_ALIASES == { + "sonnet": "claude-sonnet-5-5", + "opus": "claude-opus-5-5", + "haiku": "claude-haiku-5-5", + } + +def test_claude_aliases_cannot_be_reassigned() -> None: + from operator import setitem -def test_claude_opus_alias_uses_opus_5_5() -> None: from autoskillit.core.types.execution._type_backend import CLAUDE_MODEL_ALIASES - assert CLAUDE_MODEL_ALIASES["opus"] == "claude-opus-5-5" + with pytest.raises(TypeError): + setitem(CLAUDE_MODEL_ALIASES, "haiku", "haiku") + + +def test_claude_floor_supports_every_pinned_model() -> None: + from packaging.version import Version + + from autoskillit.core.types.execution._type_backend import ( + CLAUDE_CODE_CAPABILITIES, + CLAUDE_MODEL_ALIASES, + ) + + floor = CLAUDE_CODE_CAPABILITIES.min_version + for model_class, model_id in CLAUDE_MODEL_ALIASES.items(): + required = CLAUDE_MODEL_MIN_CLIENT_VERSIONS[model_id] + assert Version(floor) >= Version(required), ( + f"Claude class {model_class!r} pins {model_id!r}, which requires Claude Code " + f"{required}; the configured floor is {floor}." + ) def test_codex_alias_values_differ_from_keys() -> None: diff --git a/tests/infra/test_conformance_probes_workflow.py b/tests/infra/test_conformance_probes_workflow.py index b60a473683..0d80b236b7 100644 --- a/tests/infra/test_conformance_probes_workflow.py +++ b/tests/infra/test_conformance_probes_workflow.py @@ -134,8 +134,8 @@ def test_claude_probe_pins_supported_version_platform_rows(self, workflow: dict) matrix = workflow["jobs"]["claude-probe"]["strategy"]["matrix"]["include"] rows = {(row["os"], row["claude-version"], row["probe-tmpdir"]) for row in matrix} assert rows == { - ("ubuntu-22.04", "2.1.280", "/dev/shm/pytest-probes"), - ("macos-14", "2.1.280", "/tmp/pytest-probes"), + ("ubuntu-22.04", "2.1.296", "/dev/shm/pytest-probes"), + ("macos-14", "2.1.296", "/tmp/pytest-probes"), } def test_claude_probe_installs_and_exports_exact_pinned_binary(self, workflow: dict) -> None: diff --git a/tests/infra/test_e2e_harness.py b/tests/infra/test_e2e_harness.py index 64b7cd26d0..e9f9ad26dd 100644 --- a/tests/infra/test_e2e_harness.py +++ b/tests/infra/test_e2e_harness.py @@ -640,7 +640,7 @@ def test_worktrees_directory_matches_clone_source_of_truth() -> None: SANDBOX_TOKEN = "sandbox-test-token" SANDBOX = "owner/sandbox" _MINIMAX_MODEL = "MiniMax-M3[1m]" -_EXPECTED_FLEET_MODEL = "claude-sonnet-5[1m]" +_EXPECTED_FLEET_MODEL = "claude-sonnet-5-5" _EXPECTED_REQUIRED_CONTEXT_WINDOW_TOKENS = 1_000_000 _EXPECTED_SETTINGS_ENV = { "ANTHROPIC_BASE_URL": "https://api.minimax.io/anthropic", diff --git a/tests/test_llm_triage.py b/tests/test_llm_triage.py index 2d3cd57993..04540b6a4e 100644 --- a/tests/test_llm_triage.py +++ b/tests/test_llm_triage.py @@ -713,11 +713,9 @@ async def test_triage_env_excludes_ide_vars(tmp_path: Path, monkeypatch: pytest. # --------------------------------------------------------------------------- -@pytest.mark.anyio -async def test_triage_command_uses_backend_binary_name( +async def _captured_single_item_triage_cmd( tmp_path: Path, monkeypatch: pytest.MonkeyPatch -): - """The triage command first element must equal ClaudeCodeBackend().binary_name().""" +) -> list[str]: from unittest.mock import AsyncMock from autoskillit._llm_triage import triage_staleness @@ -759,13 +757,30 @@ async def test_triage_command_uses_backend_binary_name( ) await triage_staleness([item], project_root=tmp_path, backend=ClaudeCodeBackend()) - cmd = mock_run.call_args.kwargs["cmd"] + return mock_run.call_args.kwargs["cmd"] + + +@pytest.mark.anyio +async def test_triage_command_uses_backend_binary_name( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + cmd = await _captured_single_item_triage_cmd(tmp_path, monkeypatch) expected_binary = ClaudeCodeBackend().binary_name() assert cmd[0] == expected_binary, ( f"triage_cmd[0] must equal get_backend().binary_name(), got {cmd[0]!r}" ) +@pytest.mark.anyio +async def test_triage_command_uses_haiku_class_model( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from autoskillit.core import CLAUDE_MODEL_ALIASES, ClaudeFlags + + cmd = await _captured_single_item_triage_cmd(tmp_path, monkeypatch) + assert cmd[cmd.index(ClaudeFlags.MODEL) + 1] == CLAUDE_MODEL_ALIASES["haiku"] + + @pytest.mark.anyio async def test_triage_staleness_does_not_use_pty( tmp_path: Path, monkeypatch: pytest.MonkeyPatch @@ -959,3 +974,63 @@ async def test_triage_batch_claude_code_backend_does_call_subprocess( assert len(results) == 1 assert results[0]["meaningful"] is False assert results[0]["summary"] == "no change" + + +@pytest.mark.anyio +@pytest.mark.parametrize("error_channel", ["stderr", "stdout"]) +async def test_triage_batch_preserves_bounded_failure_diagnostics( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, error_channel: str +) -> None: + from unittest.mock import AsyncMock, Mock + + from autoskillit._llm_triage import _triage_batch + from autoskillit.core import SubprocessResult, TerminationReason + + error = "Provider rejected pinned model: " + "x" * 1100 + stdout = ( + json.dumps( + { + "type": "result", + "subtype": "error_during_execution", + "is_error": True, + "result": error, + "session_id": "triage-failure", + } + ) + if error_channel == "stdout" + else "" + ) + stderr = error if error_channel == "stderr" else "" + monkeypatch.setattr( + _patch_autoskillit__llm_triage, + "run_managed_async", + AsyncMock( + return_value=SubprocessResult( + returncode=1, + stdout=stdout, + stderr=stderr, + termination=TerminationReason.NATURAL_EXIT, + pid=0, + ) + ), + ) + logger = Mock() + monkeypatch.setattr(_patch_autoskillit__llm_triage, "logger", logger) + item = StaleItem( + skill="my-skill", reason="hash_mismatch", stored_value="old", current_value="new" + ) + + results = await _triage_batch( + [item], {"my-skill": "# Skill"}, cwd=tmp_path, backend=ClaudeCodeBackend() + ) + + assert results == [ + {"skill": "my-skill", "meaningful": True, "summary": "Batch triage parse failed."} + ] + logger.warning.assert_called_once_with( + "triage_staleness batch parse failed; treating all as meaningful", + batch=["my-skill"], + returncode=1, + stderr=stderr[:1000], + session_result=error[:1000] if error_channel == "stdout" else "", + )