From de3682ba46cb609cc08fc0714e9693760db4731a Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 19:24:24 -0700 Subject: [PATCH 01/15] fix: retire unreachable audit substitution probe --- src/autoskillit/core/__init__.pyi | 6 -- src/autoskillit/core/audit/__init__.py | 8 -- .../core/audit/audit_semantic_codec.py | 93 ------------------- .../core/types/constants/_type_constants.py | 2 - .../skills_extended/audit-impl/SKILL.md | 41 +------- src/autoskillit/smoke_utils/__init__.py | 3 +- src/autoskillit/smoke_utils/_helpers.py | 90 ------------------ tests/_audit_substitution_fixtures.py | 30 +++--- ...est_audit_substitution_rule_enforcement.py | 52 +++++++++++ .../test_subpackage_isolation_file_counts.py | 2 - tests/core/test_audit_substitution_probe.py | 63 +++---------- .../server/test_consumer_tool_callability.py | 10 -- .../skills/test_audit_impl_diff_discipline.py | 15 +++ tests/smoke_utils/test_facade.py | 31 ------- 14 files changed, 97 insertions(+), 349 deletions(-) create mode 100644 tests/arch/test_audit_substitution_rule_enforcement.py diff --git a/src/autoskillit/core/__init__.pyi b/src/autoskillit/core/__init__.pyi index b238e72e27..db7ce56df5 100644 --- a/src/autoskillit/core/__init__.pyi +++ b/src/autoskillit/core/__init__.pyi @@ -29,15 +29,10 @@ from .audit.audit_cycle_verifier import VerifiedAuditCycle as VerifiedAuditCycle from .audit.audit_semantic_codec import PRESCRIPTIVE_MECHANISM_CUES as PRESCRIPTIVE_MECHANISM_CUES from .audit.audit_semantic_codec import SUBSTITUTION_MARKERS as SUBSTITUTION_MARKERS from .audit.audit_semantic_codec import AuditSemanticCodecError as AuditSemanticCodecError -from .audit.audit_semantic_codec import ProbedRequirement as ProbedRequirement from .audit.audit_semantic_codec import SubstitutionFinding as SubstitutionFinding -from .audit.audit_semantic_codec import SubstitutionTrigger as SubstitutionTrigger from .audit.audit_semantic_codec import ( canonical_full_reference_records_match as canonical_full_reference_records_match, ) -from .audit.audit_semantic_codec import ( - evaluate_diff_mock_of_prescribed_symbol as evaluate_diff_mock_of_prescribed_symbol, -) from .audit.audit_semantic_codec import ( evaluate_rationale_contradiction as evaluate_rationale_contradiction, ) @@ -45,7 +40,6 @@ from .audit.audit_semantic_codec import load_audit_semantic_result as load_audit from .audit.audit_semantic_codec import ( load_standalone_audit_evidence as load_standalone_audit_evidence, ) -from .audit.audit_semantic_codec import probe_substitutions as probe_substitutions from .audit.closure_hashing import HASH_RE as HASH_RE from .audit.closure_hashing import canonical_json_bytes as canonical_json_bytes from .audit.closure_hashing import compute_bytes_hash as compute_bytes_hash diff --git a/src/autoskillit/core/audit/__init__.py b/src/autoskillit/core/audit/__init__.py index 0e9275ee52..ef61a1da1b 100644 --- a/src/autoskillit/core/audit/__init__.py +++ b/src/autoskillit/core/audit/__init__.py @@ -39,15 +39,11 @@ PRESCRIPTIVE_MECHANISM_CUES, SUBSTITUTION_MARKERS, AuditSemanticCodecError, - ProbedRequirement, SubstitutionFinding, - SubstitutionTrigger, canonical_full_reference_records_match, - evaluate_diff_mock_of_prescribed_symbol, evaluate_rationale_contradiction, load_audit_semantic_result, load_standalone_audit_evidence, - probe_substitutions, ) from autoskillit.core.audit.closure_hashing import ( HASH_RE, @@ -89,10 +85,8 @@ def __getattr__(name: str): "HASH_RE", "InventoryAdmissionEvaluator", "PRESCRIPTIVE_MECHANISM_CUES", - "ProbedRequirement", "SUBSTITUTION_MARKERS", "SubstitutionFinding", - "SubstitutionTrigger", "VerificationResult", "VerifiedAuditCycle", "canonical_full_reference_records_match", @@ -103,12 +97,10 @@ def __getattr__(name: str): "compute_report_hash", "compute_request_hash", "compute_row_hash", - "evaluate_diff_mock_of_prescribed_symbol", "evaluate_rationale_contradiction", "load_audit_semantic_result", "load_standalone_audit_evidence", "parse_canonical_json_bytes", - "probe_substitutions", "verify_closure_report", ] diff --git a/src/autoskillit/core/audit/audit_semantic_codec.py b/src/autoskillit/core/audit/audit_semantic_codec.py index 8fe0c0ed6c..d6c274d2a2 100644 --- a/src/autoskillit/core/audit/audit_semantic_codec.py +++ b/src/autoskillit/core/audit/audit_semantic_codec.py @@ -11,7 +11,6 @@ from collections.abc import Sequence from dataclasses import dataclass from datetime import date -from enum import StrEnum from pathlib import Path from typing import Any, Literal, Protocol @@ -41,16 +40,12 @@ __all__ = [ "AuditSemanticCodecError", "PRESCRIPTIVE_MECHANISM_CUES", - "ProbedRequirement", "SUBSTITUTION_MARKERS", "SubstitutionFinding", - "SubstitutionTrigger", "canonical_full_reference_records_match", - "evaluate_diff_mock_of_prescribed_symbol", "evaluate_rationale_contradiction", "load_audit_semantic_result", "load_standalone_audit_evidence", - "probe_substitutions", ] _DEFAULT_MAX_SIZE_BYTES = 10_000_000 @@ -122,25 +117,11 @@ ) _TOKEN_RE = re.compile(r"[a-z0-9]+") -_IDENTIFIER_RE = re.compile(r"(? str | None: return None -def _added_diff_text(diff_text: str) -> str: - """Return the joined text of added unified-diff lines. - - Strips the leading ``+`` marker and skips the ``+++`` file header. The - ``\\ No newline at end of file`` escape (always prefixed with a single - space, never with ``+``) is filtered incidentally because it does not - start with ``+``. The result is a corpus suitable for term and - identifier matching, not a structural reconstruction of the diff. - """ - chunks: list[str] = [] - for line in diff_text.splitlines(): - if not line.startswith("+"): - continue - if line.startswith("+++"): - continue - chunks.append(line[1:]) - return "\n".join(chunks) - - def evaluate_rationale_contradiction( requirement_id: str, requirement_text: str, @@ -193,66 +155,11 @@ def evaluate_rationale_contradiction( return None return SubstitutionFinding( requirement_id=requirement_id, - trigger=SubstitutionTrigger.RATIONALE_CONTRADICTION, matched_marker=marker, matched_prescription=cue, ) -def evaluate_diff_mock_of_prescribed_symbol( - requirement_id: str, - requirement_text: str, - diff_text: str, -) -> SubstitutionFinding | None: - """Detect added diff lines that mock an identifier named by a requirement.""" - - identifiers = { - match.group(0).casefold() - for match in _IDENTIFIER_RE.finditer(requirement_text) - if "_" in match.group(0) - } - if not identifiers: - return None - added_text = _added_diff_text(diff_text) - marker = _first_matching_term(added_text, SUBSTITUTION_MARKERS) - added_identifiers = { - match.group(0).casefold() for match in _IDENTIFIER_RE.finditer(added_text) - } - matches = sorted(identifiers & added_identifiers) - if marker is not None and matches: - return SubstitutionFinding( - requirement_id=requirement_id, - trigger=SubstitutionTrigger.DIFF_MOCK_OF_PRESCRIBED_SYMBOL, - matched_marker=marker, - matched_prescription=matches[0], - ) - return None - - -def probe_substitutions( - requirements: tuple[ProbedRequirement, ...], - diff_text: str, -) -> tuple[SubstitutionFinding, ...]: - """Apply both deterministic substitution rules to each requirement.""" - - findings: list[SubstitutionFinding] = [] - for requirement in requirements: - candidates = ( - evaluate_rationale_contradiction( - requirement.requirement_id, - requirement.requirement_text, - requirement.evidence_summary, - ), - evaluate_diff_mock_of_prescribed_symbol( - requirement.requirement_id, - requirement.requirement_text, - diff_text, - ), - ) - findings.extend(candidate for candidate in candidates if candidate is not None) - return tuple(dict.fromkeys(findings)) - - AuditSemanticCodecReason = Literal[ "artifact_read_failed", "invalid_canonical_json", diff --git a/src/autoskillit/core/types/constants/_type_constants.py b/src/autoskillit/core/types/constants/_type_constants.py index aca15b697b..91e5c68326 100644 --- a/src/autoskillit/core/types/constants/_type_constants.py +++ b/src/autoskillit/core/types/constants/_type_constants.py @@ -336,12 +336,10 @@ def _render_stop_step_evidence_doctrine(indent: str = "") -> str: "output_dir", "workspace", "diagnostics_log_dir", - "diff_path", "investigation_path", "raw_output_artifact_path", "current_authority_path", "prior_authority_path", - "requirements_path", } ) RUN_PYTHON_SENTINEL_KEYS: frozenset[str] = frozenset( diff --git a/src/autoskillit/skills_extended/audit-impl/SKILL.md b/src/autoskillit/skills_extended/audit-impl/SKILL.md index 2f4d821ed4..8527fc3ad2 100644 --- a/src/autoskillit/skills_extended/audit-impl/SKILL.md +++ b/src/autoskillit/skills_extended/audit-impl/SKILL.md @@ -97,8 +97,7 @@ manifest, or no remediation ancestor respectively. **NEVER:** - Fabricate, invent, or embellish information not supported by the available evidence or code. -- Modify source files or plan files. The only permitted writes are the temporary probe - snapshots in Step 3.4 and the declared closure/remediation evidence in Step 5. +- Modify source files or plan files. The only permitted writes are the declared closure/remediation evidence in Step 5. - Run tests — this skill audits, it does not fix - Create authority, inventory, lifecycle, execution-identity, or publication artifacts - Emit a GO verdict when any blocking finding exists unless Step 3.5 accepted its @@ -228,7 +227,6 @@ verified `ClosureReport` as evidence rather than duplicating it. - Step 2 (Diff Computation): Unchanged. - Step 3 (Slice Audit): Unchanged — slice and dispatch `audit-impl-slice-auditor` subagents against authority requirements. -- Step 3.4 (Substitution Probe): Unchanged. - Step 3.5 (Deviation Evaluation): Unchanged if `deviation_manifest_path` provided. - Step 4 (Verdict Determination): Unchanged — determine GO/NO GO from findings. - Step 5 (Output — modified): Produce a canonical `ClosureReport` before the authority: @@ -327,10 +325,7 @@ Step 0 — do NOT bypass the command form determined in Step 0: - **Branch ref:** `git log {base_branch}..{implementation_ref} --oneline` — commit history - **Branch ref:** `git diff {base_branch}...{implementation_ref}` — full diff -Retain the exact full unified diff in memory and write a byte-identical UTF-8 snapshot to -`{{AUTOSKILLIT_TEMP}}/audit-impl/implementation.diff` for Step 3.4. This temporary snapshot -is advisory probe input, not an audit authority or published artifact. When the size guard -requires grouped slice diffs, keep this full snapshot in addition to the grouped child inputs. +Retain the exact full unified diff in memory. **Diff size guard:** After the subagent returns the `--stat` output, check the diff size: @@ -438,34 +433,6 @@ paraphrase. - `UNPRESCRIBED_SUBSTITUTION` — the goal is met through a substantive mechanism or test-topology substitution - `UNSATISFIABLE_BY_CODE` — the finding is real, blocking, and requires a human decision rather than a code change -### Step 3.4 — Deterministic Substitution Probe - -Write `{{AUTOSKILLIT_TEMP}}/audit-impl/probe-requirements.json` as a UTF-8 JSON array. -Each exact-key object contains only `requirement_id`, the verbatim `requirement_text`, and -the slice auditor's `evidence_summary`. Use the Step 2 full-diff snapshot as `diff_path`. -Both input paths and the output directory must be absolute. - -Call: - -``` -run_python( - callable="autoskillit.smoke_utils.probe_audit_substitutions", - args={ - "requirements_path": "", - "diff_path": "", - "output_dir": "", - }, -) -``` - -Read the returned `probe_path`. For every flagged requirement whose current assessment is -not blocking, force its assessment to `UNPRESCRIBED_SUBSTITUTION` and append the finding's -trigger and matched cue to `evidence_summary`. This override is unconditional, like the -Category-3 rule in Step 2.5; it does not depend on how persuasively the slice auditor -justified the substitution. The server independently enforces the rationale-contradiction -rule, so skipping this step only changes a clean `NO GO` into `SEMANTIC_REJECTED`; it cannot -publish a contradictory non-blocking row. - ### Step 3.5 — Deviation Evaluation Set `deviation_entries` to the validated manifest's `deviations` array, or an empty @@ -479,7 +446,7 @@ Read the deviation manifest JSON file. For each entry in the `deviations` array, ``` portable child delegation (logical role "autoskillit:audit-impl-deviation-evaluator", model="sonnet", prompt="... - XML delimiters, ALL blocking findings from Steps 2.5, 3, and 3.4, + XML delimiters, ALL blocking findings from Steps 2.5 and 3, implementation_ref value>") ``` @@ -524,7 +491,7 @@ branch — discard those MISSING findings and re-run the slice with the correct This post-processing runs before the final verdict determination. After downgrading, apply the standard three-tier logic to the updated findings set. -For each blocking finding from Steps 2.5, 3, and 3.4: +For each blocking finding from Steps 2.5 and 3: - If matched by an ACCEPT or ACCEPT_WITH_NOTE deviation in Step 3.5 → GO-compatible - Otherwise → blocking diff --git a/src/autoskillit/smoke_utils/__init__.py b/src/autoskillit/smoke_utils/__init__.py index 52616b87a6..d2174ab2eb 100644 --- a/src/autoskillit/smoke_utils/__init__.py +++ b/src/autoskillit/smoke_utils/__init__.py @@ -24,7 +24,7 @@ detect_zero_changes, fetch_merge_queue_data, ) -from autoskillit.smoke_utils._helpers import probe_audit_substitutions, try_load_json +from autoskillit.smoke_utils._helpers import try_load_json from autoskillit.smoke_utils._investigation import extract_investigation from autoskillit.smoke_utils._merge_gate_diagnosis import diagnose_merge_gate from autoskillit.smoke_utils._review import ( @@ -133,7 +133,6 @@ "plan_review_audit_slots", "pre_iteration_cleanup", "prepare_experimental_review_publication", - "probe_audit_substitutions", "publish_experimental_review_artifacts", "render_review_finding_body", "render_unpostable_review_section", diff --git a/src/autoskillit/smoke_utils/_helpers.py b/src/autoskillit/smoke_utils/_helpers.py index d349e4e9ef..1b3c0aadbc 100644 --- a/src/autoskillit/smoke_utils/_helpers.py +++ b/src/autoskillit/smoke_utils/_helpers.py @@ -5,15 +5,6 @@ import json from pathlib import Path -from autoskillit.core import ( - ProbedRequirement, - atomic_write, - canonical_json_bytes, - probe_substitutions, -) - -_AUDIT_REQUIREMENT_KEYS = frozenset({"requirement_id", "requirement_text", "evidence_summary"}) - def _load_json(src: str) -> list | dict: """Load JSON from a string or file path. Returns a list or dict.""" @@ -32,84 +23,3 @@ def try_load_json(path: Path) -> dict | None: return json.loads(path.read_text()) except (OSError, json.JSONDecodeError): return None - - -def _read_audit_probe_input(label: str, path: Path) -> str: - """Read a probe input file with full orchestration context on failure.""" - try: - return path.read_text(encoding="utf-8") - except FileNotFoundError as exc: - raise FileNotFoundError( - f"{label} is required for probe_audit_substitutions: {path}" - ) from exc - except UnicodeDecodeError as exc: - raise ValueError(f"{label} is not valid UTF-8: {path} ({exc.reason})") from exc - - -def _decode_audit_requirements(label: str, path: Path) -> object: - """Decode a probe requirements JSON file with orchestration context.""" - text = _read_audit_probe_input(label, path) - try: - return json.loads(text) - except json.JSONDecodeError as exc: - raise ValueError( - f"{label} is not valid JSON ({exc.msg} at line {exc.lineno} " - f"column {exc.colno}): {path}" - ) from exc - - -def probe_audit_substitutions( - requirements_path: str, - diff_path: str, - output_dir: str, -) -> dict[str, str]: - """Write deterministic substitution findings for audit-impl orchestration.""" - - paths = { - "requirements_path": Path(requirements_path), - "diff_path": Path(diff_path), - "output_dir": Path(output_dir), - } - for name, path in paths.items(): - if not path.is_absolute(): - raise ValueError(f"{name} must be absolute, got {str(path)!r}") - - raw_requirements = _decode_audit_requirements("requirements_path", paths["requirements_path"]) - if not isinstance(raw_requirements, list): - raise ValueError( - f"requirements_path must contain a JSON array, got {type(raw_requirements).__name__}" - ) - requirements: list[ProbedRequirement] = [] - for index, raw in enumerate(raw_requirements): - if not isinstance(raw, dict) or frozenset(raw) != _AUDIT_REQUIREMENT_KEYS: - actual = sorted(raw) if isinstance(raw, dict) else type(raw).__name__ - raise ValueError( - f"requirements_path[{index}] must contain exactly " - f"{sorted(_AUDIT_REQUIREMENT_KEYS)!r}: got {actual}" - ) - if not all(isinstance(raw[key], str) for key in _AUDIT_REQUIREMENT_KEYS): - raise ValueError(f"requirements_path[{index}] values must all be strings") - requirements.append(ProbedRequirement(**raw)) - - findings = probe_substitutions( - tuple(requirements), - _read_audit_probe_input("diff_path", paths["diff_path"]), - ) - payload = { - "flagged_requirement_ids": list( - dict.fromkeys(finding.requirement_id for finding in findings) - ), - "findings": [ - { - "requirement_id": finding.requirement_id, - "trigger": finding.trigger.value, - "matched_marker": finding.matched_marker, - "matched_cue": finding.matched_prescription, - } - for finding in findings - ], - } - paths["output_dir"].mkdir(parents=True, exist_ok=True) - output_path = paths["output_dir"] / "audit_substitution_probe.json" - atomic_write(output_path, canonical_json_bytes(payload)) - return {"probe_path": str(output_path)} diff --git a/tests/_audit_substitution_fixtures.py b/tests/_audit_substitution_fixtures.py index 63c69b7b51..70a410e43d 100644 --- a/tests/_audit_substitution_fixtures.py +++ b/tests/_audit_substitution_fixtures.py @@ -1,4 +1,4 @@ -"""Committed fixtures for audit substitution probes and server-floor tests. +"""Committed fixtures for audit substitution server-floor tests. The historical incident (REQ-004/REQ-006/REQ-007) was preserved here as a single source of truth so the regression suite never silently skips when a @@ -8,7 +8,14 @@ from __future__ import annotations -from autoskillit.core import ProbedRequirement +from typing import NamedTuple + + +class IncidentRequirement(NamedTuple): + requirement_id: str + requirement_text: str + evidence_summary: str + INCIDENT_REQUIREMENT: str = ( "For a CPU-active child that never stops, child_deferral_ceiling=1.0, " @@ -23,19 +30,8 @@ "psutil that the real child process remains active until termination." ) -INCIDENT_DIFF: str = '''\ -"""Child-process liveness is simulated via a mock on -``_has_active_child_processes`` rather than relying on real psutil CPU-percent -sampling.""" -+ monkeypatch.setattr( -+ _patch_process__termination, -+ "_has_active_child_processes", -+ lambda pid: True, -+ ) -''' - -INCIDENT_REQUIREMENTS: tuple[ProbedRequirement, ...] = ( - ProbedRequirement( +INCIDENT_REQUIREMENTS: tuple[IncidentRequirement, ...] = ( + IncidentRequirement( requirement_id="REQ-004", requirement_text=( "child_deferral_ceiling must be honored exactly for any CPU-active " @@ -46,7 +42,7 @@ "return True instead of observing real psutil CPU sampling." ), ), - ProbedRequirement( + IncidentRequirement( requirement_id="REQ-006", requirement_text=( "For a CPU-active child that never stops, child_deferral_ceiling=1.0, " @@ -54,7 +50,7 @@ ), evidence_summary=INCIDENT_EVIDENCE, ), - ProbedRequirement( + IncidentRequirement( requirement_id="REQ-007", requirement_text=( "Real subprocess.Popen and psutil must drive the child-process " diff --git a/tests/arch/test_audit_substitution_rule_enforcement.py b/tests/arch/test_audit_substitution_rule_enforcement.py new file mode 100644 index 0000000000..7c8e760422 --- /dev/null +++ b/tests/arch/test_audit_substitution_rule_enforcement.py @@ -0,0 +1,52 @@ +"""Keep every public audit substitution rule enforced at both parent floors.""" + +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest + +from autoskillit.core.audit import audit_semantic_codec + +pytestmark = [pytest.mark.layer("arch"), pytest.mark.small] + +_REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +_FLOORS = { + "_validate_semantic_assessment_floor": ( + _REPOSITORY_ROOT / "src/autoskillit/server/_audit_authority_materializer.py" + ), + "_verify_rows": _REPOSITORY_ROOT / "src/autoskillit/core/audit/closure_verifier.py", +} + + +def _direct_calls(path: Path, function_name: str) -> set[str]: + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + function = next( + node + for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name == function_name + ) + return { + node.func.id if isinstance(node.func, ast.Name) else node.func.attr + for node in ast.walk(function) + if isinstance(node, ast.Call) and isinstance(node.func, (ast.Name, ast.Attribute)) + } + + +def test_public_substitution_rules_are_enforced_by_both_parent_floors() -> None: + rules = {name for name in audit_semantic_codec.__all__ if name.startswith("evaluate_")} + assert rules, "audit_semantic_codec must export at least one evaluate_* substitution rule" + + calls_by_floor = {floor: _direct_calls(path, floor) for floor, path in _FLOORS.items()} + missing = { + rule: sorted(floor for floor, calls in calls_by_floor.items() if rule not in calls) + for rule in sorted(rules) + } + missing = {rule: floors for rule, floors in missing.items() if floors} + + assert not missing, ( + f"orphan audit substitution rules: {missing}; enforce each rule in both parent floors " + "(_validate_semantic_assessment_floor and _verify_rows) or delete it; child-side-only " + "substitution rules are forbidden (#5177)" + ) diff --git a/tests/arch/test_subpackage_isolation_file_counts.py b/tests/arch/test_subpackage_isolation_file_counts.py index 7f60d79997..953fb19a99 100644 --- a/tests/arch/test_subpackage_isolation_file_counts.py +++ b/tests/arch/test_subpackage_isolation_file_counts.py @@ -129,7 +129,6 @@ "core/claude_env": 4, # 3 files + __init__ + buffer "core/io": 9, # 8 files + __init__ + buffer (yaml_io.py split from io.py for 750-line cap) "core/git": 6, # 5 focused modules + __init__ - # #4249 extends audit_semantic_codec.py, preserving this package's existing file budget. "core/audit": 5, # 4 files + __init__ + buffer "core/plugins": 10, # 7 files + __init__ + buffer "core/pipeline": 5, # 4 files + __init__ + buffer @@ -161,7 +160,6 @@ "workspace": 1, # was 6; #5018 moved 5 skill-capability modules into skill_capabilities/ "hooks": 27, # +1 _capture_spawn.py extracted from _capture_process.py (#4732) "hooks/guards": 42, - # #4249 extends _helpers.py, preserving this package's existing file budget. "smoke_utils": 11, } diff --git a/tests/core/test_audit_substitution_probe.py b/tests/core/test_audit_substitution_probe.py index 91ebdb24de..bb82304f89 100644 --- a/tests/core/test_audit_substitution_probe.py +++ b/tests/core/test_audit_substitution_probe.py @@ -7,33 +7,28 @@ from autoskillit.core import ( PRESCRIPTIVE_MECHANISM_CUES, SUBSTITUTION_MARKERS, - ProbedRequirement, - SubstitutionTrigger, - evaluate_diff_mock_of_prescribed_symbol, evaluate_rationale_contradiction, - probe_substitutions, ) from tests._audit_substitution_fixtures import ( - INCIDENT_DIFF, INCIDENT_EVIDENCE, INCIDENT_REQUIREMENT, INCIDENT_REQUIREMENTS, LITERAL_EVIDENCE, + IncidentRequirement, ) pytestmark = [pytest.mark.layer("core"), pytest.mark.small] @pytest.fixture -def incident_requirements() -> tuple[ProbedRequirement, ...]: +def incident_requirements() -> tuple[IncidentRequirement, ...]: """Committed incident requirements (no temp-file dependency).""" return INCIDENT_REQUIREMENTS -def test_incident_requirements_are_flagged_by_both_probe_and_server_floor( - incident_requirements: tuple[ProbedRequirement, ...], +def test_incident_requirements_are_flagged_by_server_floor( + incident_requirements: tuple[IncidentRequirement, ...], ) -> None: - findings = probe_substitutions(incident_requirements, INCIDENT_DIFF) server_floor_findings = tuple( evaluate_rationale_contradiction( requirement.requirement_id, @@ -43,14 +38,6 @@ def test_incident_requirements_are_flagged_by_both_probe_and_server_floor( for requirement in incident_requirements ) - assert {finding.requirement_id for finding in findings} == { - "REQ-004", - "REQ-006", - "REQ-007", - } - assert all( - finding.trigger is SubstitutionTrigger.RATIONALE_CONTRADICTION for finding in findings - ) assert { finding.requirement_id for finding in server_floor_findings if finding is not None } == { @@ -60,23 +47,10 @@ def test_incident_requirements_are_flagged_by_both_probe_and_server_floor( } -def test_diff_rule_detects_mock_of_a_requirement_named_symbol() -> None: - finding = evaluate_diff_mock_of_prescribed_symbol( - "REQ-006", - "Use _has_active_child_processes to observe a real child process.", - INCIDENT_DIFF, - ) - - assert finding is not None - assert finding.requirement_id == "REQ-006" - assert finding.trigger is SubstitutionTrigger.DIFF_MOCK_OF_PRESCRIBED_SYMBOL - - def test_literal_descendant_implementation_is_not_a_substitution( - incident_requirements: tuple[ProbedRequirement, ...], + incident_requirements: tuple[IncidentRequirement, ...], ) -> None: requirement = incident_requirements[1] - literal_diff = "+ assert child.cpu_percent(interval=0.1) >= 0\n" assert ( evaluate_rationale_contradiction( @@ -86,30 +60,18 @@ def test_literal_descendant_implementation_is_not_a_substitution( ) is None ) - assert ( - probe_substitutions( - ( - ProbedRequirement( - requirement_id=requirement.requirement_id, - requirement_text=requirement.requirement_text, - evidence_summary=LITERAL_EVIDENCE, - ), - ), - literal_diff, - ) - == () - ) def test_mock_evidence_without_a_prescribed_mechanism_is_not_flagged() -> None: - requirement = ProbedRequirement( - requirement_id="REQ-RENAME", - requirement_text="Rename the diagnostic docstring for the audit result.", - evidence_summary="The tests use a mock result while checking the renamed docstring.", + assert ( + evaluate_rationale_contradiction( + "REQ-RENAME", + "Rename the diagnostic docstring for the audit result.", + "The tests use a mock result while checking the renamed docstring.", + ) + is None ) - assert probe_substitutions((requirement,), INCIDENT_DIFF) == () - def test_substitution_vocabulary_spot_check_preserves_known_markers_and_cues() -> None: # Spot-check the canonical marker and cue vocabulary. The full literal @@ -130,5 +92,4 @@ def test_incident_constants_cover_the_floor_rule_inputs() -> None: """The shared fixtures must keep the floor rule inputs reachable.""" assert INCIDENT_REQUIREMENT assert INCIDENT_EVIDENCE - assert INCIDENT_DIFF.startswith('"""') assert all(req.requirement_id for req in INCIDENT_REQUIREMENTS) diff --git a/tests/server/test_consumer_tool_callability.py b/tests/server/test_consumer_tool_callability.py index 311367d8a7..a24a7964a8 100644 --- a/tests/server/test_consumer_tool_callability.py +++ b/tests/server/test_consumer_tool_callability.py @@ -84,16 +84,6 @@ "run_cmd", CatalogContext.HEADLESS_SKILL_AUTO_GATE, ): "resolve-failures/SKILL.md:165 says not via Bash or run_cmd", - ( - "skill:audit-impl", - "run_python", - CatalogContext.HEADLESS_SKILL, - ): "#5177: move the Step 3.4 probe to the parent-side floor check", - ( - "skill:audit-impl", - "run_python", - CatalogContext.HEADLESS_SKILL_AUTO_GATE, - ): "#5177: move the Step 3.4 probe to the parent-side floor check", ( "skill:process-issues", "batch_cleanup_clones", diff --git a/tests/skills/test_audit_impl_diff_discipline.py b/tests/skills/test_audit_impl_diff_discipline.py index 36848602d2..d7932a4115 100644 --- a/tests/skills/test_audit_impl_diff_discipline.py +++ b/tests/skills/test_audit_impl_diff_discipline.py @@ -198,3 +198,18 @@ def test_step2_does_not_prohibit_head_unconditionally( "{implementation_ref}..HEAD legitimately. Replace the blanket 'do NOT hardcode HEAD' " "with 'do NOT bypass the command form determined in Step 0'." ) + + +def test_audit_impl_names_no_unreachable_substitution_probe(skill_content: str) -> None: + forbidden = { + "probe_audit_substitutions", + "Step 3.4", + "probe-requirements.json", + "implementation.diff", + "run_python", + } + + present = sorted(value for value in forbidden if value in skill_content) + assert not present, ( + f"audit-impl must not prescribe the unreachable child-side substitution probe: {present}" + ) diff --git a/tests/smoke_utils/test_facade.py b/tests/smoke_utils/test_facade.py index 573882863b..3cbae4f485 100644 --- a/tests/smoke_utils/test_facade.py +++ b/tests/smoke_utils/test_facade.py @@ -88,7 +88,6 @@ def test_smoke_utils_all_exports_complete() -> None: "plan_review_audit_slots", "pre_iteration_cleanup", "prepare_experimental_review_publication", - "probe_audit_substitutions", "publish_experimental_review_artifacts", "render_review_finding_body", "render_unpostable_review_section", @@ -146,7 +145,6 @@ def test_smoke_utils_all_exports_complete() -> None: "plan_review_audit_slots", "pre_iteration_cleanup", "prepare_experimental_review_publication", - "probe_audit_substitutions", "publish_experimental_review_artifacts", "render_review_finding_body", "render_unpostable_review_section", @@ -172,13 +170,6 @@ def test_smoke_utils_callable_resolvable_via_importlib(name: str) -> None: ("annotate_pr_diff", {"pr_number": "1", "cwd": "/tmp/repo"}), ("parse_eval_manifests", {"canary_manifest": "{}", "variant_manifest": "{}"}), ("parse_agent_eval_manifests", {"canary_manifest": "{}", "variant_manifest": "{}"}), - ( - "probe_audit_substitutions", - { - "requirements_path": "/tmp/requirements.json", - "diff_path": "/tmp/implementation.diff", - }, - ), ("compute_domain_partitions", {"batch_branch": "b", "base_branch": "main", "cwd": "/tmp"}), ("fetch_merge_queue_data", {"base_branch": "main", "cwd": "/tmp"}), ("diagnose_merge_gate", {"test_stdout": "FAILED x", "test_stderr": ""}), @@ -192,28 +183,6 @@ def test_callable_rejects_relative_output_dir(callable_name: str, minimal_args: func(**minimal_args, output_dir=".autoskillit/temp/test") -@pytest.mark.parametrize( - "kwargs", - [ - { - "requirements_path": "requirements.json", - "diff_path": "/tmp/implementation.diff", - "output_dir": "/tmp/probe-output", - }, - { - "requirements_path": "/tmp/requirements.json", - "diff_path": "implementation.diff", - "output_dir": "/tmp/probe-output", - }, - ], -) -def test_probe_audit_substitutions_rejects_relative_input_paths(kwargs: dict[str, str]) -> None: - from autoskillit.smoke_utils import probe_audit_substitutions - - with pytest.raises(ValueError, match="absolute"): - probe_audit_substitutions(**kwargs) - - def test_enrich_diff_context_rejects_relative_project_dir() -> None: with pytest.raises(ValueError, match="absolute"): enrich_diff_context(pr_number="1", project_dir="relative/path", output_dir="/tmp") From 8f1a6d6b9535f318a4c65e27aa401bd8a5e53f37 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 19:52:40 -0700 Subject: [PATCH 02/15] fix: enforce skill tool callability across launch shapes --- docs/skills/visibility.md | 4 +- .../core/types/launch/_type_skill_contract.py | 1 + src/autoskillit/migration/store.py | 12 - src/autoskillit/server/AGENTS.md | 8 +- src/autoskillit/skills/AGENTS.md | 4 + src/autoskillit/skills/close-kitchen/SKILL.md | 1 + src/autoskillit/skills/open-kitchen/SKILL.md | 1 + src/autoskillit/skills_extended/AGENTS.md | 3 + .../skills_extended/make-campaign/SKILL.md | 1 + .../skills_extended/migrate-recipes/SKILL.md | 18 +- .../skills_extended/write-recipe/SKILL.md | 8 +- src/autoskillit/workspace/skills/_records.py | 13 + .../test_activate_deps_completeness.py | 38 +++ tests/migration/test_store.py | 23 -- .../server/test_consumer_tool_callability.py | 315 +++++++++++++++--- tests/server/test_session_hook_root_scope.py | 25 ++ tests/workspace/test_corridor_composition.py | 8 +- ...t_session_skills_allow_only_and_closure.py | 32 ++ tests/workspace/test_skills_projection.py | 43 +++ tests/workspace/test_skills_resolver.py | 21 ++ 20 files changed, 470 insertions(+), 109 deletions(-) diff --git a/docs/skills/visibility.md b/docs/skills/visibility.md index 97de0d5faf..6b4de08f0e 100644 --- a/docs/skills/visibility.md +++ b/docs/skills/visibility.md @@ -121,8 +121,8 @@ validated before session initialization or filesystem writes. The selected canonical documents are then projected into an exact, content-keyed plugin artifact. Machine-only authority (`uses_capabilities`, `execution_role`, -`activate_deps`, and retired `backend_requirements`) stays in AutoSkillit's private -contract and is omitted from every model-facing `SKILL.md`. Claude Code receives the +`activate_deps`, `interactive_only`, and retired `backend_requirements`) stays in +AutoSkillit's private contract and is omitted from every model-facing `SKILL.md`. Claude Code receives the sanitized projection through the same launch binding whose shared descriptor reaches the child, rather than a raw `skills_extended/` directory. Exact ORCHESTRATOR-role entries such as `process-issues` remain excluded from the L1 catalog, diff --git a/src/autoskillit/core/types/launch/_type_skill_contract.py b/src/autoskillit/core/types/launch/_type_skill_contract.py index cd0dcef2ce..db76f7d1d8 100644 --- a/src/autoskillit/core/types/launch/_type_skill_contract.py +++ b/src/autoskillit/core/types/launch/_type_skill_contract.py @@ -49,6 +49,7 @@ "activate_deps", "execution_role", "exploration_vectors", + "interactive_only", "requires_resources", "semantic_version", "semantic_requirements", diff --git a/src/autoskillit/migration/store.py b/src/autoskillit/migration/store.py index 91d2abd0e1..8abafcfea2 100644 --- a/src/autoskillit/migration/store.py +++ b/src/autoskillit/migration/store.py @@ -74,15 +74,3 @@ def has_failure(self, name: str) -> bool: def default_store_path(project_dir: Path, temp_dir: Path | None = None) -> Path: base = temp_dir if temp_dir is not None else resolve_temp_dir(project_dir, None) return base / "migrations" / "failures.json" - - -def record_from_skill( - name: str, - file_path: str, - file_type: str, - error: str, - retries_attempted: int, -) -> None: - """Entry point for migrate-recipes SKILL.md via run_python when retries exhausted.""" - store_path = default_store_path(Path.cwd()) - FailureStore(store_path).record(name, Path(file_path), file_type, error, retries_attempted) diff --git a/src/autoskillit/server/AGENTS.md b/src/autoskillit/server/AGENTS.md index 62e55efecf..b9579607d4 100644 --- a/src/autoskillit/server/AGENTS.md +++ b/src/autoskillit/server/AGENTS.md @@ -99,5 +99,9 @@ The canonical tool sets are in `core/types/constants/_type_constants_registries. INSPECTION members may also carry `kitchen` - `ALL_VISIBILITY_TAGS` — the source-of-truth set of conditional visibility tags disabled at startup - Tool tag and tier changes are checked against bundled skill and recipe consumers by - `tests/server/test_consumer_tool_callability.py`; each `EXEMPTIONS` entry needs a - cited PR, issue, or `path:line` + `tests/server/test_consumer_tool_callability.py`; safe-by-design entries are typed + `ConsumerExemption`s (`decision` cites the deciding issue; `guard` quotes an anchor + sentence that must exist in the skill text); known defects go in + `DEFERRED_VIOLATIONS` as `TrackedDeferral`s (issue, dated 180-day time bomb, + regression test). Every SESSION skill is checked in both headless contexts unless + it declares `interactive_only: true`. diff --git a/src/autoskillit/skills/AGENTS.md b/src/autoskillit/skills/AGENTS.md index c896798577..449f9d5b87 100644 --- a/src/autoskillit/skills/AGENTS.md +++ b/src/autoskillit/skills/AGENTS.md @@ -33,6 +33,10 @@ the categories/visibility conventions applied to user-invocable skills. model-initiated invocation of `open-kitchen` and `close-kitchen`. Do not remove this flag without an explicit architectural decision. +`interactive_only:` is a boolean; true forbids every headless invocation of +the skill and of any closure containing it (same concept as the +`interactive_only` hook/tool session scope). + ## Multi-concern files Files with no registration support their folder; files with several diff --git a/src/autoskillit/skills/close-kitchen/SKILL.md b/src/autoskillit/skills/close-kitchen/SKILL.md index db10587d53..83a9512afd 100644 --- a/src/autoskillit/skills/close-kitchen/SKILL.md +++ b/src/autoskillit/skills/close-kitchen/SKILL.md @@ -4,6 +4,7 @@ write_paths: inherit uses_capabilities: [open_kitchen] description: Close the AutoSkillit kitchen — hides kitchen MCP tools for this session. disable-model-invocation: true +interactive_only: true --- # Close Kitchen diff --git a/src/autoskillit/skills/open-kitchen/SKILL.md b/src/autoskillit/skills/open-kitchen/SKILL.md index a19233abc0..d90e8e6e71 100644 --- a/src/autoskillit/skills/open-kitchen/SKILL.md +++ b/src/autoskillit/skills/open-kitchen/SKILL.md @@ -4,6 +4,7 @@ write_paths: inherit uses_capabilities: [open_kitchen] description: Open the AutoSkillit kitchen — reveals all kitchen MCP tools for this session. Human-only entry point. disable-model-invocation: true +interactive_only: true --- # Open Kitchen diff --git a/src/autoskillit/skills_extended/AGENTS.md b/src/autoskillit/skills_extended/AGENTS.md index 085798333a..e7a58dab2a 100644 --- a/src/autoskillit/skills_extended/AGENTS.md +++ b/src/autoskillit/skills_extended/AGENTS.md @@ -54,6 +54,9 @@ gaps, currently out-of-scope for this plan): - `uses_capabilities:` — declared MCP capabilities (`run_skill`, `test_check`, `open_kitchen`, `github_api_write`, etc.). Drives capability-gated dispatch and audit trails. +- `interactive_only:` — boolean; true forbids every headless invocation of + this skill and of any closure containing it (same concept as the + `interactive_only` hook/tool session scope). ## Multi-concern files diff --git a/src/autoskillit/skills_extended/make-campaign/SKILL.md b/src/autoskillit/skills_extended/make-campaign/SKILL.md index 29caedfc61..107d783951 100644 --- a/src/autoskillit/skills_extended/make-campaign/SKILL.md +++ b/src/autoskillit/skills_extended/make-campaign/SKILL.md @@ -1,5 +1,6 @@ --- name: make-campaign +interactive_only: true write_paths: unrestricted description: Interactively author a campaign recipe YAML through a 6-phase guided workflow. Use when user says "make campaign", "create campaign", "author campaign", "new campaign recipe", or wants to decompose a campaign goal into dispatches. categories: diff --git a/src/autoskillit/skills_extended/migrate-recipes/SKILL.md b/src/autoskillit/skills_extended/migrate-recipes/SKILL.md index 48bf3cb70e..33866bdade 100644 --- a/src/autoskillit/skills_extended/migrate-recipes/SKILL.md +++ b/src/autoskillit/skills_extended/migrate-recipes/SKILL.md @@ -27,7 +27,7 @@ Apply versioned migration notes to an AutoSkillit recipe. ## When to Use - Called by an agent when user confirms migration (via load_recipe suggestion) -- Called by `autoskillit migrate` CLI command via `run_skill` +- Launched headlessly by the migration service when the `migrate_recipe` tool migrates a stale recipe - Can be invoked directly by user ## Arguments @@ -77,19 +77,3 @@ The orchestrator provides all context in the prompt: - If all 3 retry attempts fail validation, output the best attempt with a clear warning - If no patterns are detected (script already up to date), stamp the version and report no changes needed - If write-recipe produces unexpected output, report the error and preserve the original script - -## Failure Persistence - -If all 3 retry attempts are exhausted without a valid result, BEFORE declaring failure -you MUST persist the failure record: - - run_python: - callable: autoskillit.migration.store.record_from_skill - args: - name: {recipe stem, e.g. "my-pipeline"} - file_path: {absolute path received as script_path argument} - file_type: recipe - error: {description of last validation error} - retries_attempted: 3 - -After recording, output a clear failure summary and stop. diff --git a/src/autoskillit/skills_extended/write-recipe/SKILL.md b/src/autoskillit/skills_extended/write-recipe/SKILL.md index 26d5d7fac9..3cb127dec9 100644 --- a/src/autoskillit/skills_extended/write-recipe/SKILL.md +++ b/src/autoskillit/skills_extended/write-recipe/SKILL.md @@ -29,6 +29,8 @@ Format a workflow into a YAML recipe following the workflow schema. No positional arguments. The skill prompts interactively for workflow details. +In a headless session, skip every step that asks the user or calls an MCP kitchen tool; apply the format rules in this document, save the YAML, and return — the recipe is validated when it is next loaded. + ## Critical Constraints **NEVER:** @@ -41,7 +43,7 @@ No positional arguments. The skill prompts interactively for workflow details. **ALWAYS:** - Save the script to `.autoskillit/recipes/{name}.yaml` as the ONLY output -- Call `validate_recipe` after saving and fix any errors +- In an interactive session, ensure the saved recipe validates cleanly (Standalone Invocation Flow step 8 / Edit Mode step 5). In a headless session, follow the document-wide mode rule and return the saved YAML for host validation. - Use "recipe" terminology (not "skill script") ## Output Path Isolation (Required) @@ -540,7 +542,7 @@ When called directly as `/autoskillit:write-recipe`: - `DEAD_OUTPUT`: A `capture:` key is never referenced by any reachable downstream step via `${{ context.X }}`. Either add a `${{ context.X }}` reference in the downstream step's `with:` block, or remove the unused capture. - `IMPLICIT_HANDOFF`: A `run_skill` step has no `capture:` block. Add a `capture:` block to explicitly wire outputs to downstream steps via `${{ context.X }}`, or confirm the skill's output is intentionally unused. - Present the quality summary to the user and fix any warnings that indicate broken wiring. -9. After validation passes, generate the pipeline contract file by calling `generate_recipe_card` on the saved script. This creates `.autoskillit/recipes/contracts/{name}.yaml` alongside the recipe. Use `run_python` with `autoskillit.recipe.contracts.generate_recipe_card` passing the script path and scripts directory, or rely on `load_recipe` which auto-generates contracts on first load. +9. The pipeline contract card is generated automatically the next time the recipe is loaded; do not generate it manually. 10. Tell the user: "Saved to `.autoskillit/recipes/{name}.yaml`. Load it with `load_recipe("{name}")` via the MCP tool." After telling the user, emit the structured output token as the very last line of your @@ -594,7 +596,7 @@ When the agent is given an existing script's YAML content and a requested change 5. Call `validate_recipe` on the saved path 6. If errors, fix and re-validate until clean 7. Review quality warnings and fix data-flow issues before reporting changes -8. After validation passes, regenerate the pipeline contract file to reflect the changes. Use `run_python` with `autoskillit.recipe.contracts.generate_recipe_card` or rely on `load_recipe` auto-generation on next load. +8. The pipeline contract card is generated automatically the next time the recipe is loaded; do not generate it manually. 9. Report the changes made This edit mode is invoked when `load_recipe` routes the user's modification request through this skill. The skill receives the existing YAML as context. diff --git a/src/autoskillit/workspace/skills/_records.py b/src/autoskillit/workspace/skills/_records.py index bbb19e299f..7f61a04a1f 100644 --- a/src/autoskillit/workspace/skills/_records.py +++ b/src/autoskillit/workspace/skills/_records.py @@ -161,6 +161,15 @@ def __post_init__(self) -> None: def canonical_bytes(self) -> bytes: return self.canonical_content.encode("utf-8") + @property + def interactive_only(self) -> bool: + frontmatter = self.frontmatter + return ( + frontmatter is not None + and frontmatter.data is not None + and frontmatter.data.get("interactive_only") is True + ) + @property def source_identity(self) -> SkillSourceIdentity: """Return the validated effective source identity.""" @@ -319,6 +328,10 @@ def __post_init__(self) -> None: raise SkillContractError( f"invalid effective invocation contract for {member.name!r}: {reason}" ) + if member.interactive_only: + raise SkillContractError( + f"interactive-only skill {member.name!r} cannot run in a headless invocation" + ) if member.execution_role is not self.execution_role: actual = ( member.execution_role.value if member.execution_role is not None else "invalid" diff --git a/tests/contracts/test_activate_deps_completeness.py b/tests/contracts/test_activate_deps_completeness.py index 1470c8fcf8..ea98fb97d8 100644 --- a/tests/contracts/test_activate_deps_completeness.py +++ b/tests/contracts/test_activate_deps_completeness.py @@ -13,6 +13,7 @@ import pytest +from autoskillit.core import SkillExecutionRole, SkillSource from autoskillit.core.types.constants._type_constants import SKILL_ACTIVATE_DEPS_REQUIRED from autoskillit.core.types.constants._type_constants_registries import PACK_REGISTRY from autoskillit.workspace.session_skills import ( @@ -160,6 +161,43 @@ def test_required_activate_deps_present() -> None: assert not failures, "Missing required activate_deps:\n" + "\n".join(failures) +def test_noninteractive_session_skill_closures_exclude_interactive_only( + provider: SkillsDirectoryProvider, +) -> None: + failures: dict[str, list[str]] = {} + for skill in provider.resolver.list_all(): + if ( + skill.source not in {SkillSource.BUNDLED, SkillSource.BUNDLED_EXTENDED} + or skill.execution_role is not SkillExecutionRole.SESSION + or skill.interactive_only + ): + continue + interactive_members = sorted( + name + for name in compute_skill_closure(skill.name, provider) + if (member := provider.resolver.resolve(name)) is not None and member.interactive_only + ) + if interactive_members: + failures[skill.name] = interactive_members + assert not failures, f"headless skill closures contain interactive-only members: {failures}" + + +def test_required_activate_dependencies_are_not_interactive_only( + provider: SkillsDirectoryProvider, +) -> None: + required = { + dependency + for dependencies in SKILL_ACTIVATE_DEPS_REQUIRED.values() + for dependency in dependencies + } + interactive = sorted( + name + for name in required + if (skill := provider.resolver.resolve(name)) is not None and skill.interactive_only + ) + assert not interactive, f"required activate_deps are interactive-only: {interactive}" + + def test_all_activate_deps_resolve() -> None: """Every name in any SKILL.md's activate_deps must resolve to a real skill or pack. diff --git a/tests/migration/test_store.py b/tests/migration/test_store.py index 64707c1563..fe20dab91c 100644 --- a/tests/migration/test_store.py +++ b/tests/migration/test_store.py @@ -13,7 +13,6 @@ FailureStore, MigrationFailure, default_store_path, - record_from_skill, ) pytestmark = [pytest.mark.layer("migration"), pytest.mark.small] @@ -129,28 +128,6 @@ def test_has_failure_false(tmp_path: Path) -> None: assert store.has_failure("absent") is False -# --------------------------------------------------------------------------- -# FS10: record_from_skill() records to cwd/.autoskillit/temp/migrations/failures.json -# --------------------------------------------------------------------------- -def test_record_from_skill_helper(tmp_path: Path) -> None: - with patch("autoskillit.migration.store.Path.cwd", return_value=tmp_path): - record_from_skill( - name="my-pipeline", - file_path="/abs/path/my-pipeline.yaml", - file_type="recipe", - error="validation failed", - retries_attempted=3, - ) - expected_path = tmp_path / ".autoskillit" / "temp" / "migrations" / "failures.json" - assert expected_path.exists() - store = FailureStore(expected_path) - failures = store.load() - assert "my-pipeline" in failures - f = failures["my-pipeline"] - assert f.file_path == "/abs/path/my-pipeline.yaml" - assert f.retries_attempted == 3 - - # --------------------------------------------------------------------------- # FS11: default_store_path(project_dir) returns correct path # --------------------------------------------------------------------------- diff --git a/tests/server/test_consumer_tool_callability.py b/tests/server/test_consumer_tool_callability.py index a24a7964a8..0d4b7867a1 100644 --- a/tests/server/test_consumer_tool_callability.py +++ b/tests/server/test_consumer_tool_callability.py @@ -1,13 +1,17 @@ """Bundled consumers must be able to call tools in their actual session catalogs. -Each exempted violation in EXEMPTIONS cites the decision or guard that makes -the call safe; the assertion failure message explains the remedy order for -adding a new exemption. +Every SESSION skill is checked in both headless contexts unless it declares +``interactive_only: true``. Safe-by-design exceptions are typed and verified; +known defects are tracked as dated deferrals with regression tests. """ from __future__ import annotations +import re +from dataclasses import dataclass +from datetime import date from pathlib import Path +from typing import Literal import pytest @@ -18,6 +22,7 @@ all_tool_names, resolve_skill_name, ) +from autoskillit.core.tool_registry import get_tool_def from autoskillit.recipe.io import ( list_recipes as list_recipe_infos, ) @@ -35,65 +40,176 @@ _has_tool_operation, _logical_lines, _source_lines, + _SourceLine, +) +from autoskillit.workspace.skills import DefaultSkillResolver, SkillInfo +from tests.arch._deferred_debt import ( + TrackedDeferral, + assert_deferrals_have_regression_tests, + assert_not_stale, + assert_rationale_present, ) -from autoskillit.workspace.skills import DefaultSkillResolver from tests.server._session_catalogs import CatalogContext, SessionCatalog, build_session_catalog pytestmark = [pytest.mark.layer("server"), pytest.mark.medium] ExemptionKey = tuple[str, str, CatalogContext] -EXEMPTIONS: dict[ExemptionKey, str] = { + +@dataclass(frozen=True, slots=True) +class ConsumerExemption: + kind: Literal["decision", "guard"] + rationale: str + issue: int | None = None + anchor: str | None = None + + def __post_init__(self) -> None: + if len(self.rationale.strip()) < 20: + raise ValueError("exemption rationale must explain why the call is safe") + if self.kind == "decision" and self.issue is None: + raise ValueError("decision exemption must cite the deciding issue number") + if self.kind == "guard" and not (self.anchor and self.anchor.strip()): + raise ValueError("guard exemption must quote the guarding sentence as anchor") + + +_REVIEW_HEADLESS_GUARD = ( + "stop with `verdict=needs_human`, print `%%REVIEW_GATE::CLEAR%%`, and exit in a " + "headless session" +) +_WRITE_RECIPE_HEADLESS_GUARD = ( + "In a headless session, skip every step that asks the user or calls an MCP kitchen " + "tool; apply the format rules in this document, save the YAML, and return — the " + "recipe is validated when it is next loaded." +) + +EXEMPTIONS: dict[ExemptionKey, ConsumerExemption] = { ( "skill:audit-claims", "post_pr_review", CatalogContext.INTERACTIVE_SKILL, - ): "#4448 (ba2ccd424): review publication is headless-only", + ): ConsumerExemption( + "decision", + issue=4448, + rationale="review publication is headless-only by decision (ba2ccd424)", + ), ( "skill:resolve-review", "post_pr_review", CatalogContext.INTERACTIVE_SKILL, - ): "#4448 (ba2ccd424): review publication is headless-only", + ): ConsumerExemption( + "decision", + issue=4448, + rationale="review publication is headless-only by decision (ba2ccd424)", + ), ( "skill:review-pr", "post_pr_review", CatalogContext.INTERACTIVE_SKILL, - ): "#4448 (ba2ccd424): review publication is headless-only", + ): ConsumerExemption( + "decision", + issue=4448, + rationale="review publication is headless-only by decision (ba2ccd424)", + ), ( "skill:review-research-pr", "post_pr_review", CatalogContext.INTERACTIVE_SKILL, - ): "#4448 (ba2ccd424): review publication is headless-only", + ): ConsumerExemption( + "decision", + issue=4448, + rationale="review publication is headless-only by decision (ba2ccd424)", + ), ( "skill:review-pr", "run_python", CatalogContext.HEADLESS_SKILL, - ): "review-pr/SKILL.md:284-296 exits before this call when AUTOSKILLIT_HEADLESS=1", + ): ConsumerExemption( + "guard", + rationale="review-pr exits before annotation tooling in headless sessions", + anchor=_REVIEW_HEADLESS_GUARD, + ), ( "skill:review-pr", "run_python", CatalogContext.HEADLESS_SKILL_AUTO_GATE, - ): "review-pr/SKILL.md:284-296 exits before this call when AUTOSKILLIT_HEADLESS=1", + ): ConsumerExemption( + "guard", + rationale="review-pr exits before annotation tooling in headless sessions", + anchor=_REVIEW_HEADLESS_GUARD, + ), ( "skill:resolve-failures", "run_cmd", CatalogContext.HEADLESS_SKILL, - ): "resolve-failures/SKILL.md:165 says not via Bash or run_cmd", + ): ConsumerExemption( + "guard", + rationale="negated mention in the test_check instruction, not a run_cmd call", + anchor="not via Bash or `run_cmd`", + ), ( "skill:resolve-failures", "run_cmd", CatalogContext.HEADLESS_SKILL_AUTO_GATE, - ): "resolve-failures/SKILL.md:165 says not via Bash or run_cmd", + ): ConsumerExemption( + "guard", + rationale="negated mention in the test_check instruction, not a run_cmd call", + anchor="not via Bash or `run_cmd`", + ), + ( + "skill:write-recipe", + "kitchen_status", + CatalogContext.HEADLESS_SKILL, + ): ConsumerExemption( + "guard", + rationale="the document-wide mode rule skips kitchen tools in headless sessions", + anchor=_WRITE_RECIPE_HEADLESS_GUARD, + ), + ( + "skill:write-recipe", + "load_recipe", + CatalogContext.HEADLESS_SKILL, + ): ConsumerExemption( + "guard", + rationale="the document-wide mode rule skips kitchen tools in headless sessions", + anchor=_WRITE_RECIPE_HEADLESS_GUARD, + ), + ( + "skill:write-recipe", + "validate_recipe", + CatalogContext.HEADLESS_SKILL, + ): ConsumerExemption( + "guard", + rationale="the document-wide mode rule skips kitchen tools in headless sessions", + anchor=_WRITE_RECIPE_HEADLESS_GUARD, + ), +} + +DEFERRED_VIOLATIONS: dict[ExemptionKey, TrackedDeferral] = { ( "skill:process-issues", "batch_cleanup_clones", CatalogContext.INTERACTIVE_ORCHESTRATOR, - ): "#5179: fleet mutation tool is hidden from interactive orchestrators", + ): TrackedDeferral( + issue=5179, + added_date=date(2026, 9, 25), + rationale="fleet mutation tool is hidden from interactive orchestrators", + regression_test=( + "tests/server/test_consumer_tool_callability.py::" + "test_skill_tool_operations_are_callable" + ), + ), ( "recipe:implement-findings:batch_cleanup", "batch_cleanup_clones", CatalogContext.INTERACTIVE_ORCHESTRATOR, - ): "#5179: fleet mutation tool is hidden from interactive orchestrators", + ): TrackedDeferral( + issue=5179, + added_date=date(2026, 9, 25), + rationale="fleet mutation tool is hidden from interactive orchestrators", + regression_test=( + "tests/server/test_consumer_tool_callability.py::test_recipe_tool_steps_are_callable" + ), + ), } @@ -105,19 +221,87 @@ def bundled_recipes(tmp_path: Path) -> tuple[tuple[RecipeInfo, Recipe], ...]: return tuple((info, load_recipe_file(info.path)) for info in infos) +def _logical_text(logical: tuple[_SourceLine, ...]) -> str: + return _LOGICAL_CONTINUATION_RE.sub(" ", "\n".join(line.text for line in logical)) + + +def _pseudo_call_operation( + logicals: tuple[tuple[_SourceLine, ...], ...], + index: int, + tool_names: set[str], +) -> tuple[str, int] | None: + logical = logicals[index] + text = _logical_text(logical) + match = re.fullmatch(r"(?P[a-z_]+):\s*", text.strip()) + if match is None or (tool := match.group("tool")) not in tool_names: + return None + parameter_logical = next( + ( + candidate + for candidate in logicals[index + 1 :] + if (candidate_text := _logical_text(candidate)).strip() + and not candidate_text.lstrip().startswith("#") + ), + None, + ) + if parameter_logical is None: + return None + parameter_text = _logical_text(parameter_logical) + parameter = re.match(r"^\s+(?P[a-z_]+)\s*:", parameter_text) + header_indent = len(text) - len(text.lstrip()) + parameter_indent = len(parameter_text) - len(parameter_text.lstrip()) + definition = get_tool_def(tool) + if ( + parameter is None + or parameter_indent <= header_indent + or definition is None + or parameter.group("param") not in {item.name for item in definition.params} + ): + return None + evidence = _evidence(tool, logical + parameter_logical) + return (tool, evidence.source_span[0]) if evidence.is_genuine else None + + def _skill_tool_operations(content: str) -> list[tuple[str, int]]: operations: list[tuple[str, int]] = [] - tool_names = sorted(all_tool_names()) - for logical in _logical_lines(_source_lines(content)): - text = _LOGICAL_CONTINUATION_RE.sub(" ", "\n".join(line.text for line in logical)) - for tool in tool_names: + tool_names = set(all_tool_names()) + logicals = _logical_lines(_source_lines(content)) + for index, logical in enumerate(logicals): + text = _logical_text(logical) + for tool in sorted(tool_names): if _has_tool_operation(text, tool): evidence = _evidence(tool, logical) if evidence.is_genuine: operations.append((tool, evidence.source_span[0])) + pseudo_call = _pseudo_call_operation(logicals, index, tool_names) + if pseudo_call is not None: + operations.append(pseudo_call) return operations +def _is_exempt(key: ExemptionKey) -> bool: + return key in EXEMPTIONS or key in DEFERRED_VIOLATIONS + + +def _skill_contexts(skill: SkillInfo) -> set[CatalogContext]: + role = skill.execution_role + assert role is not None, f"{skill.name}: missing execution role; {skill.invalidities=}" + match role: + case SkillExecutionRole.SESSION: + if skill.interactive_only: + return {CatalogContext.INTERACTIVE_SKILL} + return { + CatalogContext.INTERACTIVE_SKILL, + CatalogContext.HEADLESS_SKILL, + CatalogContext.HEADLESS_SKILL_AUTO_GATE, + } + case SkillExecutionRole.ORCHESTRATOR: + return {CatalogContext.INTERACTIVE_ORCHESTRATOR} + case SkillExecutionRole.FLEET: + return {CatalogContext.FLEET_DISPATCH} + raise AssertionError(f"unknown skill execution role: {role!r}") + + def _violation( consumer: str, tool: str, @@ -142,10 +326,13 @@ def _assert_contract( consumer_prefix: str, contexts: set[CatalogContext], ) -> None: - scoped = { - key for key in EXEMPTIONS if key[0].startswith(consumer_prefix) and key[2] in contexts - } - stale = sorted(scoped - violations) + registries = {"EXEMPTIONS": EXEMPTIONS, "DEFERRED_VIOLATIONS": DEFERRED_VIOLATIONS} + stale = sorted( + f"{registry_name}: {key!r}" + for registry_name, registry in registries.items() + for key in registry + if key[0].startswith(consumer_prefix) and key[2] in contexts and key not in violations + ) assert not unexpected and not stale, ( "Bundled tool callability violations:\n" + "\n".join(unexpected) @@ -168,38 +355,16 @@ def _assert_contract( ) async def test_skill_tool_operations_are_callable( context: CatalogContext, - bundled_recipes: tuple[tuple[RecipeInfo, Recipe], ...], monkeypatch: pytest.MonkeyPatch, build_ctx, ) -> None: - child_skills = { - name - for _, recipe in bundled_recipes - for step in recipe.steps.values() - if step.tool == "run_skill" - if (name := resolve_skill_name(str(step.with_args.get("skill_command", "")))) is not None - } catalog = await build_session_catalog(context, monkeypatch=monkeypatch, build_ctx=build_ctx) callable_tools = catalog.callable_tools() unexpected: list[str] = [] violations: set[ExemptionKey] = set() for skill in DefaultSkillResolver().list_all(): - assert skill.execution_role is not None, ( - f"{skill.name}: missing execution role; invalidities={skill.invalidities}" - ) - match skill.execution_role: - case SkillExecutionRole.SESSION: - contexts = {CatalogContext.INTERACTIVE_SKILL} - if skill.name in child_skills: - contexts.update( - {CatalogContext.HEADLESS_SKILL, CatalogContext.HEADLESS_SKILL_AUTO_GATE} - ) - case SkillExecutionRole.ORCHESTRATOR: - contexts = {CatalogContext.INTERACTIVE_ORCHESTRATOR} - case SkillExecutionRole.FLEET: - contexts = {CatalogContext.FLEET_DISPATCH} - if context not in contexts: + if context not in _skill_contexts(skill): continue consumer = f"skill:{skill.name}" for tool, line in _skill_tool_operations(skill.canonical_content): @@ -207,7 +372,7 @@ async def test_skill_tool_operations_are_callable( continue key = (consumer, tool, context) violations.add(key) - if key not in EXEMPTIONS: + if not _is_exempt(key): unexpected.append(_violation(consumer, tool, line, context, catalog)) _assert_contract(unexpected, violations, consumer_prefix="skill:", contexts={context}) @@ -252,7 +417,7 @@ async def test_recipe_tool_steps_are_callable( continue key = (consumer, step.tool, context) violations.add(key) - if key not in EXEMPTIONS: + if not _is_exempt(key): unexpected.append(_violation(consumer, step.tool, line, context, catalog)) _assert_contract( @@ -261,3 +426,57 @@ async def test_recipe_tool_steps_are_callable( consumer_prefix="recipe:", contexts={CatalogContext.INTERACTIVE_ORCHESTRATOR, CatalogContext.FOOD_TRUCK}, ) + + +def test_tool_operation_detection_recognizes_pseudo_call() -> None: + content = ( + "# Workflow\n\n1. Persist the failure:\n\n run_python:\n callable: module.record\n" + ) + assert _skill_tool_operations(content) == [("run_python", 5)] + + +def test_tool_operation_detection_rejects_unregistered_parameter() -> None: + content = "1. Write config:\n\n```config.yaml\ntest_check:\n command: task test\n```\n" + assert _skill_tool_operations(content) == [] + + +def test_tool_operation_detection_rejects_same_indent_parameter() -> None: + content = "1. Run the callable:\n\nrun_python:\ncallable: module.record\n" + assert _skill_tool_operations(content) == [] + + +def test_guard_exemption_anchors_exist() -> None: + skills = {skill.name: skill for skill in DefaultSkillResolver().list_all()} + for (consumer, _tool, _context), exemption in EXEMPTIONS.items(): + if exemption.kind != "guard": + continue + assert consumer.startswith("skill:") + skill_name = consumer.removeprefix("skill:") + anchor = " ".join((exemption.anchor or "").split()) + assert anchor in " ".join(skills[skill_name].canonical_content.split()) + + +def test_deferred_violations_are_tracked() -> None: + assert_not_stale(DEFERRED_VIOLATIONS, registry_name="DEFERRED_VIOLATIONS") + assert_rationale_present(DEFERRED_VIOLATIONS, registry_name="DEFERRED_VIOLATIONS") + assert_deferrals_have_regression_tests( + DEFERRED_VIOLATIONS, + registry_name="DEFERRED_VIOLATIONS", + ) + + +def test_bundled_run_skill_targets_are_not_interactive_only( + bundled_recipes: tuple[tuple[RecipeInfo, Recipe], ...], +) -> None: + skills = {skill.name: skill for skill in DefaultSkillResolver().list_all()} + targets = { + name + for _, recipe in bundled_recipes + for step in recipe.steps.values() + if step.tool == "run_skill" + if (name := resolve_skill_name(str(step.with_args.get("skill_command", "")))) is not None + } + interactive_targets = sorted(name for name in targets if skills[name].interactive_only) + assert not interactive_targets, ( + f"bundled recipes dispatch interactive-only skills: {interactive_targets}" + ) diff --git a/tests/server/test_session_hook_root_scope.py b/tests/server/test_session_hook_root_scope.py index d14d78194f..e021a1d63a 100644 --- a/tests/server/test_session_hook_root_scope.py +++ b/tests/server/test_session_hook_root_scope.py @@ -9,6 +9,7 @@ from __future__ import annotations +import json from pathlib import Path from types import SimpleNamespace from typing import Any, cast @@ -452,6 +453,30 @@ def _install_direct_dispatch_session_manager(tool_ctx: Any, tmp_path: Path) -> N ) +def test_direct_dispatch_returns_crashed_result_for_interactive_only_skill( + tool_ctx: Any, + tmp_path: Path, +) -> None: + _install_direct_dispatch_skill(tool_ctx, name="direct-skill") + _install_direct_dispatch_session_manager(tool_ctx, tmp_path) + tool_ctx.skill_resolver.resolve_invocation.side_effect = SkillContractError( + "interactive-only skill 'direct-skill' cannot run in a headless invocation" + ) + + dispatch, error = _prepare_direct_skill_dispatch( + "/direct-skill", + tool_ctx.project_dir, + tool_ctx, + ) + + assert dispatch is None + assert error is not None + parsed = json.loads(error) + assert parsed["subtype"] == "crashed" + assert "SkillContractError" in parsed["result"] + assert "interactive-only" in parsed["result"] + + def test_direct_dispatch_holds_session_hook_root_until_cleanup( tool_ctx: Any, tmp_path: Path ) -> None: diff --git a/tests/workspace/test_corridor_composition.py b/tests/workspace/test_corridor_composition.py index 8a13afa929..6d2551d72f 100644 --- a/tests/workspace/test_corridor_composition.py +++ b/tests/workspace/test_corridor_composition.py @@ -61,7 +61,9 @@ def test_factory_result_forwarded_to_setup_session_dir(self, tmp_path: Path) -> source_infos = tuple( s for s in resolver.list_all() - if s.source is SkillSource.BUNDLED and not s.exploration_vectors + if s.source is SkillSource.BUNDLED + and not s.exploration_vectors + and not s.interactive_only )[:1] if not source_infos: pytest.skip("need at least one bundled skill without exploration vectors") @@ -105,7 +107,9 @@ def test_no_factory_no_explorer_binding(self, tmp_path: Path) -> None: source_infos = tuple( s for s in resolver.list_all() - if s.source is SkillSource.BUNDLED and not s.exploration_vectors + if s.source is SkillSource.BUNDLED + and not s.exploration_vectors + and not s.interactive_only )[:1] if not source_infos: pytest.skip("need at least one bundled skill without exploration vectors") diff --git a/tests/workspace/test_session_skills_allow_only_and_closure.py b/tests/workspace/test_session_skills_allow_only_and_closure.py index 2e90086cdc..1d3c36ad33 100644 --- a/tests/workspace/test_session_skills_allow_only_and_closure.py +++ b/tests/workspace/test_session_skills_allow_only_and_closure.py @@ -142,6 +142,7 @@ def _write_invocation_skill( execution_role: str = "session", deps: tuple[str, ...] = (), categories: tuple[str, ...] = (), + interactive_only: bool = False, ) -> None: skill_path = root / name / "SKILL.md" skill_path.parent.mkdir(parents=True, exist_ok=True) @@ -157,6 +158,8 @@ def _write_invocation_skill( frontmatter.append(f"activate_deps: [{', '.join(deps)}]") if categories: frontmatter.append(f"categories: [{', '.join(categories)}]") + if interactive_only: + frontmatter.append("interactive_only: true") evidence = { "github_api_write": "Run `gh issue edit 1 --body-file issue.md`.", "test_check": "Call `test_check()`.", @@ -184,6 +187,35 @@ def _make_effective_resolver(tmp_path: Path, monkeypatch, skills: dict[str, dict class TestEffectiveInvocationClosurePolicy: """The complete direct/pack closure supplies one validated capability contract.""" + def test_interactive_only_root_is_rejected( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from autoskillit.core import SkillExecutionRole + + resolver = _make_effective_resolver( + tmp_path, + monkeypatch, + {"root": {"interactive_only": True}}, + ) + with pytest.raises(SkillContractError, match="interactive-only"): + resolver.resolve_invocation("root", tmp_path, SkillExecutionRole.SESSION) + + def test_interactive_only_closure_member_is_rejected( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from autoskillit.core import SkillExecutionRole + + resolver = _make_effective_resolver( + tmp_path, + monkeypatch, + { + "root": {"deps": ("interactive-dependency",)}, + "interactive-dependency": {"interactive_only": True}, + }, + ) + with pytest.raises(SkillContractError, match="interactive-only"): + resolver.resolve_invocation("root", tmp_path, SkillExecutionRole.SESSION) + def test_direct_closure_does_not_scan_unrelated_catalog( self, tmp_path: Path, monkeypatch ) -> None: diff --git a/tests/workspace/test_skills_projection.py b/tests/workspace/test_skills_projection.py index f8ff15f763..74e34762f2 100644 --- a/tests/workspace/test_skills_projection.py +++ b/tests/workspace/test_skills_projection.py @@ -43,6 +43,19 @@ def test_effective_catalog_is_path_free(tmp_path: Path) -> None: ) +def test_interactive_cook_catalog_retains_kitchen_skills(tmp_path: Path) -> None: + from autoskillit.workspace.skills import DefaultSkillResolver + + catalog = DefaultSkillResolver().list_effective( + tmp_path, + SkillExecutionRole.SESSION, + cook_session=True, + ) + + names = {skill.name for skill in catalog.skills} + assert {"open-kitchen", "close-kitchen"} <= names + + def _resolver_with_visibility_skills(tmp_path: Path): from autoskillit.workspace.skills import DefaultSkillResolver @@ -621,6 +634,7 @@ def test_projection_strips_all_machine_authority_and_preserves_private_deps( "activate_deps", "execution_role", "exploration_vectors", + "interactive_only", "requires_resources", "semantic_requirements", "semantic_version", @@ -666,6 +680,35 @@ def test_projection_strips_all_machine_authority_and_preserves_private_deps( assert entry.activate_deps == ("dependency",) +def test_projection_strips_interactive_only_from_open_kitchen(tmp_path: Path) -> None: + from autoskillit.workspace import ( + EffectiveSkillCatalog, + SkillCatalogEntry, + SkillProjectionContext, + parse_frontmatter_content, + project_agent_skill_document, + ) + from autoskillit.workspace.skills import DefaultSkillResolver + + info = DefaultSkillResolver().resolve("open-kitchen") + assert info is not None + assert info.interactive_only + entry = SkillCatalogEntry.from_skill_info(info) + catalog = EffectiveSkillCatalog( + skills=(entry,), + execution_role=SkillExecutionRole.SESSION, + ) + + document = project_agent_skill_document( + entry, + SkillProjectionContext(cwd=tmp_path, catalog=catalog), + ) + projected = parse_frontmatter_content(document.content) + + assert projected.is_valid and projected.data is not None + assert "interactive_only" not in projected.data + + @pytest.mark.parametrize( ("source", "expected_reference"), [ diff --git a/tests/workspace/test_skills_resolver.py b/tests/workspace/test_skills_resolver.py index d170a158e7..8e1f052f70 100644 --- a/tests/workspace/test_skills_resolver.py +++ b/tests/workspace/test_skills_resolver.py @@ -465,6 +465,27 @@ def test_resolve_instance_cache_caches_none(monkeypatch: pytest.MonkeyPatch) -> class TestSkillExecutionRoleParsing: + @pytest.mark.parametrize( + ("declaration", "expected"), + [ + ("interactive_only: true\n", True), + ("", False), + ('interactive_only: "yes"\n', False), + ], + ) + def test_interactive_only_requires_literal_true( + self, + tmp_path: Path, + declaration: str, + expected: bool, + ) -> None: + from autoskillit.workspace.skills import _skill_info_from_frontmatter + + skill_md = tmp_path / "SKILL.md" + skill_md.write_text(f"---\nname: test\nwrite_paths: inherit\n{declaration}---\n# body") + info = _skill_info_from_frontmatter("test", SkillSource.BUNDLED, skill_md) + assert info.interactive_only is expected + def test_valid_omission_defaults_to_session(self, tmp_path: Path) -> None: from autoskillit.workspace.skills import _skill_info_from_frontmatter From 4e51da8516cba33ae80f73e9cb374b1b7f040338 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 20:23:22 -0700 Subject: [PATCH 03/15] feat: validate migrated recipes before write-back --- docs/recipes/authoring.md | 22 +- .../types/results/_type_results_records.py | 36 +- src/autoskillit/migration/adapters_recipe.py | 137 ++++++- src/autoskillit/migration/engine.py | 5 - src/autoskillit/migration/service.py | 4 +- src/autoskillit/recipe/api/_api_listing.py | 21 +- src/autoskillit/server/_factory.py | 4 +- .../skills_extended/migrate-recipes/SKILL.md | 40 +- tests/_helpers.py | 16 + tests/contracts/test_fakes_conformance.py | 1 + tests/migration/test_engine.py | 15 +- tests/migration/test_engine_recipe.py | 365 +++++++++++++++++- tests/migration/test_service.py | 60 ++- tests/recipe/test_api_orchestration.py | 99 ++++- tests/recipe/test_repository.py | 1 + tests/recipe/test_validation_result_types.py | 22 +- .../server/test_consumer_tool_callability.py | 12 +- .../test_factory_context_construction.py | 18 +- tests/server/test_service_wrappers.py | 16 +- tests/server/test_tools_migrate_recipe.py | 69 ++-- 20 files changed, 790 insertions(+), 173 deletions(-) diff --git a/docs/recipes/authoring.md b/docs/recipes/authoring.md index 6568547fa5..25516b7c6f 100644 --- a/docs/recipes/authoring.md +++ b/docs/recipes/authoring.md @@ -125,16 +125,18 @@ contract card. The Haiku verdict is one of `FRESH`, `STALE`, or `UNCERTAIN`; When a recipe declares an `autoskillit_version` older than the running package, the migration engine in `migration/engine.py` walks the migration -notes under `src/autoskillit/migrations/` and applies them in order. Each -note is a 7-step sequence: - -1. Read current YAML. -2. Validate it parses. -3. Apply the note's transformer. -4. Validate the transformed YAML. -5. Diff old vs new. -6. Write the new YAML atomically. -7. Record the new `autoskillit_version`. +notes under `src/autoskillit/migrations/` and applies them in order: + +1. The service selects the applicable migration notes. +2. It launches the `migrate-recipes` headless skill once. +3. The host requires the expected output file and the target `autoskillit_version` stamp. +4. The host validates the candidate with the same `validate_from_path` entry and configured + temp placeholder as the `validate_recipe` tool. Candidate structure, semantics, contracts, + and binding projection are compared by full finding identity against the original recipe, + while contract lookup remains anchored beside the original. Validator and metadata + exceptions fail normally; if original validation raised, the candidate must be fully valid. +5. The engine writes the accepted candidate atomically and retains a `.yaml.bak` backup. +6. The service clears a prior failure on success or records the rejected migration. Failures are recorded in `migration/store.py:FailureStore` (atomic JSON writes) so the next `autoskillit migrate` run can resume. diff --git a/src/autoskillit/core/types/results/_type_results_records.py b/src/autoskillit/core/types/results/_type_results_records.py index e7d921b05a..853fd28b99 100644 --- a/src/autoskillit/core/types/results/_type_results_records.py +++ b/src/autoskillit/core/types/results/_type_results_records.py @@ -274,11 +274,29 @@ class RecipePathValidationReport(TypedDict): quality: dict[str, object] findings: list[dict[str, str]] contracts: list[dict[str, Any]] + binding_projection_failures: list[dict[str, str]] RecipePathValidationResult = RecipePathValidationInputError | RecipePathValidationReport +def _is_string_dict_list(value: object) -> bool: + return isinstance(value, list) and all( + isinstance(item, dict) + and all( + isinstance(key, str) and isinstance(item_value, str) + for key, item_value in item.items() + ) + for item in value + ) + + +def _is_string_key_dict_list(value: object) -> bool: + return isinstance(value, list) and all( + isinstance(item, dict) and all(isinstance(key, str) for key in item) for item in value + ) + + def is_recipe_path_validation_report( result: object, ) -> TypeGuard[RecipePathValidationReport]: @@ -288,6 +306,7 @@ def is_recipe_path_validation_report( "quality", "findings", "contracts", + "binding_projection_failures", }: return False @@ -296,25 +315,16 @@ def is_recipe_path_validation_report( quality = result["quality"] findings = result["findings"] contracts = result["contracts"] + binding_projection_failures = result["binding_projection_failures"] return ( isinstance(valid, bool) and isinstance(errors, list) and all(isinstance(error, str) for error in errors) and isinstance(quality, dict) and all(isinstance(key, str) for key in quality) - and isinstance(findings, list) - and all( - isinstance(finding, dict) - and all( - isinstance(key, str) and isinstance(value, str) for key, value in finding.items() - ) - for finding in findings - ) - and isinstance(contracts, list) - and all( - isinstance(contract, dict) and all(isinstance(key, str) for key in contract) - for contract in contracts - ) + and _is_string_dict_list(findings) + and _is_string_key_dict_list(contracts) + and _is_string_dict_list(binding_projection_failures) ) diff --git a/src/autoskillit/migration/adapters_recipe.py b/src/autoskillit/migration/adapters_recipe.py index b4cdb31424..3b5305e556 100644 --- a/src/autoskillit/migration/adapters_recipe.py +++ b/src/autoskillit/migration/adapters_recipe.py @@ -6,9 +6,14 @@ from pathlib import Path from autoskillit import __version__ -from autoskillit.core import YAMLError, dump_yaml_str, get_logger +from autoskillit.core import ( + RecipePathValidationReport, + dump_yaml_str, + get_logger, + is_recipe_path_validation_report, + temp_dir_display_str, +) from autoskillit.migration.engine import ( - MIGRATE_RECIPES_MAX_RETRIES, HeadlessMigrationAdapter, HeadlessRunner, MigrationFile, @@ -19,11 +24,36 @@ logger = get_logger(__name__) +def _completed_report_errors(result: RecipePathValidationReport) -> dict[str, str]: + errors = {f"structural:{message}": message for message in result["errors"]} + for finding in result["findings"]: + if finding.get("severity") == "error": + message = finding.get("message", "") + key = f"semantic:{finding.get('rule', '')}@{finding.get('step', '')}:{message}" + errors[key] = message + for finding in result["contracts"]: + if finding.get("severity") == "error": + message = str(finding.get("message", "")) + key = f"contract:{finding.get('rule', '')}@{finding.get('step', '')}:{message}" + errors[key] = message + for failure in result["binding_projection_failures"]: + message = failure.get("message", "") + key = ( + f"binding:{failure.get('code', '')}@{failure.get('step_name', '')}" + f"@{failure.get('name', '')}:{message}" + ) + errors[key] = message + return errors + + class RecipeMigrationAdapter(HeadlessMigrationAdapter): """Headless adapter for LLM-driven recipe version migration.""" file_type = "recipe" + def __init__(self, *, temp_dir_override: str | None = None) -> None: + self._temp_dir_display = temp_dir_display_str(temp_dir_override) + def discover(self, project_dir: Path) -> list[MigrationFile]: from autoskillit.recipe import parse_recipe_metadata @@ -79,6 +109,7 @@ async def migrate( ] ) target_version = migrations[-1].to_version + baseline = self._validation_errors(file.path) content = file.path.read_text() skill_command = ( f"/autoskillit:migrate-recipes" @@ -88,6 +119,8 @@ async def migrate( f" target_version={target_version}" ) + temp_out = self.get_temp_output_path(file, temp_dir) + temp_out.unlink(missing_ok=True) raw = await run_headless( skill_command=skill_command, cwd=str(file.path.parent.parent.parent), @@ -97,10 +130,9 @@ async def migrate( success=False, name=file.name, error=raw.result or "headless session failed", - retries_attempted=MIGRATE_RECIPES_MAX_RETRIES, + retries_attempted=0, ) - temp_out = self.get_temp_output_path(file, temp_dir) if not temp_out.exists(): return MigrationResult( success=False, @@ -108,27 +140,96 @@ async def migrate( error="migrate-recipes did not produce output", ) - return MigrationResult( - success=True, - name=file.name, - migrated_content=temp_out.read_text(), - ) + try: + from autoskillit.recipe import parse_recipe_metadata + + if parse_recipe_metadata(temp_out).version != target_version: + return MigrationResult( + success=False, + name=file.name, + error=( + f"migrated recipe is not stamped with autoskillit_version {target_version}" + ), + ) + + output_errors = self._validation_errors( + temp_out, + contract_source_path=file.path, + ) + usable_baseline = {} if "exception" in baseline else baseline + new_errors = { + key: message + for key, message in output_errors.items() + if key not in usable_baseline + } + if new_errors: + summary = "; ".join(message[:500] for message in list(new_errors.values())[:5]) + return MigrationResult( + success=False, + name=file.name, + error=f"migrated recipe introduced validation errors: {summary}", + ) + return MigrationResult( + success=True, + name=file.name, + migrated_content=temp_out.read_text(), + ) + except Exception as exc: + logger.warning( + "Migrated recipe validation failed", + path=str(temp_out), + error=str(exc), + exc_info=True, + ) + return MigrationResult( + success=False, + name=file.name, + error=f"{type(exc).__name__}: {exc}", + ) def get_temp_output_path(self, file: MigrationFile, temp_dir: Path) -> Path: return temp_dir / "migrations" / f"{file.path.stem}.yaml" def validate(self, path: Path) -> tuple[bool, str]: - from autoskillit.recipe import load_recipe as _parse_recipe - from autoskillit.recipe import validate_recipe_structure + errors = self._validation_errors(path) + return not errors, "; ".join(errors.values()) + + def _validation_errors( + self, + path: Path, + *, + contract_source_path: Path | None = None, + ) -> dict[str, str]: + from autoskillit.recipe import load_recipe, validate_from_path, validate_recipe_structure + errors: dict[str, str] = {} try: - recipe = _parse_recipe(path) - errors = validate_recipe_structure(recipe) - if errors: - return False, "; ".join(str(e) for e in errors) - return True, "" - except (OSError, ValueError, YAMLError) as exc: + recipe = load_recipe(path) + errors.update( + (f"structural:{message}", message) for message in validate_recipe_structure(recipe) + ) + except Exception as exc: + logger.warning( + "Recipe file validation failed", path=str(path), error=str(exc), exc_info=True + ) + errors["exception"] = f"{type(exc).__name__}: {exc}" + + try: + result = validate_from_path( + path, + temp_dir_relpath=self._temp_dir_display, + contract_source_path=contract_source_path, + ) + if is_recipe_path_validation_report(result): + errors.update(_completed_report_errors(result)) + else: + errors.update( + (f"input:{finding['error']}", finding["error"]) + for finding in result["findings"] + ) + except Exception as exc: logger.warning( "Recipe file validation failed", path=str(path), error=str(exc), exc_info=True ) - return False, str(exc) + errors["exception"] = f"{type(exc).__name__}: {exc}" + return errors diff --git a/src/autoskillit/migration/engine.py b/src/autoskillit/migration/engine.py index bcef8050af..e93acd5f2d 100644 --- a/src/autoskillit/migration/engine.py +++ b/src/autoskillit/migration/engine.py @@ -21,10 +21,6 @@ logger = get_logger(__name__) -MIGRATE_RECIPES_MAX_RETRIES: int = 3 -"""Max validation-retry attempts for LLM-driven recipe migration (matches SKILL.md).""" - - # Shared shape for any callable that drives a headless skill execution and # returns a SkillResult. Centralized here (engine) so the abstract contract, # the concrete engine, and every adapter share one canonical name. @@ -183,7 +179,6 @@ async def migrate_file( "AdvisoryResult", "DeterministicMigrationAdapter", "HeadlessMigrationAdapter", - "MIGRATE_RECIPES_MAX_RETRIES", "MigrationAdapter", "MigrationEngine", "MigrationFile", diff --git a/src/autoskillit/migration/service.py b/src/autoskillit/migration/service.py index e8e2b53192..104fc62c97 100644 --- a/src/autoskillit/migration/service.py +++ b/src/autoskillit/migration/service.py @@ -125,11 +125,11 @@ async def _get_diagram_advisory( return None -def default_migration_engine() -> MigrationEngine: +def default_migration_engine(*, temp_dir_override: str | None = None) -> MigrationEngine: """Create a MigrationEngine with all bundled adapters registered.""" return MigrationEngine( [ - RecipeMigrationAdapter(), + RecipeMigrationAdapter(temp_dir_override=temp_dir_override), ContractMigrationAdapter(), DiagramMigrationAdapter(), SkillMigrationAdapter(), diff --git a/src/autoskillit/recipe/api/_api_listing.py b/src/autoskillit/recipe/api/_api_listing.py index 574cf08c82..6e2bc0be5c 100644 --- a/src/autoskillit/recipe/api/_api_listing.py +++ b/src/autoskillit/recipe/api/_api_listing.py @@ -94,6 +94,7 @@ def validate_from_path( path: Path, temp_dir_relpath: str = ".autoskillit/temp", *, + contract_source_path: Path | None = None, lister: SkillLister | None = None, backend_name: str | None = None, ingredient_overrides: dict[str, str] | None = None, @@ -108,11 +109,13 @@ def validate_from_path( temp_dir_relpath: Relative path to the temp directory used for ``{{AUTOSKILLIT_TEMP}}`` substitution. Defaults to ``.autoskillit/temp``. + contract_source_path: Optional original recipe path whose neighboring + contract card should validate the candidate at ``path``. Returns: Completed validation returns ``valid``, ``errors``, ``quality``, - ``findings``, and ``contracts``. File and parse errors return only - ``valid`` and ``findings``. + ``findings``, ``contracts``, and ``binding_projection_failures``. File + and parse errors return only ``valid`` and ``findings``. """ if not path.is_file(): return _invalid_validation_result(f"File not found: {path}") @@ -188,8 +191,9 @@ def validate_from_path( semantic = findings_to_dicts(semantic_findings) contract_findings: list[dict[str, Any]] = [] - recipes_dir = path.parent - recipe_name = path.stem + contract_path = contract_source_path if contract_source_path is not None else path + recipes_dir = contract_path.parent + recipe_name = contract_path.stem contract = load_recipe_card(recipe_name, recipes_dir) if contract: contract_findings = validate_recipe_cards(recipe, contract) @@ -204,4 +208,13 @@ def validate_from_path( "quality": quality, "findings": semantic, "contracts": contract_findings, + "binding_projection_failures": [ + { + "code": failure.code.value, + "step_name": failure.step_name, + "name": failure.name, + "message": failure.message, + } + for failure in ctx.binding_projection.failures + ], } diff --git a/src/autoskillit/server/_factory.py b/src/autoskillit/server/_factory.py index 097a9ce30c..512f02a886 100644 --- a/src/autoskillit/server/_factory.py +++ b/src/autoskillit/server/_factory.py @@ -573,6 +573,8 @@ async def _run_migration_headless(*args: Any, **kwargs: Any) -> SkillResult: return result ctx.migrations = DefaultMigrationService( - default_migration_engine(), run_headless=_run_migration_headless + default_migration_engine(temp_dir_override=config.workspace.temp_dir), + run_headless=_run_migration_headless, + temp_dir=temp_dir, ) return ctx diff --git a/src/autoskillit/skills_extended/migrate-recipes/SKILL.md b/src/autoskillit/skills_extended/migrate-recipes/SKILL.md index 33866bdade..118acbd73a 100644 --- a/src/autoskillit/skills_extended/migrate-recipes/SKILL.md +++ b/src/autoskillit/skills_extended/migrate-recipes/SKILL.md @@ -3,10 +3,8 @@ name: migrate-recipes write_paths: - '{{AUTOSKILLIT_TEMP}}/migrations/' uses_capabilities: [] -activate_deps: -- write-recipe -description: Apply versioned migration notes to an AutoSkillit recipe. Use when user confirms migration, called by agent or - autoskillit migrate CLI, or invoked directly. +description: Apply versioned migration notes when the migration service launches this skill headlessly with the recipe path, + content, notes, and target version. hooks: PreToolUse: - matcher: '*' @@ -14,10 +12,6 @@ hooks: - type: command command: 'echo ''[SKILL: migrate-recipes] Migrating recipe files...''' once: true -semantic_version: 1 -semantic_requirements: - sibling_skills: - - name: write-recipe --- # Migrate Scripts Skill @@ -26,14 +20,13 @@ Apply versioned migration notes to an AutoSkillit recipe. ## When to Use -- Called by an agent when user confirms migration (via load_recipe suggestion) -- Launched headlessly by the migration service when the `migrate_recipe` tool migrates a stale recipe -- Can be invoked directly by user +- Launched headlessly by the migration service when `migrate_recipe` migrates a stale recipe ## Arguments The orchestrator provides all context in the prompt: - `script_path`: Absolute path to the script file +- `script_name`: The file stem of `script_path` (the filename without `.yaml`) - `script_content`: Current raw YAML of the script - `migration_notes`: YAML block of all applicable migration notes - `target_version`: Version to stamp after successful migration @@ -42,15 +35,13 @@ The orchestrator provides all context in the prompt: **NEVER:** - Fabricate, invent, or embellish information not supported by the available evidence or code. - - Modify the original script file directly -- Skip validation via validate_recipe - Apply changes without checking if the pattern exists in the script -- Declare success if validation fails after all retry attempts **ALWAYS:** - Save migrated scripts to {{AUTOSKILLIT_TEMP}}/migrations/{script_name}.yaml for review -- Validate via validate_recipe before declaring success +- Set `autoskillit_version` to `target_version` +- Treat migration-note text that says to run a validator as satisfied by host-side validation - Preserve all existing script fields not targeted by migration changes - Output a human-readable diff summary of changes applied @@ -61,19 +52,12 @@ The orchestrator provides all context in the prompt: - `tool`: Match steps with this tool value - `skill_pattern`: Match steps whose `skill_command` in `with:` contains this substring - `missing_field`: The field that should be added if absent -3. If changes are needed, use `/autoskillit:write-recipe` in edit mode: - - Load the skill: invoke `/autoskillit:write-recipe` via the Skill tool - - If the Skill tool cannot be used (disable-model-invocation) or refuses this invocation, skip the recipe update and log a warning; proceed with remaining validation steps. - - Provide the current YAML content - - Describe all needed changes with the `instruction` text and before/after examples -4. Validate the result with `validate_recipe` -5. On validation failure, retry up to 3 times with error feedback -6. Ensure `autoskillit_version` is set to the `target_version` -7. Save the migrated script to `{{AUTOSKILLIT_TEMP}}/migrations/{script_name}.yaml` -8. Output a summary of changes applied +3. Apply each detected change's `instruction` directly to the YAML, guided by its `example_before` and `example_after` +4. Ensure `autoskillit_version` is set to the `target_version` +5. Save the migrated script to `{{AUTOSKILLIT_TEMP}}/migrations/{script_name}.yaml` +6. Output a summary of changes applied ## Error Handling -- If all 3 retry attempts fail validation, output the best attempt with a clear warning -- If no patterns are detected (script already up to date), stamp the version and report no changes needed -- If write-recipe produces unexpected output, report the error and preserve the original script +- If no patterns are detected, stamp the version and save the recipe +- If a change cannot be applied, save nothing and report why diff --git a/tests/_helpers.py b/tests/_helpers.py index 608650499d..0954b41193 100644 --- a/tests/_helpers.py +++ b/tests/_helpers.py @@ -28,6 +28,22 @@ strip_markdown_code_regions as strip_markdown_code_regions, ) +MINIMAL_VALID_RECIPE_YAML = """\ +name: minimal-recipe +description: minimal test recipe +kitchen_rules: + - Use the included stop step sentinel protocol. +ingredients: + task: + description: A task ingredient + required: true + default: my-default-task-value +steps: + done: + action: stop + message: Task completed successfully with full sentinel L3 block emitted for completion. +""" + _RUN_SKILL_WINDOW = 400 _PROSE_TRIGGER_WINDOW = 60 _PROSE_TRIGGER_WORDS = ("parameter", "pass", "forward") diff --git a/tests/contracts/test_fakes_conformance.py b/tests/contracts/test_fakes_conformance.py index d10e7cef98..20f9c623e6 100644 --- a/tests/contracts/test_fakes_conformance.py +++ b/tests/contracts/test_fakes_conformance.py @@ -508,6 +508,7 @@ def test_recipe_repository_validate_from_path_returns_configured_report(): "quality": {}, "findings": [], "contracts": [], + "binding_projection_failures": [], } repo.set_path_validated(str(script_path), report) diff --git a/tests/migration/test_engine.py b/tests/migration/test_engine.py index 93d4ef6aef..b24575e696 100644 --- a/tests/migration/test_engine.py +++ b/tests/migration/test_engine.py @@ -1,4 +1,4 @@ -"""Tests for the MigrationEngine default registration and MIGRATE_RECIPES_MAX_RETRIES.""" +"""Tests for the MigrationEngine default adapter registration.""" from __future__ import annotations @@ -17,6 +17,7 @@ MigrationResult, ) from autoskillit.migration.service import default_migration_engine +from tests._helpers import MINIMAL_VALID_RECIPE_YAML from .conftest import make_migration_note, make_skill_result @@ -68,18 +69,22 @@ async def test_engine_writes_back_on_successful_headless_run( original_content = "name: mypipe\n" recipe_path.write_text(original_content) - new_content = "name: mypipe\n# migrated\nautoskillit_version: '1.0.0'\n" + new_content = "autoskillit_version: '1.0.0'\n" + MINIMAL_VALID_RECIPE_YAML temp_dir = tmp_path / ".autoskillit" / "temp" temp_out = temp_dir / "migrations" / "mypipe.yaml" - temp_out.parent.mkdir(parents=True) - temp_out.write_text(new_content) monkeypatch.setattr( _patch_migration_adapters_recipe, "applicable_migrations", lambda *a, **kw: [make_migration_note()], ) - mock_headless = AsyncMock(return_value=make_skill_result(True)) + + async def write_output(**_kwargs: object): + temp_out.parent.mkdir(parents=True, exist_ok=True) + temp_out.write_text(new_content) + return make_skill_result(True) + + mock_headless = AsyncMock(side_effect=write_output) file = MigrationFile( name="mypipe", path=recipe_path, file_type="recipe", current_version="0.0.1" diff --git a/tests/migration/test_engine_recipe.py b/tests/migration/test_engine_recipe.py index fff6d6586a..4bd47133cb 100644 --- a/tests/migration/test_engine_recipe.py +++ b/tests/migration/test_engine_recipe.py @@ -4,23 +4,54 @@ import textwrap from pathlib import Path -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, patch import pytest import autoskillit.migration.adapters_recipe as _patch_migration_adapters_recipe from autoskillit.core.paths import pkg_root from autoskillit.migration.adapters_recipe import RecipeMigrationAdapter -from autoskillit.migration.engine import ( - MIGRATE_RECIPES_MAX_RETRIES, - MigrationFile, -) +from autoskillit.migration.engine import MigrationFile +from tests._helpers import MINIMAL_VALID_RECIPE_YAML from .conftest import make_migration_note, make_skill_result pytestmark = [pytest.mark.layer("migration"), pytest.mark.small] +def _stamped(content: str, version: str = "1.0.0") -> str: + return f"autoskillit_version: '{version}'\n{content}" + + +def _migration_file(tmp_path: Path, content: str) -> MigrationFile: + recipe_path = tmp_path / ".autoskillit" / "recipes" / "myrecipe.yaml" + recipe_path.parent.mkdir(parents=True, exist_ok=True) + recipe_path.write_text(content) + return MigrationFile( + name="myrecipe", + path=recipe_path, + file_type="recipe", + current_version="0.0.1", + ) + + +def _runner_writing(path: Path, content: str) -> AsyncMock: + async def _write_output(**_kwargs: object): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + return make_skill_result(True) + + return AsyncMock(side_effect=_write_output) + + +def _enable_migration(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + _patch_migration_adapters_recipe, + "applicable_migrations", + lambda *a, **kw: [make_migration_note()], + ) + + class TestRecipeMigrationAdapter: # ME1 def test_recipe_adapter_discover_finds_recipes( @@ -105,18 +136,15 @@ async def test_recipe_adapter_build_skill_command( recipe_path.parent.mkdir(parents=True) recipe_path.write_text("name: myrecipe\n") - # Pre-create temp output so migrate() doesn't return "no output" failure temp_dir = tmp_path / ".autoskillit" / "temp" temp_out = temp_dir / "migrations" / "myrecipe.yaml" - temp_out.parent.mkdir(parents=True) - temp_out.write_text("name: myrecipe\n# migrated\n") monkeypatch.setattr( _patch_migration_adapters_recipe, "applicable_migrations", lambda *a, **kw: [make_migration_note()], ) - mock_headless = AsyncMock(return_value=make_skill_result(True)) + mock_headless = _runner_writing(temp_out, _stamped(MINIMAL_VALID_RECIPE_YAML)) adapter = RecipeMigrationAdapter() file = MigrationFile( @@ -191,9 +219,9 @@ def test_recipe_adapter_validate_errors_non_empty_branch(self, tmp_path: Path) - assert "kitchen_rules" in error.lower() -class TestMigrateRecipesConstant: +class TestRecipeMigrationValidation: @pytest.mark.anyio - async def test_failed_headless_retries_match_constant( + async def test_failed_headless_session_is_reported( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: recipe_path = tmp_path / ".autoskillit" / "recipes" / "myrecipe.yaml" @@ -214,4 +242,317 @@ async def test_failed_headless_retries_match_constant( ) result = await adapter.migrate(file, run_headless=mock_rh, temp_dir=tmp_path) assert not result.success - assert result.retries_attempted == MIGRATE_RECIPES_MAX_RETRIES + assert result.retries_attempted == 0 + mock_rh.assert_awaited_once() + + @pytest.mark.anyio + @pytest.mark.parametrize( + ("candidate", "expected"), + [ + ( + _stamped(MINIMAL_VALID_RECIPE_YAML.replace("kitchen_rules:", "rules:")), + "kitchen_rules", + ), + ( + _stamped( + MINIMAL_VALID_RECIPE_YAML.replace( + " done:\n", + " bad:\n" + " tool: run_skill\n" + " with:\n" + " skill_command: /autoskillit:project-local\n" + " cwd: .\n" + " on_success: done\n" + " done:\n", + ) + ), + "project-local", + ), + ( + _stamped( + MINIMAL_VALID_RECIPE_YAML.replace( + " done:\n", + " bad:\n" + " tool: no_such_tool\n" + " with:\n" + " bogus: value\n" + " on_success: done\n" + " done:\n", + ) + ), + "no_such_tool", + ), + ], + ids=["structural", "semantic", "binding"], + ) + async def test_migrate_rejects_errors_introduced_by_the_migration( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + candidate: str, + expected: str, + ) -> None: + _enable_migration(monkeypatch) + adapter = RecipeMigrationAdapter() + file = _migration_file(tmp_path, _stamped(MINIMAL_VALID_RECIPE_YAML, "0.0.1")) + temp_dir = tmp_path / "temp" + runner = _runner_writing(adapter.get_temp_output_path(file, temp_dir), candidate) + + result = await adapter.migrate(file, run_headless=runner, temp_dir=temp_dir) + + assert not result.success + assert expected in (result.error or "") + runner.assert_awaited_once() + + @pytest.mark.anyio + async def test_migrate_tolerates_pre_existing_errors( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + _enable_migration(monkeypatch) + invalid = MINIMAL_VALID_RECIPE_YAML.replace( + " done:\n", + " local:\n" + " tool: run_skill\n" + " with:\n" + " skill_command: /autoskillit:project-local\n" + " cwd: .\n" + " on_success: done\n" + " done:\n", + ) + adapter = RecipeMigrationAdapter() + file = _migration_file(tmp_path, _stamped(invalid, "0.0.1")) + temp_dir = tmp_path / "temp" + runner = _runner_writing(adapter.get_temp_output_path(file, temp_dir), _stamped(invalid)) + + result = await adapter.migrate(file, run_headless=runner, temp_dir=temp_dir) + + assert result.success + + @pytest.mark.anyio + async def test_migrate_rejects_new_errors_alongside_baseline_errors( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + _enable_migration(monkeypatch) + baseline = MINIMAL_VALID_RECIPE_YAML.replace("kitchen_rules:", "rules:") + candidate = baseline.replace( + " done:\n", + " bad:\n" + " tool: no_such_tool\n" + " with:\n" + " bogus: value\n" + " on_success: done\n" + " done:\n", + ) + adapter = RecipeMigrationAdapter() + file = _migration_file(tmp_path, _stamped(baseline, "0.0.1")) + temp_dir = tmp_path / "temp" + runner = _runner_writing(adapter.get_temp_output_path(file, temp_dir), _stamped(candidate)) + + result = await adapter.migrate(file, run_headless=runner, temp_dir=temp_dir) + + assert not result.success + assert "no_such_tool" in (result.error or "") + + @pytest.mark.anyio + async def test_migrate_keeps_distinct_findings_from_same_rule_and_step( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + _enable_migration(monkeypatch) + original = """\ +name: myrecipe +description: Capture fixture. +kitchen_rules: [Follow the stop protocol.] +ingredients: + plan: + description: Plan path. + required: true + default: /tmp/plan.md +steps: + implement: + tool: run_skill + with: + skill_command: /autoskillit:implement-worktree-no-merge ${{ inputs.plan }} + cwd: . + capture: + first_bad: ${{ result.first_bad }} + on_success: done + on_failure: done + done: + action: stop + message: Done. +""" + candidate = original.replace( + " first_bad: ${{ result.first_bad }}\n", + " first_bad: ${{ result.first_bad }}\n" + " second_bad: ${{ result.second_bad }}\n", + ) + adapter = RecipeMigrationAdapter() + file = _migration_file(tmp_path, _stamped(original, "0.0.1")) + temp_dir = tmp_path / "temp" + runner = _runner_writing(adapter.get_temp_output_path(file, temp_dir), _stamped(candidate)) + + result = await adapter.migrate(file, run_headless=runner, temp_dir=temp_dir) + + assert not result.success + assert "second_bad" in (result.error or "") + + @pytest.mark.anyio + async def test_migrate_uses_original_contract_card_context( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + _enable_migration(monkeypatch) + adapter = RecipeMigrationAdapter() + file = _migration_file(tmp_path, _stamped(MINIMAL_VALID_RECIPE_YAML, "0.0.1")) + source_contracts = file.path.parent / "contracts" + source_contracts.mkdir() + (source_contracts / "myrecipe.yaml").write_text( + "dataflow:\n - step: original-card\n available: []\n required: [source_only]\n" + ) + temp_dir = tmp_path / "temp" + temp_out = adapter.get_temp_output_path(file, temp_dir) + temp_contracts = temp_out.parent / "contracts" + temp_contracts.mkdir(parents=True) + (temp_contracts / "myrecipe.yaml").write_text( + "dataflow:\n - step: temp-card\n available: []\n required: [temp_only]\n" + ) + runner = _runner_writing(temp_out, _stamped(MINIMAL_VALID_RECIPE_YAML)) + + result = await adapter.migrate(file, run_headless=runner, temp_dir=temp_dir) + + assert result.success + + @pytest.mark.anyio + @pytest.mark.parametrize("version", [None, "0.9.0"], ids=["missing", "old"]) + async def test_migrate_rejects_unstamped_output( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + version: str | None, + ) -> None: + _enable_migration(monkeypatch) + adapter = RecipeMigrationAdapter() + file = _migration_file(tmp_path, _stamped(MINIMAL_VALID_RECIPE_YAML, "0.0.1")) + temp_dir = tmp_path / "temp" + candidate = ( + MINIMAL_VALID_RECIPE_YAML + if version is None + else _stamped(MINIMAL_VALID_RECIPE_YAML, version) + ) + runner = _runner_writing(adapter.get_temp_output_path(file, temp_dir), candidate) + + result = await adapter.migrate(file, run_headless=runner, temp_dir=temp_dir) + + assert not result.success + assert "autoskillit_version 1.0.0" in (result.error or "") + + @pytest.mark.anyio + async def test_migrate_does_not_reuse_stale_output( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + _enable_migration(monkeypatch) + adapter = RecipeMigrationAdapter() + file = _migration_file(tmp_path, _stamped(MINIMAL_VALID_RECIPE_YAML, "0.0.1")) + temp_dir = tmp_path / "temp" + temp_out = adapter.get_temp_output_path(file, temp_dir) + temp_out.parent.mkdir(parents=True) + temp_out.write_text(_stamped(MINIMAL_VALID_RECIPE_YAML)) + runner = AsyncMock(return_value=make_skill_result(True)) + + result = await adapter.migrate(file, run_headless=runner, temp_dir=temp_dir) + + assert not result.success + assert "did not produce output" in (result.error or "") + + @pytest.mark.anyio + async def test_migrate_survives_validator_exceptions( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + _enable_migration(monkeypatch) + adapter = RecipeMigrationAdapter() + file = _migration_file(tmp_path, _stamped(MINIMAL_VALID_RECIPE_YAML, "0.0.1")) + temp_dir = tmp_path / "temp" + malformed = _stamped("name: x\nsteps:\n a:\n tool: run_cmd\n with: hello\n") + runner = _runner_writing(adapter.get_temp_output_path(file, temp_dir), malformed) + + result = await adapter.migrate(file, run_headless=runner, temp_dir=temp_dir) + + assert not result.success + assert "TypeError" in (result.error or "") + + @pytest.mark.anyio + @pytest.mark.parametrize("candidate_valid", [True, False]) + async def test_baseline_validator_exception_requires_valid_output( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + candidate_valid: bool, + ) -> None: + _enable_migration(monkeypatch) + malformed = ( + "autoskillit_version: '0.0.1'\n" + "name: x\nsteps:\n a:\n tool: run_cmd\n with: hello\n" + ) + adapter = RecipeMigrationAdapter() + file = _migration_file(tmp_path, malformed) + temp_dir = tmp_path / "temp" + candidate = ( + _stamped(MINIMAL_VALID_RECIPE_YAML) + if candidate_valid + else malformed.replace("0.0.1", "1.0.0") + ) + runner = _runner_writing(adapter.get_temp_output_path(file, temp_dir), candidate) + + result = await adapter.migrate(file, run_headless=runner, temp_dir=temp_dir) + + assert result.success is candidate_valid + runner.assert_awaited_once() + + def test_recipe_adapter_validate_runs_full_validation(self, tmp_path: Path) -> None: + recipe_path = tmp_path / "semantic-invalid.yaml" + recipe_path.write_text( + MINIMAL_VALID_RECIPE_YAML.replace( + " done:\n", + " bad:\n" + " tool: no_such_tool\n" + " with:\n" + " bogus: value\n" + " on_success: done\n" + " done:\n", + ) + ) + + valid, message = RecipeMigrationAdapter().validate(recipe_path) + + assert not valid + assert message + + @pytest.mark.anyio + async def test_adapter_passes_temp_display_to_both_validations( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + _enable_migration(monkeypatch) + adapter = RecipeMigrationAdapter(temp_dir_override="custom-temp") + file = _migration_file(tmp_path, _stamped(MINIMAL_VALID_RECIPE_YAML, "0.0.1")) + temp_dir = tmp_path / "custom-temp" + runner = _runner_writing( + adapter.get_temp_output_path(file, temp_dir), _stamped(MINIMAL_VALID_RECIPE_YAML) + ) + completed = { + "valid": True, + "errors": [], + "quality": {}, + "findings": [], + "contracts": [], + "binding_projection_failures": [], + } + + with patch("autoskillit.recipe.validate_from_path", return_value=completed) as validate: + result = await adapter.migrate(file, run_headless=runner, temp_dir=temp_dir) + + assert result.success + assert validate.call_count == 2 + assert all( + call.kwargs["temp_dir_relpath"] == "custom-temp" for call in validate.call_args_list + ) + assert validate.call_args_list[0].kwargs["contract_source_path"] is None + assert validate.call_args_list[1].kwargs["contract_source_path"] == file.path diff --git a/tests/migration/test_service.py b/tests/migration/test_service.py index 71180407a4..b3075c0a0a 100644 --- a/tests/migration/test_service.py +++ b/tests/migration/test_service.py @@ -7,12 +7,14 @@ import pytest +import autoskillit.migration.adapters_recipe as _patch_migration_adapters_recipe import autoskillit.migration.service as _patch_migration_service from autoskillit.migration.engine import ( AdvisoryResult, MigrationResult, ) from autoskillit.migration.service import DefaultMigrationService, default_migration_engine +from tests._helpers import MINIMAL_VALID_RECIPE_YAML from .conftest import make_migration_note, make_skill_result @@ -75,6 +77,11 @@ async def test_no_headless_runner_returns_error( "_applicable", lambda *a, **kw: [make_migration_note()], ) + monkeypatch.setattr( + _patch_migration_adapters_recipe, + "applicable_migrations", + lambda *a, **kw: [make_migration_note()], + ) service = DefaultMigrationService(default_migration_engine()) result = await service.migrate(recipe_path) @@ -89,11 +96,9 @@ async def test_successful_migration_with_headless( ) -> None: recipe_path = _seed_recipe(tmp_path) - # Pre-create temp output so RecipeMigrationAdapter finds migrated content temp_dir = tmp_path / ".autoskillit" / "temp" temp_out = temp_dir / "migrations" / "myrecipe.yaml" - temp_out.parent.mkdir(parents=True) - temp_out.write_text("name: myrecipe\n# migrated\nautoskillit_version: '1.0.0'\n") + migrated_content = "autoskillit_version: '1.0.0'\n" + MINIMAL_VALID_RECIPE_YAML # Patch the module-level ``_applicable`` alias so the service # sees a single migration note. With the headless runner wired in, @@ -104,8 +109,18 @@ async def test_successful_migration_with_headless( "_applicable", lambda *a, **kw: [make_migration_note()], ) + monkeypatch.setattr( + _patch_migration_adapters_recipe, + "applicable_migrations", + lambda *a, **kw: [make_migration_note()], + ) + + async def write_output(**_kwargs: object): + temp_out.parent.mkdir(parents=True, exist_ok=True) + temp_out.write_text(migrated_content) + return make_skill_result(True) - mock_headless = AsyncMock(return_value=make_skill_result(True)) + mock_headless = AsyncMock(side_effect=write_output) service = DefaultMigrationService( default_migration_engine(), run_headless=mock_headless, temp_dir=temp_dir ) @@ -158,6 +173,43 @@ async def test_headless_runner_failure_records_to_failure_store( entry = failure_store.load()["myrecipe"] assert entry.error == "boom" + @pytest.mark.anyio + async def test_malformed_migrated_metadata_records_failure_without_writeback( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + recipe_path = _seed_recipe(tmp_path) + original = recipe_path.read_bytes() + note = make_migration_note() + monkeypatch.setattr(_patch_migration_service, "_applicable", lambda *a, **kw: [note]) + monkeypatch.setattr( + _patch_migration_adapters_recipe, + "applicable_migrations", + lambda *a, **kw: [note], + ) + temp_dir = tmp_path / "temp" + temp_out = temp_dir / "migrations" / "myrecipe.yaml" + + async def write_malformed(**_kwargs: object): + temp_out.parent.mkdir(parents=True, exist_ok=True) + temp_out.write_text("[") + return make_skill_result(True) + + service = DefaultMigrationService( + default_migration_engine(), + run_headless=AsyncMock(side_effect=write_malformed), + temp_dir=temp_dir, + ) + + result = await service.migrate(recipe_path) + + assert "error" in result + assert recipe_path.read_bytes() == original + assert not recipe_path.with_suffix(".yaml.bak").exists() + from autoskillit.migration.store import FailureStore, default_store_path + + failure_store = FailureStore(default_store_path(tmp_path, temp_dir=temp_dir)) + assert failure_store.has_failure("myrecipe") + @pytest.mark.anyio async def test_contract_regeneration_appends_to_contracts_regenerated( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch diff --git a/tests/recipe/test_api_orchestration.py b/tests/recipe/test_api_orchestration.py index 11a65b690d..e07e728f2b 100644 --- a/tests/recipe/test_api_orchestration.py +++ b/tests/recipe/test_api_orchestration.py @@ -25,30 +25,15 @@ _ValidationResult, ) from autoskillit.recipe.schema import RecipeInfo, RecipeSource +from tests._helpers import MINIMAL_VALID_RECIPE_YAML pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] # ---- Test fixture data ---- -MINIMAL_VALID_YAML = """\ -name: minimal-recipe -description: minimal test recipe -kitchen_rules: - - Use the included stop step sentinel protocol. -ingredients: - task: - description: A task ingredient - required: true - default: my-default-task-value -steps: - done: - action: stop - message: Task completed successfully with full sentinel L3 block emitted for completion. -""" - -def _setup_recipe(tmp_path: Path, name: str, content: str = MINIMAL_VALID_YAML) -> Path: +def _setup_recipe(tmp_path: Path, name: str, content: str = MINIMAL_VALID_RECIPE_YAML) -> Path: recipes_dir = tmp_path / ".autoskillit" / "recipes" recipes_dir.mkdir(parents=True, exist_ok=True) recipe_path = recipes_dir / f"{name}.yaml" @@ -92,6 +77,84 @@ def _make_inputs(name: str = "minimal-recipe", tmp_path: Path | None = None) -> ) +def test_validate_from_path_reports_binding_failures_with_other_findings( + tmp_path: Path, +) -> None: + from autoskillit.core import is_recipe_path_validation_report + from autoskillit.recipe import validate_from_path + + recipe_path = tmp_path / "binding-errors.yaml" + recipe_path.write_text( + """\ +name: binding-errors +description: Binding projection fixture. +steps: + bad: + tool: no_such_tool + with: + bogus: value + on_success: done + done: + action: stop + message: Done. +""" + ) + + result = validate_from_path(recipe_path) + + assert is_recipe_path_validation_report(result) + assert result["errors"] + assert result["findings"] + assert result["binding_projection_failures"] == [ + { + "code": "unknown_tool", + "step_name": "bad", + "name": "no_such_tool", + "message": "Step 'bad' references unknown tool 'no_such_tool'", + } + ] + + +def test_validate_from_path_uses_original_path_for_contract_only(tmp_path: Path) -> None: + from autoskillit.core import is_recipe_path_validation_report + from autoskillit.recipe import validate_from_path + + original_dir = tmp_path / "project" / ".autoskillit" / "recipes" + candidate_dir = tmp_path / "migration-output" + (original_dir / "contracts").mkdir(parents=True) + (candidate_dir / "contracts").mkdir(parents=True) + original_path = original_dir / "source.yaml" + candidate_path = candidate_dir / "candidate.yaml" + original_path.write_text(MINIMAL_VALID_RECIPE_YAML) + candidate_path.write_text( + MINIMAL_VALID_RECIPE_YAML.replace( + " done:\n action: stop", + " bad:\n" + " tool: no_such_tool\n" + " with:\n" + " bogus: value\n" + " on_success: done\n" + " done:\n" + " action: stop", + ) + ) + (original_dir / "contracts" / "source.yaml").write_text( + "dataflow:\n - step: original-card\n available: []\n required: [source_only]\n" + ) + (candidate_dir / "contracts" / "candidate.yaml").write_text( + "dataflow:\n - step: candidate-card\n available: []\n required: [temp_only]\n" + ) + + result = validate_from_path(candidate_path, contract_source_path=original_path) + default_result = validate_from_path(candidate_path) + + assert is_recipe_path_validation_report(result) + assert is_recipe_path_validation_report(default_result) + assert any(finding.get("rule") == "unknown-tool" for finding in result["findings"]) + assert [finding["step"] for finding in result["contracts"]] == ["original-card"] + assert [finding["step"] for finding in default_result["contracts"]] == ["candidate-card"] + + # ---- Bundle field-list tests (catches silent regression if a field is dropped) ---- @@ -246,7 +309,7 @@ def test_resolve_recipe_match_returns_recipes_dir_for_both_sources(tmp_path: Pat version=None, recipe_version=None, content_hash="", - content=MINIMAL_VALID_YAML, + content=MINIMAL_VALID_RECIPE_YAML, source=RecipeSource.BUILTIN, path=Path("/nonexistent/builtin.yaml"), ) diff --git a/tests/recipe/test_repository.py b/tests/recipe/test_repository.py index 2661dc1fa4..56e8eb7204 100644 --- a/tests/recipe/test_repository.py +++ b/tests/recipe/test_repository.py @@ -150,6 +150,7 @@ def test_validate_from_path_delegates_to_api(tmp_path: Path) -> None: "quality": {}, "findings": [], "contracts": [], + "binding_projection_failures": [], } mock_api = MagicMock(return_value=expected) diff --git a/tests/recipe/test_validation_result_types.py b/tests/recipe/test_validation_result_types.py index 3480b7d3ea..15ebca8b2e 100644 --- a/tests/recipe/test_validation_result_types.py +++ b/tests/recipe/test_validation_result_types.py @@ -44,6 +44,14 @@ def _completed_report(valid: bool = True) -> dict[str, object]: "quality": {"reachable": 1}, "findings": [{"rule": "example", "message": "finding"}], "contracts": [{"step": 42}], + "binding_projection_failures": [ + { + "code": "unknown_tool", + "step_name": "example-step", + "name": "example-tool", + "message": "unknown tool", + } + ], } @@ -64,7 +72,14 @@ def test_recipe_path_validation_typed_dicts_have_exact_required_keys_and_annotat } assert RecipePathValidationReport.__required_keys__ == frozenset( - {"valid", "errors", "quality", "findings", "contracts"} + { + "valid", + "errors", + "quality", + "findings", + "contracts", + "binding_projection_failures", + } ) assert get_type_hints(RecipePathValidationReport) == { "valid": bool, @@ -72,6 +87,7 @@ def test_recipe_path_validation_typed_dicts_have_exact_required_keys_and_annotat "quality": dict[str, object], "findings": list[dict[str, str]], "contracts": list[dict[str, Any]], + "binding_projection_failures": list[dict[str, str]], } @@ -162,11 +178,14 @@ def test_recipe_path_validation_report_guard_rejects_wrong_top_level_shape( ("quality", {42: "not a string key"}), ("findings", "not a list"), ("contracts", "not a list"), + ("binding_projection_failures", "not a list"), ("findings", [{"rule": 42}]), ("contracts", [{42: "not a string key"}]), + ("binding_projection_failures", [{"code": 42}]), ("quality", _MappingOnly({"reachable": 1})), ("findings", [_MappingOnly({"rule": "example"})]), ("contracts", [_MappingOnly({"step": 42})]), + ("binding_projection_failures", [_MappingOnly({"code": "unknown_tool"})]), ], ) def test_recipe_path_validation_report_guard_rejects_malformed_fields( @@ -252,6 +271,7 @@ def examine( assert_type(result["quality"], dict[str, object]) assert_type(result["findings"], list[dict[str, str]]) assert_type(result["contracts"], list[dict[str, Any]]) + assert_type(result["binding_projection_failures"], list[dict[str, str]]) assert_type(early_error["valid"], Literal[False]) """ snippet_path = tmp_path / "recipe_path_validation_result_narrowing.py" diff --git a/tests/server/test_consumer_tool_callability.py b/tests/server/test_consumer_tool_callability.py index 0d4b7867a1..09944b9671 100644 --- a/tests/server/test_consumer_tool_callability.py +++ b/tests/server/test_consumer_tool_callability.py @@ -269,7 +269,12 @@ def _skill_tool_operations(content: str) -> list[tuple[str, int]]: for index, logical in enumerate(logicals): text = _logical_text(logical) for tool in sorted(tool_names): - if _has_tool_operation(text, tool): + validate_imperative = re.search( + rf"\bvalidate\b[^\n]{{0,100}}\b{re.escape(tool)}\b", + text, + re.IGNORECASE, + ) + if _has_tool_operation(text, tool) or validate_imperative: evidence = _evidence(tool, logical) if evidence.is_genuine: operations.append((tool, evidence.source_span[0])) @@ -435,6 +440,11 @@ def test_tool_operation_detection_recognizes_pseudo_call() -> None: assert _skill_tool_operations(content) == [("run_python", 5)] +def test_tool_operation_detection_recognizes_validate_imperative() -> None: + content = "# Workflow\n\n1. Validate via `validate_recipe` before declaring success\n" + assert _skill_tool_operations(content) == [("validate_recipe", 3)] + + def test_tool_operation_detection_rejects_unregistered_parameter() -> None: content = "1. Write config:\n\n```config.yaml\ntest_check:\n command: task test\n```\n" assert _skill_tool_operations(content) == [] diff --git a/tests/server/test_factory_context_construction.py b/tests/server/test_factory_context_construction.py index f568f836e1..eab2d2438a 100644 --- a/tests/server/test_factory_context_construction.py +++ b/tests/server/test_factory_context_construction.py @@ -8,8 +8,8 @@ import pytest -from autoskillit.config import AutomationConfig -from autoskillit.core import AuditAdmissionStoreAuthority +from autoskillit.config import AutomationConfig, WorkspaceConfig +from autoskillit.core import AuditAdmissionStoreAuthority, resolve_temp_dir, temp_dir_display_str from autoskillit.core.types import SkillResult from autoskillit.execution.headless import DefaultHeadlessExecutor from autoskillit.execution.runtime.db import DefaultDatabaseReader @@ -225,6 +225,20 @@ def test_make_context_service_fields_are_typed_instances(tmp_path): assert type(ctx.github_review_poster).__name__ == "DefaultGitHubReviewPoster" +def test_make_context_wires_configured_migration_temp_root(tmp_path: Path) -> None: + config = AutomationConfig(workspace=WorkspaceConfig(temp_dir="custom-temp")) + + ctx = make_context(config, runner=_runner(), project_dir=tmp_path) + + assert isinstance(ctx.migrations, DefaultMigrationService) + assert ctx.migrations._temp_dir_override == resolve_temp_dir( + tmp_path, config.workspace.temp_dir + ) + adapter = ctx.migrations._engine.get_adapter("recipe") + assert adapter is not None + assert adapter._temp_dir_display == temp_dir_display_str(config.workspace.temp_dir) + + def test_make_context_creates_isolated_context_admission_ledgers(tmp_path) -> None: first = make_context(AutomationConfig(), runner=_runner(), project_dir=tmp_path) second = make_context(AutomationConfig(), runner=_runner(), project_dir=tmp_path) diff --git a/tests/server/test_service_wrappers.py b/tests/server/test_service_wrappers.py index e4295acc2d..478d613075 100644 --- a/tests/server/test_service_wrappers.py +++ b/tests/server/test_service_wrappers.py @@ -17,6 +17,7 @@ import autoskillit.recipe as recipe import autoskillit.recipe.io as recipe_io import autoskillit.recipe.repository as recipe_repository +from tests._helpers import MINIMAL_VALID_RECIPE_YAML from tests.recipe._testing import count_discovery_calls, isolate_recipe_discovery_cache pytestmark = [pytest.mark.layer("server"), pytest.mark.small] @@ -285,13 +286,8 @@ async def test_migrate_contracts_regenerated_included_in_migrated_result( # SW- (fake_mig_dir / "0.0.0-migration.yaml").write_text(migration_yaml) monkeypatch.setattr(ml, "_migrations_dir", lambda: fake_mig_dir) - # Create temp output file so RecipeMigrationAdapter finds migrated content temp_mig_dir = tmp_path / ".autoskillit" / "temp" / "migrations" - temp_mig_dir.mkdir(parents=True) - migrated_content = ( - f"name: test-recipe\nsteps: []\nautoskillit_version: '{installed_ver}'\n" - ) - (temp_mig_dir / "test-recipe.yaml").write_text(migrated_content) + migrated_content = f"autoskillit_version: '{installed_ver}'\n" + MINIMAL_VALID_RECIPE_YAML success_result = SkillResult( success=True, @@ -304,8 +300,14 @@ async def test_migrate_contracts_regenerated_included_in_migrated_result( # SW- retry_reason=RetryReason.NONE, stderr="", ) + + async def write_output(**_kwargs: object): + temp_mig_dir.mkdir(parents=True, exist_ok=True) + (temp_mig_dir / "test-recipe.yaml").write_text(migrated_content) + return success_result + service = DefaultMigrationService( - default_migration_engine(), run_headless=AsyncMock(return_value=success_result) + default_migration_engine(), run_headless=AsyncMock(side_effect=write_output) ) monkeypatch.setattr(recipe, "load_recipe_card", lambda *a, **kw: None) diff --git a/tests/server/test_tools_migrate_recipe.py b/tests/server/test_tools_migrate_recipe.py index 9e391a5f37..c7d4c2e2d1 100644 --- a/tests/server/test_tools_migrate_recipe.py +++ b/tests/server/test_tools_migrate_recipe.py @@ -13,11 +13,31 @@ from autoskillit.core.types import RetryReason from autoskillit.pipeline.gate import GATED_TOOLS, UNGATED_TOOLS, DefaultGateState from autoskillit.server.tools.tools_recipe import migrate_recipe +from tests._helpers import MINIMAL_VALID_RECIPE_YAML from tests.server._helpers import _MINIMAL_SCRIPT_YAML pytestmark = [pytest.mark.layer("server"), pytest.mark.small] +def _successful_headless_writing(path, content: str) -> AsyncMock: + async def write_output(*_args, **_kwargs): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + return SkillResult( + success=True, + result="ok", + session_id="", + subtype="success", + is_error=False, + exit_code=0, + needs_retry=False, + retry_reason=RetryReason.NONE, + stderr="", + ) + + return AsyncMock(side_effect=write_output) + + class TestMigrateRecipe: """P4: migrate_recipe is a gated tool that runs migration engine and regenerates cards.""" @@ -65,7 +85,7 @@ def _setup_migration_env( temp_mig_dir = tmp_path / ".autoskillit" / "temp" / "migrations" temp_mig_dir.mkdir(parents=True) - migrated_content = _MINIMAL_SCRIPT_YAML + f"autoskillit_version: '{installed_ver}'\n" + migrated_content = f"autoskillit_version: '{installed_ver}'\n" + MINIMAL_VALID_RECIPE_YAML return { "recipe_path": recipe_path, "temp_mig_dir": temp_mig_dir, @@ -116,20 +136,8 @@ async def test_migrate_recipe_up_to_date(self, tmp_path, monkeypatch): # SRV-UP async def test_auto_migrates_outdated_recipe(self, tmp_path, monkeypatch, tool_ctx): """LR1: When recipe version < installed, _run_headless_core is called once.""" ctx = self._setup_migration_env(tmp_path, monkeypatch, tool_ctx) - (ctx["temp_mig_dir"] / "test-script.yaml").write_text(ctx["migrated_content"]) - - mock_headless = AsyncMock( - return_value=SkillResult( - success=True, - result="ok", - session_id="", - subtype="success", - is_error=False, - exit_code=0, - needs_retry=False, - retry_reason=RetryReason.NONE, - stderr="", - ) + mock_headless = _successful_headless_writing( + ctx["temp_mig_dir"] / "test-script.yaml", ctx["migrated_content"] ) with ( patch.object(_patch_execution_headless, "run_headless_core", mock_headless), @@ -154,7 +162,6 @@ async def test_clears_failure_record_after_successful_migration( recipe_contracts = importlib.import_module("autoskillit.recipe.contracts") ctx = self._setup_migration_env(tmp_path, monkeypatch, tool_ctx) - (ctx["temp_mig_dir"] / "test-script.yaml").write_text(ctx["migrated_content"]) store = FailureStore(default_store_path(tmp_path)) store.record( @@ -166,18 +173,8 @@ async def test_clears_failure_record_after_successful_migration( ) assert store.has_failure("test-script") - mock_headless = AsyncMock( - return_value=SkillResult( - success=True, - result="ok", - session_id="", - subtype="success", - is_error=False, - exit_code=0, - needs_retry=False, - retry_reason=RetryReason.NONE, - stderr="", - ) + mock_headless = _successful_headless_writing( + ctx["temp_mig_dir"] / "test-script.yaml", ctx["migrated_content"] ) with ( patch.object(_patch_execution_headless, "run_headless_core", mock_headless), @@ -307,20 +304,8 @@ async def test_migrate_recipe_regenerates_stale_contract( ): """migrate_recipe with version migration also regenerates stale contracts.""" ctx = self._setup_migration_env(tmp_path, monkeypatch, tool_ctx) - (ctx["temp_mig_dir"] / "test-script.yaml").write_text(ctx["migrated_content"]) - - mock_headless = AsyncMock( - return_value=SkillResult( - success=True, - result="ok", - session_id="", - subtype="success", - is_error=False, - exit_code=0, - needs_retry=False, - retry_reason=RetryReason.NONE, - stderr="", - ) + mock_headless = _successful_headless_writing( + ctx["temp_mig_dir"] / "test-script.yaml", ctx["migrated_content"] ) with ( patch.object(_patch_execution_headless, "run_headless_core", mock_headless), From acce1243124eb74be5304a226e73d146d2b87f17 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 20:25:48 -0700 Subject: [PATCH 04/15] test: align migration fixtures with host validation --- tests/arch/test_effective_graph_provenance.py | 5 +++++ tests/migration/test_engine_recipe.py | 12 ++++++++++-- tests/recipe/test_api_orchestration.py | 2 +- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/tests/arch/test_effective_graph_provenance.py b/tests/arch/test_effective_graph_provenance.py index a1c856c4d6..8d6156e048 100644 --- a/tests/arch/test_effective_graph_provenance.py +++ b/tests/arch/test_effective_graph_provenance.py @@ -60,6 +60,11 @@ "_load_campaign_recipe", "load_recipe(match.path)", ), # Validate the selected campaign before dispatching its fleet. + ( + "migration/adapters_recipe.py", + "_validation_errors", + "load_recipe(path)", + ), # Validate original and candidate recipe structure before migration write-back. } ) diff --git a/tests/migration/test_engine_recipe.py b/tests/migration/test_engine_recipe.py index 4bd47133cb..d6dfc478bb 100644 --- a/tests/migration/test_engine_recipe.py +++ b/tests/migration/test_engine_recipe.py @@ -250,7 +250,12 @@ async def test_failed_headless_session_is_reported( ("candidate", "expected"), [ ( - _stamped(MINIMAL_VALID_RECIPE_YAML.replace("kitchen_rules:", "rules:")), + _stamped( + MINIMAL_VALID_RECIPE_YAML.replace( + "kitchen_rules:\n - Use the included stop step sentinel protocol.", + "kitchen_rules: []", + ) + ), "kitchen_rules", ), ( @@ -333,7 +338,10 @@ async def test_migrate_rejects_new_errors_alongside_baseline_errors( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: _enable_migration(monkeypatch) - baseline = MINIMAL_VALID_RECIPE_YAML.replace("kitchen_rules:", "rules:") + baseline = MINIMAL_VALID_RECIPE_YAML.replace( + "kitchen_rules:\n - Use the included stop step sentinel protocol.", + "kitchen_rules: []", + ) candidate = baseline.replace( " done:\n", " bad:\n" diff --git a/tests/recipe/test_api_orchestration.py b/tests/recipe/test_api_orchestration.py index e07e728f2b..72c3e15aa8 100644 --- a/tests/recipe/test_api_orchestration.py +++ b/tests/recipe/test_api_orchestration.py @@ -110,7 +110,7 @@ def test_validate_from_path_reports_binding_failures_with_other_findings( "code": "unknown_tool", "step_name": "bad", "name": "no_such_tool", - "message": "Step 'bad' references unknown tool 'no_such_tool'", + "message": "tool 'no_such_tool' is not present in the canonical registry", } ] From 9ae21a4be8140a0bf9eca97087a7c167f758b497 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 21:04:36 -0700 Subject: [PATCH 05/15] fix: make declared agent inspection tools headless-callable --- docs/execution/tool-access.md | 2 +- .../constants/_type_constants_registries.py | 2 +- src/autoskillit/server/AGENTS.md | 3 ++- .../server/tools/tools_session_logs.py | 8 +++--- .../test_agent_tool_surface_alignment.py | 8 ++++++ ...test_type_constants_tool_classification.py | 1 + tests/pipeline/test_gate.py | 1 - tests/server/test_session_log_tools.py | 26 ++++++++++++++----- ...st_session_type_visibility_orchestrator.py | 4 +++ 9 files changed, 41 insertions(+), 14 deletions(-) diff --git a/docs/execution/tool-access.md b/docs/execution/tool-access.md index 8776787ead..b62c4a5da1 100644 --- a/docs/execution/tool-access.md +++ b/docs/execution/tool-access.md @@ -196,6 +196,7 @@ tag abbreviations are defined in the glossary above. | `finalize_review_audit` | AS, K, HL, KC | `server/tools/tools_review_audit/_handlers.py` | | `post_pr_review` | AS, HL, GH | `server/tools/tools_pr_ops.py` | | `delegate_evidence_reader` | AS, K, HL, KC | `server/tools/tools_evidence_reader.py` | +| `inspect_session_logs` | AS, K, HL, KC | `server/tools/tools_session_logs.py` | ### AUTHENTICATED EVIDENCE READER @@ -284,7 +285,6 @@ batch, assignment, source artifact, and digest. | `write_telemetry_files` | AS, K, KC, TL | `server/tools/tools_status.py` | | `read_db` | AS, K, KC | `server/tools/tools_status.py` | | `analyze_tool_sequences` | AS, K, KC, TL | `server/tools/tools_status.py` | -| `inspect_session_logs` | AS, K, KC | `server/tools/tools_session_logs.py` | ### KITCHEN — Recipes diff --git a/src/autoskillit/core/types/constants/_type_constants_registries.py b/src/autoskillit/core/types/constants/_type_constants_registries.py index a692acc10a..aec6ce7f17 100644 --- a/src/autoskillit/core/types/constants/_type_constants_registries.py +++ b/src/autoskillit/core/types/constants/_type_constants_registries.py @@ -180,7 +180,6 @@ "submit_exploration_query", "get_exploration_page", "resume_exploration_context", - "inspect_session_logs", "read_authorized_artifact", "get_authorized_artifact_page", } @@ -191,6 +190,7 @@ "delegate_evidence_reader", "fetch_github_issue", "get_issue_title", + "inspect_session_logs", "test_check", "unlock_agent_pack", "commit_files", diff --git a/src/autoskillit/server/AGENTS.md b/src/autoskillit/server/AGENTS.md index b9579607d4..f2a1fad5b9 100644 --- a/src/autoskillit/server/AGENTS.md +++ b/src/autoskillit/server/AGENTS.md @@ -104,4 +104,5 @@ The canonical tool sets are in `core/types/constants/_type_constants_registries. sentence that must exist in the skill text); known defects go in `DEFERRED_VIOLATIONS` as `TrackedDeferral`s (issue, dated 180-day time bomb, regression test). Every SESSION skill is checked in both headless contexts unless - it declares `interactive_only: true`. + it declares `interactive_only: true`; MCP tools declared by non-explorer agents must + be in `HEADLESS_TOOLS` (`tests/contracts/test_agent_tool_surface_alignment.py`). diff --git a/src/autoskillit/server/tools/tools_session_logs.py b/src/autoskillit/server/tools/tools_session_logs.py index 2c91457c8f..4f6809e9c7 100644 --- a/src/autoskillit/server/tools/tools_session_logs.py +++ b/src/autoskillit/server/tools/tools_session_logs.py @@ -23,7 +23,6 @@ ) from autoskillit.execution import read_session_index_rows, resolve_log_dir from autoskillit.server import _get_ctx, mcp -from autoskillit.server.lifecycle._guards import _require_enabled from autoskillit.server.lifecycle._session_scope import SCOPE_ANY, session_scoped from autoskillit.server.tools._cancellation_shield import _cancellation_shield @@ -544,11 +543,14 @@ def _validate_arguments( @mcp.tool( - tags={"autoskillit", "kitchen", "kitchen-core"}, + tags={"autoskillit", "kitchen", "kitchen-core", "headless"}, annotations={"readOnlyHint": True}, ) @session_scoped(SCOPE_ANY) @_cancellation_shield() +# No _require_enabled() — headless skill sessions need inspect_session_logs without +# opening the kitchen (session-log-reader). The kitchen/headless tags govern visibility; +# the bounded read-only surface is narrower than the native Read/Bash already available. async def inspect_session_logs( operation: str, session_ids: list[str] | None = None, @@ -563,8 +565,6 @@ async def inspect_session_logs( Never raises. """ try: - if (gate := _require_enabled()) is not None: - return gate _validate_arguments( operation, session_ids, diff --git a/tests/contracts/test_agent_tool_surface_alignment.py b/tests/contracts/test_agent_tool_surface_alignment.py index b36470c07e..7d7ff444bf 100644 --- a/tests/contracts/test_agent_tool_surface_alignment.py +++ b/tests/contracts/test_agent_tool_surface_alignment.py @@ -11,6 +11,7 @@ CODEX_MODEL_ALIASES, DIRECT_PREFIX, EXPLORATION_TOOLS, + HEADLESS_TOOLS, PLUGIN_PREFIX, SkillExecutionRole, SkillSource, @@ -42,7 +43,14 @@ def test_authored_agent_mcp_tools_derive_from_core_authority() -> None: is ToolInitializationOperation.INSPECTION ) if definition.name in BUNDLED_EXPLORER_ROLES: + # Claude-headless explorer reachability is tracked separately in #5322. assert short in EXPLORATION_TOOLS + else: + assert short in HEADLESS_TOOLS, ( + f"Agent {definition.name!r} declares {short!r}, which is not a " + "headless tool; spawned agents share the spawning headless skill " + "session's catalog" + ) def test_projected_artifact_agent_tools_carry_plugin_namespace(tmp_path: Path) -> None: diff --git a/tests/core/test_type_constants_tool_classification.py b/tests/core/test_type_constants_tool_classification.py index 98d9e22495..cfb7ae17d6 100644 --- a/tests/core/test_type_constants_tool_classification.py +++ b/tests/core/test_type_constants_tool_classification.py @@ -46,6 +46,7 @@ def test_headless_tools_contains_expected_names() -> None: "delegate_evidence_reader", "fetch_github_issue", "get_issue_title", + "inspect_session_logs", } diff --git a/tests/pipeline/test_gate.py b/tests/pipeline/test_gate.py index a86c316a84..4a45bda9ec 100644 --- a/tests/pipeline/test_gate.py +++ b/tests/pipeline/test_gate.py @@ -66,7 +66,6 @@ def test_gated_tools_contains_expected_names(): "submit_exploration_query", "get_exploration_page", "resume_exploration_context", - "inspect_session_logs", "read_authorized_artifact", "get_authorized_artifact_page", } diff --git a/tests/server/test_session_log_tools.py b/tests/server/test_session_log_tools.py index 7bc9611d5d..bf6e70b39b 100644 --- a/tests/server/test_session_log_tools.py +++ b/tests/server/test_session_log_tools.py @@ -23,12 +23,27 @@ async def _inspect(**kwargs: Any) -> dict[str, Any]: @pytest.mark.asyncio -async def test_inspection_requires_an_open_kitchen(tool_ctx) -> None: - result = json.loads(await session_logs.inspect_session_logs(operation="index")) +async def test_inspection_does_not_require_an_open_kitchen(tool_ctx, retained_logs) -> None: + result = await _inspect(operation="index", session_ids=["session-one"]) - assert result["success"] is False - assert result["subtype"] == "gate_error" - assert "open_kitchen" in result["result"] + assert tool_ctx.gate.enabled is False + assert result["status"] == "answered" + assert result["sessions"] == [ + { + "session_id": "session-one", + "dir_name": "session-one", + "backend": "claude-code", + "kitchen_id": "kitchen-one", + "step_name": "implement", + "handles": ["summary", "anomalies", "audit", "turn_usage", "transcript"], + "retry": { + "success": False, + "subtype": "missing_completion_marker", + "needs_retry": True, + "retry_reason": "early_stop", + }, + } + ] @pytest.fixture @@ -73,7 +88,6 @@ def retained_logs(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> dict[str, backend=None, ) monkeypatch.setattr(session_logs, "_get_ctx", lambda: ctx) - monkeypatch.setattr(session_logs, "_require_enabled", lambda: None) monkeypatch.setattr(session_logs, "claude_code_log_path", lambda cwd, session_id: transcript) monkeypatch.setattr(session_logs, "_CONTINUATION_KEY", b"k" * 32) monkeypatch.setattr(session_logs, "_CONTINUATION_CLOCK", lambda: 1_000.0) diff --git a/tests/server/test_session_type_visibility_orchestrator.py b/tests/server/test_session_type_visibility_orchestrator.py index 344447103f..7fd2e54003 100644 --- a/tests/server/test_session_type_visibility_orchestrator.py +++ b/tests/server/test_session_type_visibility_orchestrator.py @@ -99,6 +99,10 @@ async def test_skill_headless_without_auto_gate_only_headless(monkeypatch): tools = list(await mcp.list_tools()) tool_names = {t.name for t in tools} assert "test_check" in tool_names, "test_check should be visible for skill+headless" + assert HEADLESS_TOOLS <= tool_names, ( + f"Headless tools should be visible for skill+headless; missing " + f"{sorted(HEADLESS_TOOLS - tool_names)}" + ) for name in GATED_TOOLS - HEADLESS_TOOLS: assert name not in tool_names, f"{name} (kitchen) should be hidden for skill+headless" From f47aedb507276ef2619d1109658459e2493c805e Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 22:54:18 -0700 Subject: [PATCH 06/15] fix: address migration audit coverage and skill wording --- .../skills_extended/migrate-recipes/SKILL.md | 12 ++--- tests/migration/test_engine_recipe.py | 54 +++++++++++++++---- 2 files changed, 50 insertions(+), 16 deletions(-) diff --git a/src/autoskillit/skills_extended/migrate-recipes/SKILL.md b/src/autoskillit/skills_extended/migrate-recipes/SKILL.md index 118acbd73a..61a825c97b 100644 --- a/src/autoskillit/skills_extended/migrate-recipes/SKILL.md +++ b/src/autoskillit/skills_extended/migrate-recipes/SKILL.md @@ -3,8 +3,8 @@ name: migrate-recipes write_paths: - '{{AUTOSKILLIT_TEMP}}/migrations/' uses_capabilities: [] -description: Apply versioned migration notes when the migration service launches this skill headlessly with the recipe path, - content, notes, and target version. +description: Apply versioned migration notes when the migration service launches this skill headlessly with script_path, + script_content, migration_notes and target_version. hooks: PreToolUse: - matcher: '*' @@ -20,7 +20,7 @@ Apply versioned migration notes to an AutoSkillit recipe. ## When to Use -- Launched headlessly by the migration service when `migrate_recipe` migrates a stale recipe +- Launched headlessly by the migration service when the `migrate_recipe` tool migrates a stale recipe ## Arguments @@ -40,8 +40,7 @@ The orchestrator provides all context in the prompt: **ALWAYS:** - Save migrated scripts to {{AUTOSKILLIT_TEMP}}/migrations/{script_name}.yaml for review -- Set `autoskillit_version` to `target_version` -- Treat migration-note text that says to run a validator as satisfied by host-side validation +- Always set `autoskillit_version` to `target_version`. The host validates the saved file against the original recipe and rejects errors your changes introduce; migration-note text that says to run a validator is satisfied by the host. - Preserve all existing script fields not targeted by migration changes - Output a human-readable diff summary of changes applied @@ -59,5 +58,4 @@ The orchestrator provides all context in the prompt: ## Error Handling -- If no patterns are detected, stamp the version and save the recipe -- If a change cannot be applied, save nothing and report why +- If no patterns are detected, stamp the version and save. If a change cannot be applied, save nothing and report why. diff --git a/tests/migration/test_engine_recipe.py b/tests/migration/test_engine_recipe.py index d6dfc478bb..7ebdd8fab0 100644 --- a/tests/migration/test_engine_recipe.py +++ b/tests/migration/test_engine_recipe.py @@ -334,23 +334,59 @@ async def test_migrate_tolerates_pre_existing_errors( assert result.success @pytest.mark.anyio + @pytest.mark.parametrize( + ("baseline_error", "introduced_error", "expected"), + [ + ("structural", "semantic", "unknown skill 'new-project-local'"), + ( + "semantic", + "binding", + "tool 'no_such_tool' is not present in the canonical registry", + ), + ], + ids=["structural-then-semantic", "semantic-then-binding"], + ) async def test_migrate_rejects_new_errors_alongside_baseline_errors( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + baseline_error: str, + introduced_error: str, + expected: str, ) -> None: _enable_migration(monkeypatch) - baseline = MINIMAL_VALID_RECIPE_YAML.replace( - "kitchen_rules:\n - Use the included stop step sentinel protocol.", - "kitchen_rules: []", - ) - candidate = baseline.replace( - " done:\n", + baseline = MINIMAL_VALID_RECIPE_YAML + if baseline_error == "structural": + baseline = baseline.replace( + "kitchen_rules:\n - Use the included stop step sentinel protocol.", + "kitchen_rules: []", + ) + else: + baseline = baseline.replace( + " done:\n", + " local:\n" + " tool: run_skill\n" + " with:\n" + " skill_command: /autoskillit:project-local\n" + " cwd: .\n" + " on_success: done\n" + " done:\n", + ) + introduced_step = ( " bad:\n" + " tool: run_skill\n" + " with:\n" + " skill_command: /autoskillit:new-project-local\n" + " cwd: .\n" + " on_success: done\n" + if introduced_error == "semantic" + else " bad:\n" " tool: no_such_tool\n" " with:\n" " bogus: value\n" " on_success: done\n" - " done:\n", ) + candidate = baseline.replace(" done:\n", introduced_step + " done:\n") adapter = RecipeMigrationAdapter() file = _migration_file(tmp_path, _stamped(baseline, "0.0.1")) temp_dir = tmp_path / "temp" @@ -359,7 +395,7 @@ async def test_migrate_rejects_new_errors_alongside_baseline_errors( result = await adapter.migrate(file, run_headless=runner, temp_dir=temp_dir) assert not result.success - assert "no_such_tool" in (result.error or "") + assert expected in (result.error or "") @pytest.mark.anyio async def test_migrate_keeps_distinct_findings_from_same_rule_and_step( From 7482b9c9642c17a0f40bf138502fa69b1063c80a Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 26 Oct 2026 09:22:44 -0700 Subject: [PATCH 07/15] fix(review): validate migrated recipes in a single pass --- src/autoskillit/migration/adapters_recipe.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/autoskillit/migration/adapters_recipe.py b/src/autoskillit/migration/adapters_recipe.py index 3b5305e556..328a0c295f 100644 --- a/src/autoskillit/migration/adapters_recipe.py +++ b/src/autoskillit/migration/adapters_recipe.py @@ -200,20 +200,9 @@ def _validation_errors( *, contract_source_path: Path | None = None, ) -> dict[str, str]: - from autoskillit.recipe import load_recipe, validate_from_path, validate_recipe_structure + from autoskillit.recipe import validate_from_path errors: dict[str, str] = {} - try: - recipe = load_recipe(path) - errors.update( - (f"structural:{message}", message) for message in validate_recipe_structure(recipe) - ) - except Exception as exc: - logger.warning( - "Recipe file validation failed", path=str(path), error=str(exc), exc_info=True - ) - errors["exception"] = f"{type(exc).__name__}: {exc}" - try: result = validate_from_path( path, From b5c8aa04bded28c0f18c20f574afe7a4f43fa0c6 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 26 Oct 2026 09:38:09 -0700 Subject: [PATCH 08/15] fix(review): use recipe terminology in migration prompts --- src/autoskillit/migration/adapters_recipe.py | 4 +-- .../skills_extended/migrate-recipes/SKILL.md | 25 ++++++++++--------- tests/migration/test_engine_recipe.py | 4 +-- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/autoskillit/migration/adapters_recipe.py b/src/autoskillit/migration/adapters_recipe.py index 328a0c295f..2b0dcbf19d 100644 --- a/src/autoskillit/migration/adapters_recipe.py +++ b/src/autoskillit/migration/adapters_recipe.py @@ -113,8 +113,8 @@ async def migrate( content = file.path.read_text() skill_command = ( f"/autoskillit:migrate-recipes" - f" script_path={file.path}" - f" script_content={json.dumps(content)}" + f" recipe_path={file.path}" + f" recipe_content={json.dumps(content)}" f" migration_notes={json.dumps(notes_yaml)}" f" target_version={target_version}" ) diff --git a/src/autoskillit/skills_extended/migrate-recipes/SKILL.md b/src/autoskillit/skills_extended/migrate-recipes/SKILL.md index 61a825c97b..4bab46fbc8 100644 --- a/src/autoskillit/skills_extended/migrate-recipes/SKILL.md +++ b/src/autoskillit/skills_extended/migrate-recipes/SKILL.md @@ -3,8 +3,8 @@ name: migrate-recipes write_paths: - '{{AUTOSKILLIT_TEMP}}/migrations/' uses_capabilities: [] -description: Apply versioned migration notes when the migration service launches this skill headlessly with script_path, - script_content, migration_notes and target_version. +description: Apply versioned migration notes when the migration service launches this skill headlessly with recipe_path, + recipe_content, migration_notes and target_version. hooks: PreToolUse: - matcher: '*' @@ -14,7 +14,7 @@ hooks: once: true --- -# Migrate Scripts Skill +# Migrate Recipes Skill Apply versioned migration notes to an AutoSkillit recipe. @@ -25,35 +25,36 @@ Apply versioned migration notes to an AutoSkillit recipe. ## Arguments The orchestrator provides all context in the prompt: -- `script_path`: Absolute path to the script file -- `script_name`: The file stem of `script_path` (the filename without `.yaml`) -- `script_content`: Current raw YAML of the script +- `recipe_path`: Absolute path to the recipe file +- `recipe_content`: Current raw YAML of the recipe - `migration_notes`: YAML block of all applicable migration notes - `target_version`: Version to stamp after successful migration +Derive `recipe_name` from the file stem of `recipe_path` (the filename without `.yaml`). + ## Critical Constraints **NEVER:** - Fabricate, invent, or embellish information not supported by the available evidence or code. -- Modify the original script file directly -- Apply changes without checking if the pattern exists in the script +- Modify the original recipe file directly +- Apply changes without checking if the pattern exists in the recipe **ALWAYS:** -- Save migrated scripts to {{AUTOSKILLIT_TEMP}}/migrations/{script_name}.yaml for review +- Save migrated recipes to {{AUTOSKILLIT_TEMP}}/migrations/{recipe_name}.yaml for review - Always set `autoskillit_version` to `target_version`. The host validates the saved file against the original recipe and rejects errors your changes introduce; migration-note text that says to run a validator is satisfied by the host. -- Preserve all existing script fields not targeted by migration changes +- Preserve all existing recipe fields not targeted by migration changes - Output a human-readable diff summary of changes applied ## Workflow 1. Parse the migration notes to understand what changes are needed -2. For each change, check if the script contains the outdated pattern described in `detect`: +2. For each change, check if the recipe contains the outdated pattern described in `detect`: - `tool`: Match steps with this tool value - `skill_pattern`: Match steps whose `skill_command` in `with:` contains this substring - `missing_field`: The field that should be added if absent 3. Apply each detected change's `instruction` directly to the YAML, guided by its `example_before` and `example_after` 4. Ensure `autoskillit_version` is set to the `target_version` -5. Save the migrated script to `{{AUTOSKILLIT_TEMP}}/migrations/{script_name}.yaml` +5. Save the migrated recipe to `{{AUTOSKILLIT_TEMP}}/migrations/{recipe_name}.yaml` 6. Output a summary of changes applied ## Error Handling diff --git a/tests/migration/test_engine_recipe.py b/tests/migration/test_engine_recipe.py index 7ebdd8fab0..32cfeb18e7 100644 --- a/tests/migration/test_engine_recipe.py +++ b/tests/migration/test_engine_recipe.py @@ -156,8 +156,8 @@ async def test_recipe_adapter_build_skill_command( call_kwargs = mock_headless.call_args.kwargs assert "skill_command" in call_kwargs skill_cmd: str = call_kwargs["skill_command"] - assert "script_path=" in skill_cmd - assert "script_content=" in skill_cmd + assert f"recipe_path={recipe_path}" in skill_cmd + assert "recipe_content=" in skill_cmd assert "migration_notes=" in skill_cmd assert "target_version=" in skill_cmd From c46789a6d8c419b778b4bce9223e8d263f499c9b Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 26 Oct 2026 09:39:16 -0700 Subject: [PATCH 09/15] test(review): exercise executable pseudo-call parameter validation --- tests/server/test_consumer_tool_callability.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/server/test_consumer_tool_callability.py b/tests/server/test_consumer_tool_callability.py index 09944b9671..fac6ee7d62 100644 --- a/tests/server/test_consumer_tool_callability.py +++ b/tests/server/test_consumer_tool_callability.py @@ -446,7 +446,7 @@ def test_tool_operation_detection_recognizes_validate_imperative() -> None: def test_tool_operation_detection_rejects_unregistered_parameter() -> None: - content = "1. Write config:\n\n```config.yaml\ntest_check:\n command: task test\n```\n" + content = "# Workflow\n\n1. Run the check:\n\n test_check:\n command: task test\n" assert _skill_tool_operations(content) == [] From 9a94d2dd35b8f2769ea69993b7b8e44132e97fc6 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 26 Oct 2026 09:40:04 -0700 Subject: [PATCH 10/15] fix(review): narrow validation tool-call detection --- tests/server/test_consumer_tool_callability.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/server/test_consumer_tool_callability.py b/tests/server/test_consumer_tool_callability.py index fac6ee7d62..2efbbeac31 100644 --- a/tests/server/test_consumer_tool_callability.py +++ b/tests/server/test_consumer_tool_callability.py @@ -270,7 +270,7 @@ def _skill_tool_operations(content: str) -> list[tuple[str, int]]: text = _logical_text(logical) for tool in sorted(tool_names): validate_imperative = re.search( - rf"\bvalidate\b[^\n]{{0,100}}\b{re.escape(tool)}\b", + rf"\bvalidate\s+(?:via|using|with)\s+`?\b{re.escape(tool)}\b", text, re.IGNORECASE, ) @@ -445,6 +445,11 @@ def test_tool_operation_detection_recognizes_validate_imperative() -> None: assert _skill_tool_operations(content) == [("validate_recipe", 3)] +def test_tool_operation_detection_ignores_unrelated_validate_reference() -> None: + content = "# Workflow\n\n1. Validate the plan before `commit_files`\n" + assert _skill_tool_operations(content) == [] + + def test_tool_operation_detection_rejects_unregistered_parameter() -> None: content = "# Workflow\n\n1. Run the check:\n\n test_check:\n command: task test\n" assert _skill_tool_operations(content) == [] From 2bd49c442aa7125febfef09bac77318f26a67eca Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 26 Oct 2026 09:40:49 -0700 Subject: [PATCH 11/15] test(review): name audit coverage for the surviving rationale rule --- ...stitution_probe.py => test_audit_rationale_contradiction.py} | 2 +- tests/core/test_grab_bag_split_completeness.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename tests/core/{test_audit_substitution_probe.py => test_audit_rationale_contradiction.py} (97%) diff --git a/tests/core/test_audit_substitution_probe.py b/tests/core/test_audit_rationale_contradiction.py similarity index 97% rename from tests/core/test_audit_substitution_probe.py rename to tests/core/test_audit_rationale_contradiction.py index bb82304f89..671f871f85 100644 --- a/tests/core/test_audit_substitution_probe.py +++ b/tests/core/test_audit_rationale_contradiction.py @@ -1,4 +1,4 @@ -"""Regression coverage for audit substitutions of prescribed mechanisms.""" +"""Regression coverage for the audit rationale-contradiction rule.""" from __future__ import annotations diff --git a/tests/core/test_grab_bag_split_completeness.py b/tests/core/test_grab_bag_split_completeness.py index e0e13395c4..db45dbf6e6 100644 --- a/tests/core/test_grab_bag_split_completeness.py +++ b/tests/core/test_grab_bag_split_completeness.py @@ -459,7 +459,7 @@ def test_no_unintended_new_test_files_under_tests_core() -> None: "test_audit_cycle_attacks.py", "test_audit_cycle_authority.py", "test_audit_semantic_codec.py", - "test_audit_substitution_probe.py", + "test_audit_rationale_contradiction.py", "test_backend_capabilities.py", "test_backend_dataclasses.py", "test_backend_event_kind.py", From df3f8dde7860c77ef797cd8fa4e57005412e3315 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 26 Oct 2026 09:41:30 -0700 Subject: [PATCH 12/15] test(review): identify unknown tools in migration validation errors --- tests/migration/test_engine_recipe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/migration/test_engine_recipe.py b/tests/migration/test_engine_recipe.py index 32cfeb18e7..9b81ac3a2e 100644 --- a/tests/migration/test_engine_recipe.py +++ b/tests/migration/test_engine_recipe.py @@ -568,7 +568,7 @@ def test_recipe_adapter_validate_runs_full_validation(self, tmp_path: Path) -> N valid, message = RecipeMigrationAdapter().validate(recipe_path) assert not valid - assert message + assert "no_such_tool" in message @pytest.mark.anyio async def test_adapter_passes_temp_display_to_both_validations( From c5b6d4edc82fffdf33f9078d2e4477673090827d Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 26 Oct 2026 09:42:17 -0700 Subject: [PATCH 13/15] docs(review): qualify recipe validation steps by session mode --- src/autoskillit/skills_extended/write-recipe/SKILL.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/autoskillit/skills_extended/write-recipe/SKILL.md b/src/autoskillit/skills_extended/write-recipe/SKILL.md index 3cb127dec9..6116757c44 100644 --- a/src/autoskillit/skills_extended/write-recipe/SKILL.md +++ b/src/autoskillit/skills_extended/write-recipe/SKILL.md @@ -538,7 +538,9 @@ When called directly as `/autoskillit:write-recipe`: 5. Generate the script in the YAML format above 6. Before saving, call `kitchen_status` to get `package_version` and stamp `autoskillit_version: "{package_version}"` as the second top-level field (after `name`). This is required for the migration system to track script age. 7. Save to `.autoskillit/recipes/{name}.yaml` (create the directory if needed) -8. Call `validate_recipe` with the saved file path. If errors are returned, fix them and re-validate until clean. Review the `quality.warnings` in the response: +8. In an interactive session, call `validate_recipe` with the saved file path. If errors are returned, fix them and re-validate until clean. + In a headless session, return the saved YAML for host validation. + For interactive validation responses, review `quality.warnings`: - `DEAD_OUTPUT`: A `capture:` key is never referenced by any reachable downstream step via `${{ context.X }}`. Either add a `${{ context.X }}` reference in the downstream step's `with:` block, or remove the unused capture. - `IMPLICIT_HANDOFF`: A `run_skill` step has no `capture:` block. Add a `capture:` block to explicitly wire outputs to downstream steps via `${{ context.X }}`, or confirm the skill's output is intentionally unused. - Present the quality summary to the user and fix any warnings that indicate broken wiring. @@ -593,9 +595,9 @@ When the agent is given an existing script's YAML content and a requested change - Save changes to the original file - Save as a new script (prompt for name) - Use temporarily without saving -5. Call `validate_recipe` on the saved path -6. If errors, fix and re-validate until clean -7. Review quality warnings and fix data-flow issues before reporting changes +5. In an interactive session, call `validate_recipe` on the saved path. In a headless session, return the saved YAML for host validation. +6. For interactive validation errors, fix and re-validate until clean +7. For interactive validation responses, review quality warnings and fix data-flow issues before reporting changes 8. The pipeline contract card is generated automatically the next time the recipe is loaded; do not generate it manually. 9. Report the changes made From 8578e6c50d624589286d39522d3d72c44866c71d Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 26 Oct 2026 10:02:45 -0700 Subject: [PATCH 14/15] test(review): retire the removed migration raw-read allowance --- tests/arch/test_effective_graph_provenance.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/arch/test_effective_graph_provenance.py b/tests/arch/test_effective_graph_provenance.py index 8d6156e048..a1c856c4d6 100644 --- a/tests/arch/test_effective_graph_provenance.py +++ b/tests/arch/test_effective_graph_provenance.py @@ -60,11 +60,6 @@ "_load_campaign_recipe", "load_recipe(match.path)", ), # Validate the selected campaign before dispatching its fleet. - ( - "migration/adapters_recipe.py", - "_validation_errors", - "load_recipe(path)", - ), # Validate original and candidate recipe structure before migration write-back. } ) From 9590203b4d2df2d9f248677823da5db589d2a5b4 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 26 Oct 2026 10:03:21 -0700 Subject: [PATCH 15/15] test(review): remove the obsolete recipe validation exemption --- tests/server/test_consumer_tool_callability.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/server/test_consumer_tool_callability.py b/tests/server/test_consumer_tool_callability.py index 2efbbeac31..cdbbecce3c 100644 --- a/tests/server/test_consumer_tool_callability.py +++ b/tests/server/test_consumer_tool_callability.py @@ -173,15 +173,6 @@ def __post_init__(self) -> None: rationale="the document-wide mode rule skips kitchen tools in headless sessions", anchor=_WRITE_RECIPE_HEADLESS_GUARD, ), - ( - "skill:write-recipe", - "validate_recipe", - CatalogContext.HEADLESS_SKILL, - ): ConsumerExemption( - "guard", - rationale="the document-wide mode rule skips kitchen tools in headless sessions", - anchor=_WRITE_RECIPE_HEADLESS_GUARD, - ), } DEFERRED_VIOLATIONS: dict[ExemptionKey, TrackedDeferral] = {