From 79b53637d634c6bf407d963b219094b45e27cb11 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sat, 24 Oct 2026 16:02:37 -0700 Subject: [PATCH 01/17] fix: reject binding failures throughout recipe validation --- .autoskillit/recipes/eval/agent-eval.yaml | 1 + .autoskillit/recipes/eval/skill-eval.yaml | 3 +- docs/recipes/authoring.md | 2 +- docs/recipes/overview.md | 4 + .../cli/prompts/_prompts_kitchen.py | 1 + src/autoskillit/core/tool_registry.py | 22 +-- .../migrations/0.0.0-to-0.10.1293.yaml | 28 +++ src/autoskillit/recipe/api/_api_listing.py | 4 +- .../_api_orchestration_validate.py | 14 +- src/autoskillit/recipe/registry.py | 38 ++++- src/autoskillit/recipe/rules/AGENTS.md | 4 +- src/autoskillit/recipe/rules/rules_inputs.py | 3 + src/autoskillit/recipe/rules/rules_tools.py | 43 +++++ src/autoskillit/recipes/bem-wrapper.json | 7 + src/autoskillit/recipes/bem-wrapper.yaml | 6 + .../campaigns/promote-to-main-campaign.json | 3 +- .../campaigns/promote-to-main-campaign.yaml | 1 + src/autoskillit/recipes/diagrams/research.md | 2 +- .../recipes/promote-to-main-wrapper.json | 1 + .../recipes/promote-to-main-wrapper.yaml | 1 + .../recipes/research-implement.json | 2 + .../recipes/research-implement.yaml | 2 + src/autoskillit/recipes/research.json | 2 + src/autoskillit/recipes/research.yaml | 2 + .../server/recipe/_recipe_execution.py | 8 +- tests/_test_filter.py | 7 + tests/contracts/test_delivery_mode_ledger.py | 2 +- ...t_tool_parameter_binding_migration_note.py | 21 +++ .../fixtures/composed_projection_child.yaml | 1 + .../fixtures/composed_projection_parent.yaml | 1 + tests/recipe/fixtures/deferred_skip_only.yaml | 1 + .../recipe/fixtures/spec_diagram_recipe.yaml | 69 -------- .../test_binding_failure_reachability.py | 161 ++++++++++++++++++ .../test_binding_findings_survive_pruning.py | 149 ++++++++++++++++ tests/recipe/test_compute_recipe_validity.py | 74 +++++++- tests/recipe/test_delivery_segments.py | 1 + tests/recipe/test_rules_dataflow_capture.py | 2 + .../recipe/test_rules_skill_input_template.py | 7 +- tests/recipe/test_sub_recipe_validation.py | 1 + .../server/test_attested_admission_replay.py | 24 +-- ...test_recipe_execution_snapshot_totality.py | 36 ++++ tests/server/test_tool_registry_parity.py | 8 + tests/server/test_tools_recipe.py | 5 + 43 files changed, 640 insertions(+), 134 deletions(-) create mode 100644 src/autoskillit/migrations/0.0.0-to-0.10.1293.yaml create mode 100644 tests/migration/test_tool_parameter_binding_migration_note.py delete mode 100644 tests/recipe/fixtures/spec_diagram_recipe.yaml create mode 100644 tests/recipe/test_binding_failure_reachability.py create mode 100644 tests/recipe/test_binding_findings_survive_pruning.py create mode 100644 tests/server/test_recipe_execution_snapshot_totality.py diff --git a/.autoskillit/recipes/eval/agent-eval.yaml b/.autoskillit/recipes/eval/agent-eval.yaml index de3eb67e11..7697f8d46f 100644 --- a/.autoskillit/recipes/eval/agent-eval.yaml +++ b/.autoskillit/recipes/eval/agent-eval.yaml @@ -90,6 +90,7 @@ steps: provider: minimax with: skill_command: "placeholder — see note for orchestration logic" + cwd: "${{ inputs.source_dir }}" on_success: compile_scorecard on_failure: escalate note: >- diff --git a/.autoskillit/recipes/eval/skill-eval.yaml b/.autoskillit/recipes/eval/skill-eval.yaml index 9ae48ecd0d..d95b5dd0be 100644 --- a/.autoskillit/recipes/eval/skill-eval.yaml +++ b/.autoskillit/recipes/eval/skill-eval.yaml @@ -22,7 +22,7 @@ ingredients: required: true source_dir: description: Path to the source project directory (for judge-eval CWD). - default: "" + required: true output_dir: description: >- Base directory for eval run output. Defaults to @@ -76,6 +76,7 @@ steps: tool: run_skill with: skill_command: "placeholder — see note for orchestration logic" + cwd: "${{ inputs.source_dir }}" on_success: compile_scorecard on_failure: escalate note: >- diff --git a/docs/recipes/authoring.md b/docs/recipes/authoring.md index a85427236e..6568547fa5 100644 --- a/docs/recipes/authoring.md +++ b/docs/recipes/authoring.md @@ -91,7 +91,7 @@ The rule families live in `src/autoskillit/recipe/rules_*.py` (28 files): | `rules_skill_content.py` | SKILL.md bash-block placeholder validation (`undefined-bash-placeholder`) | | `rules_temp_path.py` | Temp-path hygiene: steps that write to paths outside `.autoskillit/temp/` | | `rules_skills.py` | `skill_command` resolvability against the bundled skill registry (`unknown-skill-command`) | -| `rules_tools.py` | MCP tool name and parameter validity (`unknown-tool`, `dead-with-param`) | +| `rules_tools.py` | MCP tool name and parameter validity (`unknown-tool`, `dead-with-param`, `invalid-tool-invocation-binding`) | | `rules_verdict.py` | Skill verdict routing completeness — every emitted verdict value must be routed | | `rules_worktree.py` | Worktree retry lifecycle — every step that creates a worktree must have a downstream merge or cleanup | diff --git a/docs/recipes/overview.md b/docs/recipes/overview.md index bb9e77ad46..6603cf6d37 100644 --- a/docs/recipes/overview.md +++ b/docs/recipes/overview.md @@ -127,12 +127,16 @@ ingredients: task: description: What to do required: true + source_dir: + description: Absolute path to the repository root + required: true steps: step_one: tool: run_skill with: skill_command: "/autoskillit:investigate ${{ inputs.task }}" + cwd: "${{ inputs.source_dir }}" capture: investigation_path: "${{ result.investigation_path }}" on_success: done diff --git a/src/autoskillit/cli/prompts/_prompts_kitchen.py b/src/autoskillit/cli/prompts/_prompts_kitchen.py index 72042a1b91..b055c42071 100755 --- a/src/autoskillit/cli/prompts/_prompts_kitchen.py +++ b/src/autoskillit/cli/prompts/_prompts_kitchen.py @@ -298,6 +298,7 @@ def _build_fleet_dispatch_prompt( ingredients={{ "issue_urls": "", "base_branch": "", + "source_dir": "{project_root}", }}, capture={{"execution_map": "${{{{ result.execution_map }}}}"}}, dispatch_name="bem-pre-step", diff --git a/src/autoskillit/core/tool_registry.py b/src/autoskillit/core/tool_registry.py index 5a0bf890b3..dd5c1d538b 100644 --- a/src/autoskillit/core/tool_registry.py +++ b/src/autoskillit/core/tool_registry.py @@ -75,10 +75,7 @@ def food_truck_visibility_tags(packs: Collection[str] | None) -> tuple[str, ...] "delegate_evidence_reader", ("role", "role_data"), required=("role", "role_data"), - wire_types={ - "role": ToolWireType.STRING, - "role_data": ToolWireType.OBJECT, - }, + wire_types={"role": ToolWireType.STRING, "role_data": ToolWireType.OBJECT}, ), _tool( "read_authorized_artifact", @@ -89,10 +86,7 @@ def food_truck_visibility_tags(packs: Collection[str] | None) -> tuple[str, ...] "get_authorized_artifact_page", ("continuation", "page_size"), required=("continuation",), - wire_types={ - "continuation": ToolWireType.STRING, - "page_size": ToolWireType.INTEGER, - }, + wire_types={"continuation": ToolWireType.STRING, "page_size": ToolWireType.INTEGER}, ), _tool("unlock_agent_pack", ("pack_name",), required=("pack_name",)), _tool( @@ -380,7 +374,7 @@ def food_truck_visibility_tags(packs: Collection[str] | None) -> tuple[str, ...] "commit_files", ("paths", "message", "cwd", "step_name", "self_revert_base_sha"), required=("paths", "message", "cwd"), - wire_types={"self_revert_base_sha": ToolWireType.STRING}, + wire_types={"paths": ToolWireType.ARRAY, "self_revert_base_sha": ToolWireType.STRING}, ), _tool( "write_audit_semantic_result", @@ -459,6 +453,7 @@ def food_truck_visibility_tags(packs: Collection[str] | None) -> tuple[str, ...] "prepare_issue", ("title", "body", "repo", "labels", "dry_run", "split"), required=("title", "body"), + wire_types={"labels": ToolWireType.ARRAY}, ), _tool("claim_issue", ("issue_url", "label", "allow_reentry"), required=("issue_url",)), _tool( @@ -492,7 +487,11 @@ def food_truck_visibility_tags(packs: Collection[str] | None) -> tuple[str, ...] ("project_dir", "_autoskillit_exploration_request_token"), roles={"_autoskillit_exploration_request_token": ToolParamRole.ORCHESTRATOR_SCOPING}, ), - _tool("lock_ingredients", ("locked", "pipeline_id", "unlock")), + _tool( + "lock_ingredients", + ("locked", "pipeline_id", "unlock"), + wire_types={"locked": ToolWireType.OBJECT, "unlock": ToolWireType.ARRAY}, + ), _tool( "declare_join_batch", ("skill_name", "assignments", "session_id", "top_level_parent"), @@ -517,6 +516,7 @@ def food_truck_visibility_tags(packs: Collection[str] | None) -> tuple[str, ...] "record_pipeline_step", ("pipeline_id", "op", "dependencies", "step_name"), roles={"step_name": ToolParamRole.CHILD_INPUT}, + wire_types={"dependencies": ToolWireType.OBJECT}, ), _tool("get_pr_reviews", ("pr_number", "cwd", "repo"), required=("pr_number", "cwd")), _verify_review_receipt(), @@ -637,7 +637,7 @@ def food_truck_visibility_tags(packs: Collection[str] | None) -> tuple[str, ...] "read_db", ("db_path", "query", "params", "timeout"), required=("db_path", "query"), - wire_types={"params": ToolWireType.ARRAY}, + wire_types={"params": ToolWireType.STRING}, ), _tool( "test_check", diff --git a/src/autoskillit/migrations/0.0.0-to-0.10.1293.yaml b/src/autoskillit/migrations/0.0.0-to-0.10.1293.yaml new file mode 100644 index 0000000000..3b6ae1dd7f --- /dev/null +++ b/src/autoskillit/migrations/0.0.0-to-0.10.1293.yaml @@ -0,0 +1,28 @@ +from_version: "0.0.0" +to_version: "0.10.1293" +description: "Declare required tool parameters and match registered wire types." +changes: + - id: required-tool-parameters-declared + description: >- + Recipe validation rejects missing required tool parameters and values with the + wrong wire shape before execution. Quoted numerals and booleans on INTEGER + and BOOLEAN parameters, such as stale_threshold: "30", are now rejected even + where FastMCP previously coerced them. Whole-value ${{ }} templates on list + or mapping parameters were already invalid at runtime and now receive an + earlier validation error. + instruction: >- + Write integer and boolean literals without quotes. Add cwd (usually + "${{ inputs.source_dir }}") under with: for every run_skill and run_cmd step, + and callable for every run_python step. Supply actual lists or mappings for + ARRAY and OBJECT parameters, then run validate_recipe. + example_before: | + run: + tool: run_cmd + with: + cmd: git status + example_after: | + run: + tool: run_cmd + with: + cmd: git status + cwd: "${{ inputs.source_dir }}" diff --git a/src/autoskillit/recipe/api/_api_listing.py b/src/autoskillit/recipe/api/_api_listing.py index 288cd06ce3..59538311fd 100644 --- a/src/autoskillit/recipe/api/_api_listing.py +++ b/src/autoskillit/recipe/api/_api_listing.py @@ -189,7 +189,9 @@ def validate_from_path( if contract: contract_findings = validate_recipe_cards(recipe, contract) - valid = compute_recipe_validity(errors, semantic_findings, contract_findings) + valid = compute_recipe_validity( + errors, semantic_findings, contract_findings, binding_projection=ctx.binding_projection + ) return { "valid": valid, diff --git a/src/autoskillit/recipe/api_orchestration/_api_orchestration_validate.py b/src/autoskillit/recipe/api_orchestration/_api_orchestration_validate.py index 794a13ad14..555b8097cf 100644 --- a/src/autoskillit/recipe/api_orchestration/_api_orchestration_validate.py +++ b/src/autoskillit/recipe/api_orchestration/_api_orchestration_validate.py @@ -50,8 +50,8 @@ ) from autoskillit.recipe.ingredients._recipe_raw_repair import _resolve_skip_guards_in_content from autoskillit.recipe.registry import RuleFinding as _RuleFinding +from autoskillit.recipe.registry import declaration_scoped_rule_names from autoskillit.recipe.rules.rules_plan_set_gate import STAGE_OBLIGATION_GATE_DOMINANCE_RULE -from autoskillit.recipe.rules.rules_tools import STEP_IDENTITY_MISMATCH_RULE from autoskillit.recipe.schema import Recipe, RecipeStep __all__ = ["_record_pipeline_error", "_run_validation_pipeline"] @@ -67,9 +67,10 @@ def _reconcile_semantic_findings( suppressed: _Sequence[str] | None, ) -> tuple[list[_RuleFinding], list[dict[str, Any]]]: """Reconcile pruning diagnostics and suppress displayed version findings.""" + declaration_rules = declaration_scoped_rule_names() + semantic_findings = [f for f in semantic_findings if f.rule not in declaration_rules] if skip_resolutions and any(value is not True for value in skip_resolutions.values()): semantic_findings = filter_pruning_false_positives(semantic_findings, pre_prune_findings) - semantic_suggestions = _orch.findings_to_dicts(semantic_findings) graph_aware_rules = {"capture-inversion-detection", "dead-output"} pre_prune_keys = {(finding.rule, finding.step_name) for finding in pre_prune_findings} for finding in semantic_findings: @@ -88,7 +89,7 @@ def _reconcile_semantic_findings( retained_pre_prune_findings = [ finding for finding in pre_prune_findings - if finding.rule == STEP_IDENTITY_MISMATCH_RULE + if finding.rule in declaration_rules or ( finding.rule == STAGE_OBLIGATION_GATE_DOMINANCE_RULE and "cannot declare skip_when_false" in finding.message @@ -102,8 +103,7 @@ def _reconcile_semantic_findings( for finding in retained_pre_prune_findings if (finding.rule, finding.step_name, finding.message) not in retained_keys ) - if retained_pre_prune_findings: - semantic_suggestions = _orch.findings_to_dicts(semantic_findings) + semantic_suggestions = _orch.findings_to_dicts(semantic_findings) if recipe_name in (suppressed or ()): from autoskillit.recipe.validator import filter_version_rule @@ -358,7 +358,9 @@ def _run_validation_pipeline( if check_diagram_staleness(name, recipes_dir, match.path): suggestions.extend(diagram_stale_to_suggestions(name)) t0 = _orch._t("diagram", t0, name) - valid = _orch.compute_recipe_validity(errors, semantic_findings, contract_findings) + valid = _orch.compute_recipe_validity( + errors, semantic_findings, contract_findings, binding_projection=_post_prune_bindings + ) except YAMLError as exc: _orch.logger.warning("Recipe YAML parse error", name=name, exc_info=True) diff --git a/src/autoskillit/recipe/registry.py b/src/autoskillit/recipe/registry.py index 70f1040e2b..1e09905e4d 100644 --- a/src/autoskillit/recipe/registry.py +++ b/src/autoskillit/recipe/registry.py @@ -12,7 +12,7 @@ from collections.abc import Callable from dataclasses import dataclass -from autoskillit.core import Severity +from autoskillit.core import RecipeBindingProjection, Severity # Imported at module level for type annotations (no circular import: _analysis.py does # not import registry.py). The runtime isinstance() check in run_semantic_rules also @@ -54,6 +54,7 @@ class RuleDef: description: str severity: Severity check: Callable[[ValidationContext], list[RuleFinding]] + declaration_scoped: bool = False _RULE_REGISTRY: list[RuleDef] = [] @@ -89,6 +90,8 @@ def semantic_rule( name: str, description: str, severity: Severity = Severity.WARNING, + *, + declaration_scoped: bool = False, ) -> Callable: """Decorator that registers a semantic validation rule.""" @@ -100,13 +103,29 @@ def decorator( return fn raise RuntimeError(f"Cannot register rule {name!r} after registry finalization") _RULE_REGISTRY.append( - RuleDef(name=name, description=description, severity=severity, check=fn) + RuleDef( + name=name, + description=description, + severity=severity, + check=fn, + declaration_scoped=declaration_scoped, + ) ) return fn return decorator +def declaration_scoped_rule_names() -> frozenset[str]: + """Return rules that validate declarations independently of surviving pruning. + + Their pre-prune findings are authoritative because that pass sees every + declared step. Finding text may describe the unpruned context, such as the + prior captures suggested by ``missing-ingredient``. + """ + return frozenset(rule.name for rule in _RULE_REGISTRY if rule.declaration_scoped) + + def make_finding( rule_name: str, step_name: str, @@ -257,9 +276,20 @@ def compute_recipe_validity( errors: list[str], semantic_findings: list[RuleFinding], contract_findings: list[dict], + *, + binding_projection: RecipeBindingProjection, ) -> bool: - """Return True if no schema, semantic, or contract errors are present.""" + """Require no errors and an installable post-prune binding projection. + + This projection is the object ``build_recipe_execution_snapshot`` installs, + so validity includes installability. + """ has_schema_errors = bool(errors) has_semantic_errors = any(f.severity == Severity.ERROR for f in semantic_findings) has_contract_errors = any(f.get("severity") == "error" for f in contract_findings) - return not has_schema_errors and not has_semantic_errors and not has_contract_errors + return ( + not has_schema_errors + and not has_semantic_errors + and not has_contract_errors + and not binding_projection.failures + ) diff --git a/src/autoskillit/recipe/rules/AGENTS.md b/src/autoskillit/recipe/rules/AGENTS.md index 9866518489..7316948b0a 100644 --- a/src/autoskillit/recipe/rules/AGENTS.md +++ b/src/autoskillit/recipe/rules/AGENTS.md @@ -52,6 +52,6 @@ Files with no registration support their folder; files with several registration - `rules_skill_content_shell_safety.py` — Registers SKILL.md shell-safety semantic rules. - `rules_skill_content_skill_contract.py` — Registers SKILL.md skill-contract semantic rules. - `rules_skills.py` — Registers semantic rules for skill_command resolvability. -- `rules_tools.py` — Registers semantic rules for MCP tool name validity. +- `rules_tools.py` — Registers semantic rules for MCP tool name and invocation binding validity. - `rules_verdict.py` — Registers semantic rules for skill verdict routing completeness and consistency. -- `rules_worktree.py` — Registers worktree and retry validation rules for recipe pipelines. \ No newline at end of file +- `rules_worktree.py` — Registers worktree and retry validation rules for recipe pipelines. diff --git a/src/autoskillit/recipe/rules/rules_inputs.py b/src/autoskillit/recipe/rules/rules_inputs.py index 34d637176a..6f2f7e498b 100644 --- a/src/autoskillit/recipe/rules/rules_inputs.py +++ b/src/autoskillit/recipe/rules/rules_inputs.py @@ -62,6 +62,7 @@ def _unresolved_template_reason(bound: BoundValue, *, per_item: bool) -> str | N "that compile to static values runtime admission cannot accept (recipe validation only)." ), severity=Severity.ERROR, + declaration_scoped=True, ) def _check_unresolved_skill_input_template(ctx: ValidationContext) -> list[RuleFinding]: findings: list[RuleFinding] = [] @@ -127,6 +128,7 @@ def _check_outdated_version(ctx: ValidationContext) -> list[RuleFinding]: name="missing-ingredient", description="Skill steps must provide all required ingredients via context or recipe inputs.", severity=Severity.ERROR, + declaration_scoped=True, ) def _check_unsatisfied_skill_input(ctx: ValidationContext) -> list[RuleFinding]: wf = ctx.recipe @@ -170,6 +172,7 @@ def _check_unsatisfied_skill_input(ctx: ValidationContext) -> list[RuleFinding]: name="invalid-skill-invocation-binding", description="Structured child-skill inputs must bind unambiguously to the selected contract.", severity=Severity.ERROR, + declaration_scoped=True, ) def _check_invalid_skill_invocation_binding( ctx: ValidationContext, diff --git a/src/autoskillit/recipe/rules/rules_tools.py b/src/autoskillit/recipe/rules/rules_tools.py index c5d40ded99..7805eba89f 100644 --- a/src/autoskillit/recipe/rules/rules_tools.py +++ b/src/autoskillit/recipe/rules/rules_tools.py @@ -123,6 +123,7 @@ def _check_constant_step_no_with_args(ctx: ValidationContext) -> list[RuleFindin name="unknown-tool", description="step.tool must be a registered MCP tool name", severity=Severity.ERROR, + declaration_scoped=True, ) def _check_unknown_tool(ctx: ValidationContext) -> list[RuleFinding]: findings: list[RuleFinding] = [] @@ -178,6 +179,7 @@ def _check_subset_disabled_tool(ctx: ValidationContext) -> list[RuleFinding]: name="dead-with-param", description="with: key does not match any known parameter of the step's tool", severity=Severity.ERROR, + declaration_scoped=True, ) def _check_dead_with_params(ctx: ValidationContext) -> list[RuleFinding]: findings: list[RuleFinding] = [] @@ -204,10 +206,51 @@ def _check_dead_with_params(ctx: ValidationContext) -> list[RuleFinding]: return findings +@semantic_rule( + name="invalid-tool-invocation-binding", + description=( + "Recipe tool steps must supply every required tool parameter with a value " + "of the registered wire type." + ), + severity=Severity.ERROR, + declaration_scoped=True, +) +def _check_invalid_tool_invocation_binding(ctx: ValidationContext) -> list[RuleFinding]: + findings: list[RuleFinding] = [] + reported_codes = { + BindingFailureCode.MISSING_TOOL_PARAMETER, + BindingFailureCode.INVALID_TOOL_PARAMETER_TYPE, + } + for step_name, step in ctx.recipe.steps.items(): + if step.tool is None: + continue + invocation = ctx.binding_projection.for_step(step_name) + if invocation is None: + continue + for failure in invocation.failures: + if failure.code not in reported_codes: + continue + message = f"step '{step_name}': tool '{step.tool}' {failure.message}" + if failure.code is BindingFailureCode.MISSING_TOOL_PARAMETER and failure.name == "cwd": + message += ( + ' — add cwd: "${{ inputs.source_dir }}"' + " (or a context worktree path) under with:" + ) + findings.append( + make_finding( + rule_name="invalid-tool-invocation-binding", + step_name=step_name, + message=message, + ) + ) + return findings + + @semantic_rule( name=STEP_IDENTITY_MISMATCH_RULE, description="A step's with: identity parameter must equal its YAML step key", severity=Severity.ERROR, + declaration_scoped=True, ) def _check_step_identity(ctx: ValidationContext) -> list[RuleFinding]: findings: list[RuleFinding] = [] diff --git a/src/autoskillit/recipes/bem-wrapper.json b/src/autoskillit/recipes/bem-wrapper.json index 537a58875a..fd6aeca505 100644 --- a/src/autoskillit/recipes/bem-wrapper.json +++ b/src/autoskillit/recipes/bem-wrapper.json @@ -20,6 +20,10 @@ "description": "Base branch for dependency analysis", "default": "develop", "authority": "config" + }, + "source_dir": { + "description": "Absolute path to the repository root. Required — no default prevents silent cwd inheritance from the MCP server when an empty string is passed.", + "required": true } }, "steps": { @@ -56,6 +60,7 @@ "issue_numbers": "${{ context.issue_numbers }}", "base_ref": "${{ inputs.base_branch }}" }, + "cwd": "${{ inputs.source_dir }}", "step_name": "run_bem", "output_dir": "." }, @@ -83,6 +88,7 @@ "tool": "run_cmd", "with": { "cmd": "echo 'execution_map=${{ context.execution_map }}'", + "cwd": "${{ inputs.source_dir }}", "step_name": "emit_result" }, "on_success": "done", @@ -92,6 +98,7 @@ "tool": "run_cmd", "with": { "cmd": "echo 'execution_map=none'", + "cwd": "${{ inputs.source_dir }}", "step_name": "emit_empty_result" }, "on_success": "done", diff --git a/src/autoskillit/recipes/bem-wrapper.yaml b/src/autoskillit/recipes/bem-wrapper.yaml index b270773d45..d1e1e486ea 100644 --- a/src/autoskillit/recipes/bem-wrapper.yaml +++ b/src/autoskillit/recipes/bem-wrapper.yaml @@ -25,6 +25,9 @@ ingredients: description: "Base branch for dependency analysis" default: "develop" authority: config + source_dir: + description: "Absolute path to the repository root. Required — no default prevents silent cwd inheritance from the MCP server when an empty string is passed." + required: true steps: refetch_issues: @@ -52,6 +55,7 @@ steps: skill_inputs: issue_numbers: "${{ context.issue_numbers }}" base_ref: "${{ inputs.base_branch }}" + cwd: "${{ inputs.source_dir }}" step_name: run_bem output_dir: "." retries: 1 @@ -75,6 +79,7 @@ steps: tool: run_cmd with: cmd: "echo 'execution_map=${{ context.execution_map }}'" + cwd: "${{ inputs.source_dir }}" step_name: emit_result on_success: done on_failure: done @@ -83,6 +88,7 @@ steps: tool: run_cmd with: cmd: "echo 'execution_map=none'" + cwd: "${{ inputs.source_dir }}" step_name: emit_empty_result on_success: done on_failure: done diff --git a/src/autoskillit/recipes/campaigns/promote-to-main-campaign.json b/src/autoskillit/recipes/campaigns/promote-to-main-campaign.json index 6f0393aa7d..9dbdbbbde0 100644 --- a/src/autoskillit/recipes/campaigns/promote-to-main-campaign.json +++ b/src/autoskillit/recipes/campaigns/promote-to-main-campaign.json @@ -82,7 +82,8 @@ "task": "Analyze issue dependencies and compute distribution plan", "ingredients": { "issue_urls": "${{ campaign.issue_urls }}", - "base_branch": "${{ inputs.batch_branch }}" + "base_branch": "${{ inputs.batch_branch }}", + "source_dir": "${{ inputs.source_dir }}" }, "capture": { "execution_map": "${{ result.execution_map }}" diff --git a/src/autoskillit/recipes/campaigns/promote-to-main-campaign.yaml b/src/autoskillit/recipes/campaigns/promote-to-main-campaign.yaml index 6465bf72c6..c95631cc43 100644 --- a/src/autoskillit/recipes/campaigns/promote-to-main-campaign.yaml +++ b/src/autoskillit/recipes/campaigns/promote-to-main-campaign.yaml @@ -73,6 +73,7 @@ dispatches: ingredients: issue_urls: "${{ campaign.issue_urls }}" base_branch: "${{ inputs.batch_branch }}" + source_dir: "${{ inputs.source_dir }}" capture: execution_map: "${{ result.execution_map }}" depends_on: diff --git a/src/autoskillit/recipes/diagrams/research.md b/src/autoskillit/recipes/diagrams/research.md index 9ef3b7c438..55849e4f71 100644 --- a/src/autoskillit/recipes/diagrams/research.md +++ b/src/autoskillit/recipes/diagrams/research.md @@ -1,4 +1,4 @@ - + ## research diff --git a/src/autoskillit/recipes/promote-to-main-wrapper.json b/src/autoskillit/recipes/promote-to-main-wrapper.json index c999a2b56e..01a401d9f4 100644 --- a/src/autoskillit/recipes/promote-to-main-wrapper.json +++ b/src/autoskillit/recipes/promote-to-main-wrapper.json @@ -72,6 +72,7 @@ "tool": "run_cmd", "with": { "cmd": "echo 'pr_url = ${{ context.pr_url }}' && echo 'verdict = ${{ context.verdict }}' && echo 'category_summary = ${{ context.category_summary }}'", + "cwd": "${{ inputs.source_dir }}", "step_name": "emit_result" }, "on_success": "done", diff --git a/src/autoskillit/recipes/promote-to-main-wrapper.yaml b/src/autoskillit/recipes/promote-to-main-wrapper.yaml index d674f0ee44..89952bb68d 100644 --- a/src/autoskillit/recipes/promote-to-main-wrapper.yaml +++ b/src/autoskillit/recipes/promote-to-main-wrapper.yaml @@ -60,6 +60,7 @@ steps: tool: run_cmd with: cmd: "echo 'pr_url = ${{ context.pr_url }}' && echo 'verdict = ${{ context.verdict }}' && echo 'category_summary = ${{ context.category_summary }}'" + cwd: "${{ inputs.source_dir }}" step_name: "emit_result" on_success: done on_failure: done diff --git a/src/autoskillit/recipes/research-implement.json b/src/autoskillit/recipes/research-implement.json index 40052480e9..e0c5f0eb67 100644 --- a/src/autoskillit/recipes/research-implement.json +++ b/src/autoskillit/recipes/research-implement.json @@ -394,6 +394,7 @@ "tool": "run_cmd", "with": { "cmd": "git worktree remove --force \"${{ context.worktree_path }}\"", + "cwd": "${{ inputs.source_dir }}", "step_name": "cleanup_phase_worktree" }, "on_success": "plan_phase", @@ -615,6 +616,7 @@ "tool": "run_cmd", "with": { "cmd": "git worktree remove --force \"${{ context.worktree_path }}\"", + "cwd": "${{ inputs.source_dir }}", "step_name": "pre_remediation_cleanup" }, "on_success": "plan_phase", diff --git a/src/autoskillit/recipes/research-implement.yaml b/src/autoskillit/recipes/research-implement.yaml index b17ec5e20d..6143ae2d53 100644 --- a/src/autoskillit/recipes/research-implement.yaml +++ b/src/autoskillit/recipes/research-implement.yaml @@ -356,6 +356,7 @@ steps: tool: run_cmd with: cmd: git worktree remove --force "${{ context.worktree_path }}" + cwd: "${{ inputs.source_dir }}" step_name: cleanup_phase_worktree on_success: plan_phase on_failure: plan_phase @@ -516,6 +517,7 @@ steps: tool: run_cmd with: cmd: git worktree remove --force "${{ context.worktree_path }}" + cwd: "${{ inputs.source_dir }}" step_name: pre_remediation_cleanup on_success: plan_phase on_failure: plan_phase diff --git a/src/autoskillit/recipes/research.json b/src/autoskillit/recipes/research.json index 24dd7bd927..eb920c5527 100644 --- a/src/autoskillit/recipes/research.json +++ b/src/autoskillit/recipes/research.json @@ -770,6 +770,7 @@ "tool": "run_cmd", "with": { "cmd": "git worktree remove --force \"${{ context.worktree_path }}\"", + "cwd": "${{ inputs.source_dir }}", "step_name": "cleanup_phase_worktree" }, "on_success": "plan_phase", @@ -991,6 +992,7 @@ "tool": "run_cmd", "with": { "cmd": "git worktree remove --force \"${{ context.worktree_path }}\"", + "cwd": "${{ inputs.source_dir }}", "step_name": "pre_remediation_cleanup" }, "on_success": "plan_phase", diff --git a/src/autoskillit/recipes/research.yaml b/src/autoskillit/recipes/research.yaml index c47a133599..f9a2eb854e 100644 --- a/src/autoskillit/recipes/research.yaml +++ b/src/autoskillit/recipes/research.yaml @@ -698,6 +698,7 @@ steps: tool: run_cmd with: cmd: git worktree remove --force "${{ context.worktree_path }}" + cwd: "${{ inputs.source_dir }}" step_name: cleanup_phase_worktree on_success: plan_phase on_failure: plan_phase @@ -858,6 +859,7 @@ steps: tool: run_cmd with: cmd: git worktree remove --force "${{ context.worktree_path }}" + cwd: "${{ inputs.source_dir }}" step_name: pre_remediation_cleanup on_success: plan_phase on_failure: plan_phase diff --git a/src/autoskillit/server/recipe/_recipe_execution.py b/src/autoskillit/server/recipe/_recipe_execution.py index a80be6106e..6bc716d1ff 100644 --- a/src/autoskillit/server/recipe/_recipe_execution.py +++ b/src/autoskillit/server/recipe/_recipe_execution.py @@ -307,10 +307,10 @@ def build_recipe_execution_snapshot( ): dynamic_skill_step_names.add(step_name) continue - if not invocation.attested: - continue - if invocation.skill_name is None: - raise AssertionError("attested invocation is missing its skill identity") + if not invocation.attested or invocation.skill_name is None: + raise AssertionError( + f"uninstallable run_skill step {step_name!r}: {invocation.failures}" + ) skill_identity = compute_skill_contract_identity(invocation.skill_name) digest = compute_invocation_template_digest( execution_id=active_execution_id, diff --git a/tests/_test_filter.py b/tests/_test_filter.py index bd1ea92c8f..3277e0f69a 100644 --- a/tests/_test_filter.py +++ b/tests/_test_filter.py @@ -146,6 +146,12 @@ class SourceMapRejection(enum.StrEnum): # target the scan reaches from it. TEST_HELPER_CASCADE: dict[str, frozenset[str]] = { "tests/_test_filter.py": _TEST_FILTER_VALIDATORS, + "tests/recipe/test_skill_invocation_binding.py": frozenset( + { + "recipe/test_binding_failure_reachability.py", + "recipe/test_skill_invocation_binding.py", + } + ), "tests/arch/_helpers.py": _ARCH_HELPER_TEST_TARGETS, "tests/arch/_rules.py": _ARCH_HELPER_TEST_TARGETS, "tests/fleet/_reaper_test_support.py": frozenset({"fleet"}), @@ -1355,6 +1361,7 @@ class SourceMapRejection(enum.StrEnum): "server/test_serve_idempotence.py", "server/test_attestation_delivery_reachability.py", "server/test_recipe_execution_receipt_completeness.py", + "server/test_recipe_execution_snapshot_totality.py", "server/test_attested_admission_replay.py", "server/test_recipe_checkpoint_delivery_regressions.py", "server/test_tools_status_kitchen.py", diff --git a/tests/contracts/test_delivery_mode_ledger.py b/tests/contracts/test_delivery_mode_ledger.py index 0bf1013de4..d4acd862ad 100644 --- a/tests/contracts/test_delivery_mode_ledger.py +++ b/tests/contracts/test_delivery_mode_ledger.py @@ -178,7 +178,7 @@ def test_delivery_mode_is_pinned( # Each stored maximum includes 10% headroom above the measured serialized char count. # Update when a recipe grows/shrinks past these thresholds. _EXPECTED_MAX_SERIALIZED_CHARS: dict[tuple[str, str], int] = { - ("bem-wrapper", "claude-code"): 24_100, + ("bem-wrapper", "claude-code"): 27_900, ("consolidate-health-reports", "claude-code"): 13_300, ("consolidate-health-reports", "codex"): 12_000, ("full-audit", "claude-code"): 37_000, diff --git a/tests/migration/test_tool_parameter_binding_migration_note.py b/tests/migration/test_tool_parameter_binding_migration_note.py new file mode 100644 index 0000000000..c6a63f3153 --- /dev/null +++ b/tests/migration/test_tool_parameter_binding_migration_note.py @@ -0,0 +1,21 @@ +"""The tool-parameter migration note reaches recipes from supported releases.""" + +import pytest + +from autoskillit.migration.loader import applicable_migrations + +pytestmark = [pytest.mark.layer("migration"), pytest.mark.small] + + +@pytest.mark.parametrize("recipe_version", ["0.10.1200", "0.10.1288", "0.10.1291", "0.10.1292"]) +def test_tool_parameter_binding_migration_applies_on_upgrade(recipe_version: str) -> None: + to_version = "0.10.1293" + notes = applicable_migrations(recipe_version, to_version) + matching = [ + note + for note in notes + if any(change.id == "required-tool-parameters-declared" for change in note.changes) + ] + + assert len(matching) == 1 + assert matching[0].to_version == to_version diff --git a/tests/recipe/fixtures/composed_projection_child.yaml b/tests/recipe/fixtures/composed_projection_child.yaml index 0ded8a3f9d..d28a461f02 100644 --- a/tests/recipe/fixtures/composed_projection_child.yaml +++ b/tests/recipe/fixtures/composed_projection_child.yaml @@ -11,5 +11,6 @@ steps: tool: run_python with: step_name: child_step + callable: autoskillit.smoke_utils.check_loop_iteration on_success: done on_failure: escalate diff --git a/tests/recipe/fixtures/composed_projection_parent.yaml b/tests/recipe/fixtures/composed_projection_parent.yaml index 9248795f7e..ace714271b 100644 --- a/tests/recipe/fixtures/composed_projection_parent.yaml +++ b/tests/recipe/fixtures/composed_projection_parent.yaml @@ -12,6 +12,7 @@ steps: tool: run_python with: step_name: parent_step + callable: autoskillit.smoke_utils.check_loop_iteration on_success: child_placeholder on_failure: escalate child_placeholder: diff --git a/tests/recipe/fixtures/deferred_skip_only.yaml b/tests/recipe/fixtures/deferred_skip_only.yaml index f130ed161d..d6bf3d9f53 100644 --- a/tests/recipe/fixtures/deferred_skip_only.yaml +++ b/tests/recipe/fixtures/deferred_skip_only.yaml @@ -10,6 +10,7 @@ steps: tool: run_python with: step_name: guarded + callable: autoskillit.smoke_utils.check_loop_iteration skip_when_false: inputs.enabled on_success: Z on_failure: escalate diff --git a/tests/recipe/fixtures/spec_diagram_recipe.yaml b/tests/recipe/fixtures/spec_diagram_recipe.yaml deleted file mode 100644 index 03eb3719e7..0000000000 --- a/tests/recipe/fixtures/spec_diagram_recipe.yaml +++ /dev/null @@ -1,69 +0,0 @@ -name: spec-diagram-oracle -description: Minimal recipe for spec-oracle testing. Covers all layout features. -autoskillit_version: "0.3.0" - -ingredients: - plan_path: - description: Pre-existing plan file path (skip make-plan if provided) - default: "false" - task: - description: > - The implementation task - required: true - -steps: - plan: - tool: run_skill - with: - skill_command: /autoskillit:make-plan - capture: - plan_path: "${{ result.plan_path }}" - on_success: implement - on_failure: escalate - skip_when_false: plan_path - - implement: - tool: run_skill - with: - skill_command: /autoskillit:implement-worktree-no-merge - retries: 0 - on_success: verify - on_failure: fix - on_context_limit: retry_worktree - - verify: - tool: run_skill - with: - skill_command: /autoskillit:dry-walkthrough - on_success: next_or_done - on_failure: escalate - - fix: - tool: run_skill - with: - skill_command: /autoskillit:resolve-failures - on_success: verify - on_failure: escalate - - retry_worktree: - tool: run_skill - with: - skill_command: /autoskillit:retry-worktree - on_success: verify - on_failure: escalate - - next_or_done: - action: route - on_result: - - when: "${{ result.next }} == more_parts" - route: plan - - route: done - note: "Checks if more plan_parts remain; routes back to plan for the next part." - - done: - action: stop - message: "Complete." - - escalate: - action: stop - message: "Failed." diff --git a/tests/recipe/test_binding_failure_reachability.py b/tests/recipe/test_binding_failure_reachability.py new file mode 100644 index 0000000000..04205a122a --- /dev/null +++ b/tests/recipe/test_binding_failure_reachability.py @@ -0,0 +1,161 @@ +"""Every invocation-binding failure reaches its declaration-scoped error rule.""" + +from __future__ import annotations + +import pytest + +from autoskillit.core import BindingFailureCode, Severity +from autoskillit.recipe import make_validation_context +from autoskillit.recipe._binding import bind_recipe +from autoskillit.recipe.registry import _RULE_REGISTRY, run_semantic_rules +from autoskillit.recipe.schema import Recipe, RecipeStep +from tests.recipe.test_skill_invocation_binding import _manifest, _required_inputs + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + +_DRY_WALKTHROUGH = "/autoskillit:dry-walkthrough" +_REQUIRED_INPUTS = _required_inputs() + +_TRIGGERS: dict[BindingFailureCode, tuple[str, dict[str, object]]] = { + BindingFailureCode.UNKNOWN_TOOL: ("no_such_tool", {"x": 1}), + BindingFailureCode.UNKNOWN_TOOL_PARAMETER: ( + "run_cmd", + {"cmd": "pwd", "cwd": "/r", "bogus": 1}, + ), + BindingFailureCode.MISSING_TOOL_PARAMETER: ("run_cmd", {"cmd": "pwd"}), + BindingFailureCode.INVALID_TOOL_PARAMETER_TYPE: ( + "bulk_close_issues", + { + "issue_numbers": "${{ context.issue_numbers }}", + "comment": "c", + "cwd": "/r", + }, + ), + BindingFailureCode.UNKNOWN_SKILL: ( + "run_skill", + {"skill_command": "/autoskillit:no-such-skill", "cwd": "/r"}, + ), + BindingFailureCode.UNKNOWN_SKILL_INPUT: ( + "run_skill", + { + "skill_command": _DRY_WALKTHROUGH, + "cwd": "/r", + "skill_inputs": {**_REQUIRED_INPUTS, "unknown_child": "v"}, + }, + ), + BindingFailureCode.MISSING_SKILL_INPUT: ( + "run_skill", + { + "skill_command": _DRY_WALKTHROUGH, + "cwd": "/r", + "skill_inputs": { + name: value for name, value in _REQUIRED_INPUTS.items() if name != "plan_path" + }, + }, + ), + BindingFailureCode.AMBIGUOUS_SKILL_INPUT: ( + "run_skill", + { + "skill_command": f"{_DRY_WALKTHROUGH} /tmp/x.md", + "cwd": "/r", + "skill_inputs": dict(_REQUIRED_INPUTS), + }, + ), + BindingFailureCode.DEAD_SKILL_INPUT: ( + "run_skill", + { + "skill_command": ( + f"{_DRY_WALKTHROUGH} /tmp/a https://x /extra /e2 /e3 /e4 /e5 /e6 /e7 /e8" + ), + "cwd": "/r", + }, + ), + BindingFailureCode.INVALID_SKILL_INPUT_TYPE: ( + "run_skill", + { + "skill_command": _DRY_WALKTHROUGH, + "cwd": "/r", + "skill_inputs": {**_REQUIRED_INPUTS, "round": "abc"}, + }, + ), + BindingFailureCode.INVALID_SKILL_COMMAND: ( + "run_skill", + { + "skill_command": f"{_DRY_WALKTHROUGH} /tmp/plan.md https://x", + "cwd": "/r", + }, + ), +} + +_EXPECTED_RULE: dict[BindingFailureCode, str] = { + BindingFailureCode.UNKNOWN_TOOL: "unknown-tool", + BindingFailureCode.UNKNOWN_TOOL_PARAMETER: "dead-with-param", + BindingFailureCode.MISSING_TOOL_PARAMETER: "invalid-tool-invocation-binding", + BindingFailureCode.INVALID_TOOL_PARAMETER_TYPE: "invalid-tool-invocation-binding", + BindingFailureCode.MISSING_SKILL_INPUT: "missing-ingredient", + BindingFailureCode.UNKNOWN_SKILL: "invalid-skill-invocation-binding", + BindingFailureCode.UNKNOWN_SKILL_INPUT: "invalid-skill-invocation-binding", + BindingFailureCode.AMBIGUOUS_SKILL_INPUT: "invalid-skill-invocation-binding", + BindingFailureCode.DEAD_SKILL_INPUT: "invalid-skill-invocation-binding", + BindingFailureCode.INVALID_SKILL_INPUT_TYPE: "invalid-skill-invocation-binding", + BindingFailureCode.INVALID_SKILL_COMMAND: "invalid-skill-invocation-binding", +} + +_EXPECTED_PARAMETER = { + BindingFailureCode.UNKNOWN_TOOL_PARAMETER: "bogus", + BindingFailureCode.MISSING_TOOL_PARAMETER: "cwd", + BindingFailureCode.INVALID_TOOL_PARAMETER_TYPE: "issue_numbers", +} + + +def _recipe_for(code: BindingFailureCode) -> Recipe: + tool, with_args = _TRIGGERS[code] + step = RecipeStep(name="s", tool=tool, with_args=dict(with_args)) + if code is BindingFailureCode.INVALID_SKILL_COMMAND: + step.declared_with_args = { + "skill_command": f"{_DRY_WALKTHROUGH} /tmp/plan.md", + "cwd": "/r", + } + return Recipe(name="r", description="d", steps={"s": step}) + + +def test_tables_cover_every_code() -> None: + assert len(BindingFailureCode.__members__) == len(BindingFailureCode) + assert set(_TRIGGERS) == set(_EXPECTED_RULE) == set(BindingFailureCode) + + +@pytest.mark.parametrize("code", list(_TRIGGERS), ids=lambda code: code.name.lower()) +def test_code_reaches_expected_rule(code: BindingFailureCode) -> None: + recipe = _recipe_for(code) + projection = bind_recipe(recipe, manifest=_manifest()) + + assert code in {failure.code for failure in projection.for_step("s").failures} + + context = make_validation_context(recipe, binding_projection=projection) + findings = run_semantic_rules(context) + finding = next( + ( + candidate + for candidate in findings + if candidate.rule == _EXPECTED_RULE[code] + and candidate.step_name == "s" + and candidate.severity is Severity.ERROR + ), + None, + ) + assert finding is not None + if code in _EXPECTED_PARAMETER: + assert _EXPECTED_PARAMETER[code] in finding.message + + +def test_binding_reporters_are_error_and_declaration_scoped() -> None: + expected_rules = set(_EXPECTED_RULE.values()) | { + "step-identity-mismatch", + "unresolved-skill-input-template", + } + rules = {rule.name: rule for rule in _RULE_REGISTRY} + + for name in expected_rules: + rule = rules[name] + assert rule.severity is Severity.ERROR + assert rule.declaration_scoped is True diff --git a/tests/recipe/test_binding_findings_survive_pruning.py b/tests/recipe/test_binding_findings_survive_pruning.py new file mode 100644 index 0000000000..374be330fa --- /dev/null +++ b/tests/recipe/test_binding_findings_survive_pruning.py @@ -0,0 +1,149 @@ +"""Declaration binding errors survive ingredient-based step pruning.""" + +from __future__ import annotations + +import textwrap +from pathlib import Path + +import pytest + +from autoskillit.recipe._api import load_and_validate + +pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] + + +def _install_recipe(tmp_path: Path, name: str, content: str) -> None: + recipes_dir = tmp_path / ".autoskillit" / "recipes" + recipes_dir.mkdir(parents=True, exist_ok=True) + (recipes_dir / f"{name}.yaml").write_text(textwrap.dedent(content)) + + +def test_missing_ingredient_survives_pruned_producer(tmp_path: Path) -> None: + _install_recipe( + tmp_path, + "probe-prune", + """ + name: probe-prune + description: Probe whether pruning hides a missing skill input finding. + kitchen_rules: + - Follow the declared routes. + ingredients: + flag: + description: Whether to run the producer step. + default: "false" + steps: + maybe: + tool: run_python + with: + callable: autoskillit.smoke_utils.check_loop_iteration + step_name: maybe + skip_when_false: inputs.flag + capture: + issue_numbers: "${{ result.next_iteration }}" + on_success: bem + on_failure: done + on_skip: bem + bem: + tool: run_skill + with: + skill_command: "/autoskillit:build-execution-map" + skill_inputs: + base_ref: "main" + cwd: "/tmp" + step_name: bem + output_dir: "." + capture: + execution_map: "${{ result.execution_map }}" + on_success: emit + on_failure: done + emit: + tool: run_cmd + with: + cmd: "echo 'execution_map=${{ context.execution_map }}'" + cwd: "/tmp" + step_name: emit + on_success: done + on_failure: done + done: + action: stop + message: Emit the L3 sentinel block with success=true and reason=Completed. + """, + ) + + result = load_and_validate( + "probe-prune", project_dir=tmp_path, ingredient_overrides={"flag": "false"} + ) + + assert result["valid"] is False + assert any( + suggestion.get("rule") == "missing-ingredient" + and suggestion.get("step") == "bem" + and suggestion.get("severity") == "error" + for suggestion in result["suggestions"] + ) + + +def test_pruned_step_binding_failure_is_reported(tmp_path: Path) -> None: + _install_recipe( + tmp_path, + "pruned-binding-failure", + """ + name: pruned-binding-failure + description: Probe whether pruning hides a tool binding failure. + kitchen_rules: + - Follow the declared routes. + ingredients: + flag: + description: Whether to run the producer step. + default: "false" + steps: + maybe: + tool: run_cmd + with: + cmd: "echo producer" + skip_when_false: inputs.flag + capture: + issue_numbers: "${{ result.next_iteration }}" + on_success: bem + on_failure: done + on_skip: bem + bem: + tool: run_skill + with: + skill_command: "/autoskillit:build-execution-map" + skill_inputs: + base_ref: "main" + cwd: "/tmp" + step_name: bem + output_dir: "." + capture: + execution_map: "${{ result.execution_map }}" + on_success: emit + on_failure: done + emit: + tool: run_cmd + with: + cmd: "echo 'execution_map=${{ context.execution_map }}'" + cwd: "/tmp" + step_name: emit + on_success: done + on_failure: done + done: + action: stop + message: Emit the L3 sentinel block with success=true and reason=Completed. + """, + ) + + result = load_and_validate( + "pruned-binding-failure", + project_dir=tmp_path, + ingredient_overrides={"flag": "false"}, + ) + + assert result["valid"] is False + assert any( + suggestion.get("rule") == "invalid-tool-invocation-binding" + and suggestion.get("step") == "maybe" + and suggestion.get("severity") == "error" + for suggestion in result["suggestions"] + ) diff --git a/tests/recipe/test_compute_recipe_validity.py b/tests/recipe/test_compute_recipe_validity.py index 0edee318d9..6c7f711f07 100644 --- a/tests/recipe/test_compute_recipe_validity.py +++ b/tests/recipe/test_compute_recipe_validity.py @@ -4,40 +4,100 @@ import pytest +from autoskillit.core import RecipeBindingProjection from autoskillit.core.types import Severity +from autoskillit.recipe._binding import bind_recipe from autoskillit.recipe.registry import RuleFinding, compute_recipe_validity +from autoskillit.recipe.schema import Recipe, RecipeStep pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] class TestComputeRecipeValidity: def test_all_clean_returns_true(self) -> None: - assert compute_recipe_validity([], [], []) is True + assert ( + compute_recipe_validity([], [], [], binding_projection=RecipeBindingProjection({})) + is True + ) def test_schema_error_returns_false(self) -> None: - assert compute_recipe_validity(["missing name field"], [], []) is False + assert ( + compute_recipe_validity( + ["missing name field"], [], [], binding_projection=RecipeBindingProjection({}) + ) + is False + ) def test_semantic_error_returns_false(self) -> None: finding = RuleFinding( rule="test-rule", severity=Severity.ERROR, step_name="s1", message="fail" ) - assert compute_recipe_validity([], [finding], []) is False + assert ( + compute_recipe_validity( + [], [finding], [], binding_projection=RecipeBindingProjection({}) + ) + is False + ) def test_semantic_warning_returns_true(self) -> None: finding = RuleFinding( rule="test-rule", severity=Severity.WARNING, step_name="s1", message="warn" ) - assert compute_recipe_validity([], [finding], []) is True + assert ( + compute_recipe_validity( + [], [finding], [], binding_projection=RecipeBindingProjection({}) + ) + is True + ) def test_contract_error_returns_false(self) -> None: finding = {"rule": "contract-unsatisfied-input", "severity": "error", "step": "s1"} - assert compute_recipe_validity([], [], [finding]) is False + assert ( + compute_recipe_validity( + [], [], [finding], binding_projection=RecipeBindingProjection({}) + ) + is False + ) def test_contract_warning_returns_true(self) -> None: finding = {"rule": "stale-contract", "severity": "warning", "step": "s1"} - assert compute_recipe_validity([], [], [finding]) is True + assert ( + compute_recipe_validity( + [], [], [finding], binding_projection=RecipeBindingProjection({}) + ) + is True + ) def test_multiple_error_sources_returns_false(self) -> None: semantic = RuleFinding(rule="r1", severity=Severity.ERROR, step_name="s1", message="fail") contract = {"rule": "r2", "severity": "error", "step": "s2"} - assert compute_recipe_validity(["schema err"], [semantic], [contract]) is False + assert ( + compute_recipe_validity( + ["schema err"], + [semantic], + [contract], + binding_projection=RecipeBindingProjection({}), + ) + is False + ) + + def test_binding_failures_invalidate_recipe(self) -> None: + invalid = Recipe( + name="binding-invalid", + description="A required run_cmd parameter is missing.", + steps={"step": RecipeStep(name="step", tool="run_cmd", with_args={"cmd": "pwd"})}, + ) + valid = Recipe( + name="binding-valid", + description="All required run_cmd parameters are present.", + steps={ + "step": RecipeStep( + name="step", tool="run_cmd", with_args={"cmd": "pwd", "cwd": "/r"} + ) + }, + ) + + assert ( + compute_recipe_validity([], [], [], binding_projection=bind_recipe(invalid)) is False + ) + assert compute_recipe_validity([], [], [], binding_projection=bind_recipe(valid)) is True diff --git a/tests/recipe/test_delivery_segments.py b/tests/recipe/test_delivery_segments.py index 532b328c04..a0f1380510 100644 --- a/tests/recipe/test_delivery_segments.py +++ b/tests/recipe/test_delivery_segments.py @@ -188,6 +188,7 @@ def test_finalization_drops_segments_emptied_by_unreachable_sweep(tmp_path) -> N tool: run_python with: step_name: start + callable: autoskillit.smoke_utils.check_loop_iteration on_success: finish on_failure: escalate finish: diff --git a/tests/recipe/test_rules_dataflow_capture.py b/tests/recipe/test_rules_dataflow_capture.py index dc53152154..34c7c012df 100644 --- a/tests/recipe/test_rules_dataflow_capture.py +++ b/tests/recipe/test_rules_dataflow_capture.py @@ -6,6 +6,7 @@ import pytest +from autoskillit.core import RecipeBindingProjection from autoskillit.core.io import load_yaml from autoskillit.core.types import Severity from autoskillit.recipe.io import _parse_recipe @@ -178,6 +179,7 @@ def test_undeclared_capture_key_blocks_validity(self) -> None: errors=[], semantic_findings=findings, contract_findings=[], + binding_projection=RecipeBindingProjection({}), ) diff --git a/tests/recipe/test_rules_skill_input_template.py b/tests/recipe/test_rules_skill_input_template.py index 98e9cff714..292afac29c 100644 --- a/tests/recipe/test_rules_skill_input_template.py +++ b/tests/recipe/test_rules_skill_input_template.py @@ -12,7 +12,7 @@ import pytest -from autoskillit.core import BindingFailureCode +from autoskillit.core import BindingFailureCode, RecipeBindingProjection from autoskillit.core.types import Severity from autoskillit.recipe._binding import bind_recipe from autoskillit.recipe.registry import ( @@ -324,4 +324,7 @@ def test_merge_pr_structured_placeholder_reproduction_is_invalid() -> None: assert all(finding.severity == Severity.ERROR for finding in findings) assert any("complexity" in finding.message for finding in findings) assert any("pr_number" in finding.message for finding in findings) - assert compute_recipe_validity([], findings, []) is False + assert ( + compute_recipe_validity([], findings, [], binding_projection=RecipeBindingProjection({})) + is False + ) diff --git a/tests/recipe/test_sub_recipe_validation.py b/tests/recipe/test_sub_recipe_validation.py index 8d41fc7f12..e39746ec6a 100644 --- a/tests/recipe/test_sub_recipe_validation.py +++ b/tests/recipe/test_sub_recipe_validation.py @@ -51,6 +51,7 @@ def _write_sub_recipe(sub_dir: Path, name: str = "test-sub") -> None: tool: run_cmd with: cmd: echo hi + cwd: /tmp on_success: done on_failure: escalate """) diff --git a/tests/server/test_attested_admission_replay.py b/tests/server/test_attested_admission_replay.py index d9cbc60d79..169afa90ef 100644 --- a/tests/server/test_attested_admission_replay.py +++ b/tests/server/test_attested_admission_replay.py @@ -39,20 +39,7 @@ pytestmark = [pytest.mark.layer("server"), pytest.mark.medium] REPO_ROOT = Path(__file__).resolve().parents[2] -_PINNED: dict[tuple[str, str], Any] = { - ("bem-wrapper", "run_bem"): pytest.param( - "bem-wrapper", - "run_bem", - "step_not_installed", - "missing_tool_parameter: cwd", - id="bem-wrapper-run_bem", - marks=pytest.mark.xfail( - strict=True, - raises=AdmissionDefect, - reason="Blocked by #5310: missing required cwd silently prevents step installation", - ), - ), -} +_PINNED: dict[tuple[str, str], Any] = {} @cache @@ -149,12 +136,6 @@ def _dynamic_step_outcome(step_name: str) -> NoReturn: ) -def _uninstalled_step_outcome(projection: Any, step_name: str) -> tuple[str, str]: - failures = projection.invocations[step_name].failures - detail = "; ".join(f"{failure.code.value}: {failure.name}" for failure in failures) - return "step_not_installed", detail - - def _templated_step_outcome( step_name: str, step_body: dict[str, Any], @@ -200,9 +181,6 @@ def _step_outcome( snapshot = installed.snapshot if step_name in snapshot.dynamic_skill_step_names: _dynamic_step_outcome(step_name) - if step_name not in snapshot.templates: - projection = loaded["_finalized_projection"].binding_projection - return _uninstalled_step_outcome(projection, step_name) return _templated_step_outcome( step_name, steps[step_name], diff --git a/tests/server/test_recipe_execution_snapshot_totality.py b/tests/server/test_recipe_execution_snapshot_totality.py new file mode 100644 index 0000000000..b71899e232 --- /dev/null +++ b/tests/server/test_recipe_execution_snapshot_totality.py @@ -0,0 +1,36 @@ +"""Failed run_skill bindings cannot be represented by incomplete snapshots.""" + +from __future__ import annotations + +import pytest + +from autoskillit.recipe._binding import bind_recipe +from autoskillit.recipe.schema import Recipe, RecipeStep +from autoskillit.server.recipe._recipe_execution import build_recipe_execution_snapshot + +pytestmark = [pytest.mark.layer("server"), pytest.mark.small] + +_HASH = "sha256:" + "0" * 64 + + +def test_snapshot_rejects_failing_run_skill_binding() -> None: + recipe = Recipe( + name="r", + description="d", + steps={ + "run_bem": RecipeStep( + name="run_bem", + tool="run_skill", + with_args={"skill_command": "/autoskillit:build-execution-map"}, + ) + }, + ) + projection = bind_recipe(recipe) + + with pytest.raises(AssertionError, match=r"run_bem.*missing_tool_parameter"): + build_recipe_execution_snapshot( + recipe_name="r", + content_hash=_HASH, + composite_hash=_HASH, + projection=projection, + ) diff --git a/tests/server/test_tool_registry_parity.py b/tests/server/test_tool_registry_parity.py index a7fd916006..a236dd02b0 100644 --- a/tests/server/test_tool_registry_parity.py +++ b/tests/server/test_tool_registry_parity.py @@ -237,6 +237,14 @@ def test_registry_preserves_typed_handler_wire_contracts() -> None: "post_state": ToolWireType.STRING, "step_name": ToolWireType.STRING, }, + "commit_files": {"paths": ToolWireType.ARRAY}, + "lock_ingredients": { + "locked": ToolWireType.OBJECT, + "unlock": ToolWireType.ARRAY, + }, + "prepare_issue": {"labels": ToolWireType.ARRAY}, + "read_db": {"params": ToolWireType.STRING}, + "record_pipeline_step": {"dependencies": ToolWireType.OBJECT}, } for tool_name, wire_types in expected.items(): diff --git a/tests/server/test_tools_recipe.py b/tests/server/test_tools_recipe.py index bdf536bbe2..31ed636c84 100644 --- a/tests/server/test_tools_recipe.py +++ b/tests/server/test_tools_recipe.py @@ -178,6 +178,9 @@ async def test_validate_recipe_with_on_result(self, tmp_path): "steps:\n" " classify:\n" " tool: classify_fix\n" + " with:\n" + " worktree_path: /tmp/worktree\n" + " base_branch: main\n" " on_result:\n" " field: restart_scope\n" " routes:\n" @@ -243,6 +246,8 @@ async def test_validate_recipe_includes_semantic_findings(self, tmp_path): "steps:\n" " check:\n" " tool: test_check\n" + " with:\n" + " worktree_path: /tmp/worktree\n" " model: sonnet\n" " on_success: done\n" " on_failure: escalate\n" From cb4a50c0eaaed0d79d12f2266fecfee083e479ab Mon Sep 17 00:00:00 2001 From: Trecek Date: Sat, 24 Oct 2026 16:56:15 -0700 Subject: [PATCH 02/17] fix: address binding validity audit findings --- src/autoskillit/migrations/0.0.0-to-0.10.1293.yaml | 12 ++++++------ .../api_orchestration/_api_orchestration_validate.py | 3 --- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/autoskillit/migrations/0.0.0-to-0.10.1293.yaml b/src/autoskillit/migrations/0.0.0-to-0.10.1293.yaml index 3b6ae1dd7f..355746df9c 100644 --- a/src/autoskillit/migrations/0.0.0-to-0.10.1293.yaml +++ b/src/autoskillit/migrations/0.0.0-to-0.10.1293.yaml @@ -4,12 +4,12 @@ description: "Declare required tool parameters and match registered wire types." changes: - id: required-tool-parameters-declared description: >- - Recipe validation rejects missing required tool parameters and values with the - wrong wire shape before execution. Quoted numerals and booleans on INTEGER - and BOOLEAN parameters, such as stale_threshold: "30", are now rejected even - where FastMCP previously coerced them. Whole-value ${{ }} templates on list - or mapping parameters were already invalid at runtime and now receive an - earlier validation error. + Quoted numerals and booleans on INTEGER and BOOLEAN parameters, such as + stale_threshold: "30", are now rejected even where FastMCP previously coerced + them. Recipe validation also rejects missing required tool parameters and + values with the wrong wire shape before execution. Whole-value ${{ }} templates + on list or mapping parameters were already invalid at runtime and now receive + an earlier validation error. instruction: >- Write integer and boolean literals without quotes. Add cwd (usually "${{ inputs.source_dir }}") under with: for every run_skill and run_cmd step, diff --git a/src/autoskillit/recipe/api_orchestration/_api_orchestration_validate.py b/src/autoskillit/recipe/api_orchestration/_api_orchestration_validate.py index 555b8097cf..7b4e9c0da9 100644 --- a/src/autoskillit/recipe/api_orchestration/_api_orchestration_validate.py +++ b/src/autoskillit/recipe/api_orchestration/_api_orchestration_validate.py @@ -59,7 +59,6 @@ def _reconcile_semantic_findings( semantic_findings: list[_RuleFinding], - semantic_suggestions: list[dict[str, Any]], *, pre_prune_findings: list[_RuleFinding], skip_resolutions: dict[str, bool | None], @@ -314,11 +313,9 @@ def _run_validation_pipeline( effective_routing_edges=_effective_analysis_edges, ) semantic_findings = _orch.run_semantic_rules(val_ctx) - semantic_suggestions = _orch.findings_to_dicts(semantic_findings) t0 = _orch._t("semantic_rules", t0, name) semantic_findings, semantic_suggestions = _reconcile_semantic_findings( semantic_findings, - semantic_suggestions, pre_prune_findings=_pre_prune_findings, skip_resolutions=_skip_resolutions, recipe_name=name, From 981f84075071ecdd586f1d6bca54281e072521d8 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sat, 24 Oct 2026 17:40:30 -0700 Subject: [PATCH 03/17] fix(review): retain declaration binding errors in path validation --- src/autoskillit/recipe/api/_api_listing.py | 7 ++++++- .../_api_orchestration_validate.py | 16 +++++++--------- src/autoskillit/recipe/helpers/_rule_helpers.py | 12 ++++++++++++ .../test_binding_findings_survive_pruning.py | 13 +++++++++++++ 4 files changed, 38 insertions(+), 10 deletions(-) diff --git a/src/autoskillit/recipe/api/_api_listing.py b/src/autoskillit/recipe/api/_api_listing.py index 59538311fd..574cf08c82 100644 --- a/src/autoskillit/recipe/api/_api_listing.py +++ b/src/autoskillit/recipe/api/_api_listing.py @@ -17,7 +17,10 @@ ) from autoskillit.recipe.analysis._analysis import make_validation_context from autoskillit.recipe.contracts.contracts import load_recipe_card, validate_recipe_cards -from autoskillit.recipe.helpers._rule_helpers import filter_pruning_false_positives +from autoskillit.recipe.helpers._rule_helpers import ( + filter_pruning_false_positives, + retain_declaration_findings, +) from autoskillit.recipe.ingredients._recipe_composition import _prune_skipped_steps from autoskillit.recipe.ingredients._recipe_ingredients import RecipeListItem from autoskillit.recipe.io import ( @@ -178,6 +181,8 @@ def validate_from_path( # this rule. The filter remains useful for other rules like # dead-output where pruning genuinely introduces new findings. semantic_findings = filter_pruning_false_positives(semantic_findings, _pre_prune_findings) + if ingredient_overrides: + semantic_findings = retain_declaration_findings(semantic_findings, _pre_prune_findings) quality = build_quality_dict(report) semantic = findings_to_dicts(semantic_findings) diff --git a/src/autoskillit/recipe/api_orchestration/_api_orchestration_validate.py b/src/autoskillit/recipe/api_orchestration/_api_orchestration_validate.py index 7b4e9c0da9..59202d2f02 100644 --- a/src/autoskillit/recipe/api_orchestration/_api_orchestration_validate.py +++ b/src/autoskillit/recipe/api_orchestration/_api_orchestration_validate.py @@ -34,7 +34,10 @@ _LoadPipelineInputs, _ValidationResult, ) -from autoskillit.recipe.helpers._rule_helpers import filter_pruning_false_positives +from autoskillit.recipe.helpers._rule_helpers import ( + filter_pruning_false_positives, + retain_declaration_findings, +) from autoskillit.recipe.ingredients._recipe_composition import ( _analysis_edges_from_effective_routes, _assert_content_integrity, @@ -50,7 +53,6 @@ ) from autoskillit.recipe.ingredients._recipe_raw_repair import _resolve_skip_guards_in_content from autoskillit.recipe.registry import RuleFinding as _RuleFinding -from autoskillit.recipe.registry import declaration_scoped_rule_names from autoskillit.recipe.rules.rules_plan_set_gate import STAGE_OBLIGATION_GATE_DOMINANCE_RULE from autoskillit.recipe.schema import Recipe, RecipeStep @@ -66,8 +68,6 @@ def _reconcile_semantic_findings( suppressed: _Sequence[str] | None, ) -> tuple[list[_RuleFinding], list[dict[str, Any]]]: """Reconcile pruning diagnostics and suppress displayed version findings.""" - declaration_rules = declaration_scoped_rule_names() - semantic_findings = [f for f in semantic_findings if f.rule not in declaration_rules] if skip_resolutions and any(value is not True for value in skip_resolutions.values()): semantic_findings = filter_pruning_false_positives(semantic_findings, pre_prune_findings) graph_aware_rules = {"capture-inversion-detection", "dead-output"} @@ -85,14 +85,12 @@ def _reconcile_semantic_findings( message=finding.message, ) + semantic_findings = retain_declaration_findings(semantic_findings, pre_prune_findings) retained_pre_prune_findings = [ finding for finding in pre_prune_findings - if finding.rule in declaration_rules - or ( - finding.rule == STAGE_OBLIGATION_GATE_DOMINANCE_RULE - and "cannot declare skip_when_false" in finding.message - ) + if finding.rule == STAGE_OBLIGATION_GATE_DOMINANCE_RULE + and "cannot declare skip_when_false" in finding.message ] retained_keys = { (finding.rule, finding.step_name, finding.message) for finding in semantic_findings diff --git a/src/autoskillit/recipe/helpers/_rule_helpers.py b/src/autoskillit/recipe/helpers/_rule_helpers.py index 95a0a7a654..f632819bdc 100644 --- a/src/autoskillit/recipe/helpers/_rule_helpers.py +++ b/src/autoskillit/recipe/helpers/_rule_helpers.py @@ -12,6 +12,7 @@ from autoskillit.core import get_logger from autoskillit.recipe.analysis._analysis import ValidationContext +from autoskillit.recipe.registry import declaration_scoped_rule_names from autoskillit.recipe.schema import ( extract_sentinel_json_blocks, ) @@ -236,6 +237,17 @@ def _identify_optional_output_fields(contract: SkillContract) -> set[str]: return optional +def retain_declaration_findings( + post_prune_findings: list[RuleFinding], + pre_prune_findings: list[RuleFinding], +) -> list[RuleFinding]: + """Keep declaration diagnostics from the original recipe, including pruned steps.""" + declaration_rules = declaration_scoped_rule_names() + return [f for f in post_prune_findings if f.rule not in declaration_rules] + [ + f for f in pre_prune_findings if f.rule in declaration_rules + ] + + def filter_pruning_false_positives( post_prune_findings: list[RuleFinding], pre_prune_findings: list[RuleFinding], diff --git a/tests/recipe/test_binding_findings_survive_pruning.py b/tests/recipe/test_binding_findings_survive_pruning.py index 374be330fa..3c397e0d04 100644 --- a/tests/recipe/test_binding_findings_survive_pruning.py +++ b/tests/recipe/test_binding_findings_survive_pruning.py @@ -8,6 +8,7 @@ import pytest from autoskillit.recipe._api import load_and_validate +from autoskillit.recipe.api._api_listing import validate_from_path pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] @@ -147,3 +148,15 @@ def test_pruned_step_binding_failure_is_reported(tmp_path: Path) -> None: and suggestion.get("severity") == "error" for suggestion in result["suggestions"] ) + + path_result = validate_from_path( + tmp_path / ".autoskillit/recipes/pruned-binding-failure.yaml", + ingredient_overrides={"flag": "false"}, + ) + assert path_result["valid"] is False + assert any( + finding.get("rule") == "invalid-tool-invocation-binding" + and finding.get("step") == "maybe" + and finding.get("severity") == "error" + for finding in path_result["findings"] + ) From faa2d960e59b16f5acae7f572ddfbdd08a192b87 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sat, 24 Oct 2026 17:41:48 -0700 Subject: [PATCH 04/17] fix(review): make cwd remediation independent of ingredient naming --- src/autoskillit/recipe/rules/rules_tools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/autoskillit/recipe/rules/rules_tools.py b/src/autoskillit/recipe/rules/rules_tools.py index 7805eba89f..66d55ae74b 100644 --- a/src/autoskillit/recipe/rules/rules_tools.py +++ b/src/autoskillit/recipe/rules/rules_tools.py @@ -233,8 +233,8 @@ def _check_invalid_tool_invocation_binding(ctx: ValidationContext) -> list[RuleF message = f"step '{step_name}': tool '{step.tool}' {failure.message}" if failure.code is BindingFailureCode.MISSING_TOOL_PARAMETER and failure.name == "cwd": message += ( - ' — add cwd: "${{ inputs.source_dir }}"' - " (or a context worktree path) under with:" + " — add cwd under with:, bound to the recipe's repository" + " or context worktree path" ) findings.append( make_finding( From a33704f30f69cfec286685baabf6d80b2753a954 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sat, 24 Oct 2026 17:42:44 -0700 Subject: [PATCH 05/17] test(review): share invocation binding contract builders --- tests/_test_filter.py | 2 +- tests/recipe/_binding_helpers.py | 32 +++++++++++++++++++ .../test_binding_failure_reachability.py | 2 +- tests/recipe/test_skill_invocation_binding.py | 32 +------------------ 4 files changed, 35 insertions(+), 33 deletions(-) create mode 100644 tests/recipe/_binding_helpers.py diff --git a/tests/_test_filter.py b/tests/_test_filter.py index 3277e0f69a..f8e23a5295 100644 --- a/tests/_test_filter.py +++ b/tests/_test_filter.py @@ -146,7 +146,7 @@ class SourceMapRejection(enum.StrEnum): # target the scan reaches from it. TEST_HELPER_CASCADE: dict[str, frozenset[str]] = { "tests/_test_filter.py": _TEST_FILTER_VALIDATORS, - "tests/recipe/test_skill_invocation_binding.py": frozenset( + "tests/recipe/_binding_helpers.py": frozenset( { "recipe/test_binding_failure_reachability.py", "recipe/test_skill_invocation_binding.py", diff --git a/tests/recipe/_binding_helpers.py b/tests/recipe/_binding_helpers.py new file mode 100644 index 0000000000..a7ba7e1890 --- /dev/null +++ b/tests/recipe/_binding_helpers.py @@ -0,0 +1,32 @@ +"""Shared contract builders for recipe invocation-binding tests.""" + + +def _manifest() -> dict[str, object]: + return { + "skills": { + "dry-walkthrough": { + "inputs": [ + {"name": "plan_path", "type": "file_path", "required": True}, + {"name": "issue_url", "type": "string", "required": True}, + {"name": "optional_note", "type": "string"}, + {"name": "audit_cycle_path", "type": "file_path", "required": True}, + { + "name": "plan_disposition_path", + "type": "file_path", + "required": True, + }, + {"name": "enabled", "type": "boolean"}, + {"name": "round", "type": "integer"}, + ] + } + } + } + + +def _required_inputs() -> dict[str, str]: + return { + "plan_path": "/tmp/plans/current plan.md", + "issue_url": "https://example.test/issues/42?x=a&y=b", + "audit_cycle_path": "/tmp/audit/cycle.json", + "plan_disposition_path": "/tmp/audit/plan disposition.json", + } diff --git a/tests/recipe/test_binding_failure_reachability.py b/tests/recipe/test_binding_failure_reachability.py index 04205a122a..49926e5969 100644 --- a/tests/recipe/test_binding_failure_reachability.py +++ b/tests/recipe/test_binding_failure_reachability.py @@ -9,7 +9,7 @@ from autoskillit.recipe._binding import bind_recipe from autoskillit.recipe.registry import _RULE_REGISTRY, run_semantic_rules from autoskillit.recipe.schema import Recipe, RecipeStep -from tests.recipe.test_skill_invocation_binding import _manifest, _required_inputs +from tests.recipe._binding_helpers import _manifest, _required_inputs pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] diff --git a/tests/recipe/test_skill_invocation_binding.py b/tests/recipe/test_skill_invocation_binding.py index a411dd6658..9858105fde 100644 --- a/tests/recipe/test_skill_invocation_binding.py +++ b/tests/recipe/test_skill_invocation_binding.py @@ -26,32 +26,11 @@ from autoskillit.recipe._recipe_composition import _prune_skipped_steps from autoskillit.recipe.io import load_recipe from autoskillit.recipe.schema import RecipeStep +from tests.recipe._binding_helpers import _manifest, _required_inputs pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] -def _manifest() -> dict[str, object]: - return { - "skills": { - "dry-walkthrough": { - "inputs": [ - {"name": "plan_path", "type": "file_path", "required": True}, - {"name": "issue_url", "type": "string", "required": True}, - {"name": "optional_note", "type": "string"}, - {"name": "audit_cycle_path", "type": "file_path", "required": True}, - { - "name": "plan_disposition_path", - "type": "file_path", - "required": True, - }, - {"name": "enabled", "type": "boolean"}, - {"name": "round", "type": "integer"}, - ] - } - } - } - - def _step( skill_inputs: dict[str, str | int | bool], *, @@ -71,15 +50,6 @@ def _step( ) -def _required_inputs() -> dict[str, str]: - return { - "plan_path": "/tmp/plans/current plan.md", - "issue_url": "https://example.test/issues/42?x=a&y=b", - "audit_cycle_path": "/tmp/audit/cycle.json", - "plan_disposition_path": "/tmp/audit/plan disposition.json", - } - - @pytest.mark.parametrize( "declared,effective,state,origin", [ From 0cbf7c0bdb630693a18f5818d01323cd1f6640cc Mon Sep 17 00:00:00 2001 From: Trecek Date: Sat, 24 Oct 2026 17:44:08 -0700 Subject: [PATCH 06/17] test(review): share pruning recipe across binding failure probes --- .../test_binding_findings_survive_pruning.py | 115 +++++------------- 1 file changed, 28 insertions(+), 87 deletions(-) diff --git a/tests/recipe/test_binding_findings_survive_pruning.py b/tests/recipe/test_binding_findings_survive_pruning.py index 3c397e0d04..9b66dc0ec2 100644 --- a/tests/recipe/test_binding_findings_survive_pruning.py +++ b/tests/recipe/test_binding_findings_survive_pruning.py @@ -19,13 +19,28 @@ def _install_recipe(tmp_path: Path, name: str, content: str) -> None: (recipes_dir / f"{name}.yaml").write_text(textwrap.dedent(content)) -def test_missing_ingredient_survives_pruned_producer(tmp_path: Path) -> None: +@pytest.mark.parametrize( + ("producer_tool", "producer_args", "rule", "step"), + [ + ( + "run_python", + "{callable: autoskillit.smoke_utils.check_loop_iteration, step_name: maybe}", + "missing-ingredient", + "bem", + ), + ("run_cmd", "{cmd: echo producer}", "invalid-tool-invocation-binding", "maybe"), + ], + ids=["missing-skill-input", "pruned-tool-binding"], +) +def test_binding_findings_survive_pruning( + tmp_path: Path, producer_tool: str, producer_args: str, rule: str, step: str +) -> None: _install_recipe( tmp_path, "probe-prune", """ name: probe-prune - description: Probe whether pruning hides a missing skill input finding. + description: Probe whether pruning hides a declaration binding error. kitchen_rules: - Follow the declared routes. ingredients: @@ -34,10 +49,8 @@ def test_missing_ingredient_survives_pruned_producer(tmp_path: Path) -> None: default: "false" steps: maybe: - tool: run_python - with: - callable: autoskillit.smoke_utils.check_loop_iteration - step_name: maybe + tool: _PRODUCER_TOOL_ + with: _PRODUCER_ARGS_ skip_when_false: inputs.flag capture: issue_numbers: "${{ result.next_iteration }}" @@ -68,7 +81,7 @@ def test_missing_ingredient_survives_pruned_producer(tmp_path: Path) -> None: done: action: stop message: Emit the L3 sentinel block with success=true and reason=Completed. - """, + """.replace("_PRODUCER_TOOL_", producer_tool).replace("_PRODUCER_ARGS_", producer_args), ) result = load_and_validate( @@ -76,87 +89,15 @@ def test_missing_ingredient_survives_pruned_producer(tmp_path: Path) -> None: ) assert result["valid"] is False - assert any( - suggestion.get("rule") == "missing-ingredient" - and suggestion.get("step") == "bem" - and suggestion.get("severity") == "error" - for suggestion in result["suggestions"] - ) - - -def test_pruned_step_binding_failure_is_reported(tmp_path: Path) -> None: - _install_recipe( - tmp_path, - "pruned-binding-failure", - """ - name: pruned-binding-failure - description: Probe whether pruning hides a tool binding failure. - kitchen_rules: - - Follow the declared routes. - ingredients: - flag: - description: Whether to run the producer step. - default: "false" - steps: - maybe: - tool: run_cmd - with: - cmd: "echo producer" - skip_when_false: inputs.flag - capture: - issue_numbers: "${{ result.next_iteration }}" - on_success: bem - on_failure: done - on_skip: bem - bem: - tool: run_skill - with: - skill_command: "/autoskillit:build-execution-map" - skill_inputs: - base_ref: "main" - cwd: "/tmp" - step_name: bem - output_dir: "." - capture: - execution_map: "${{ result.execution_map }}" - on_success: emit - on_failure: done - emit: - tool: run_cmd - with: - cmd: "echo 'execution_map=${{ context.execution_map }}'" - cwd: "/tmp" - step_name: emit - on_success: done - on_failure: done - done: - action: stop - message: Emit the L3 sentinel block with success=true and reason=Completed. - """, - ) - - result = load_and_validate( - "pruned-binding-failure", - project_dir=tmp_path, - ingredient_overrides={"flag": "false"}, - ) - - assert result["valid"] is False - assert any( - suggestion.get("rule") == "invalid-tool-invocation-binding" - and suggestion.get("step") == "maybe" - and suggestion.get("severity") == "error" - for suggestion in result["suggestions"] - ) - path_result = validate_from_path( - tmp_path / ".autoskillit/recipes/pruned-binding-failure.yaml", + tmp_path / ".autoskillit/recipes/probe-prune.yaml", ingredient_overrides={"flag": "false"}, ) assert path_result["valid"] is False - assert any( - finding.get("rule") == "invalid-tool-invocation-binding" - and finding.get("step") == "maybe" - and finding.get("severity") == "error" - for finding in path_result["findings"] - ) + for findings in (result["suggestions"], path_result["findings"]): + assert any( + finding.get("rule") == rule + and finding.get("step") == step + and finding.get("severity") == "error" + for finding in findings + ) From a7d500ecbec1349893f8d0268799555fc88d0e39 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sat, 24 Oct 2026 17:44:44 -0700 Subject: [PATCH 07/17] test(review): require the exact declaration-scoped rule set --- tests/recipe/test_binding_failure_reachability.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/recipe/test_binding_failure_reachability.py b/tests/recipe/test_binding_failure_reachability.py index 49926e5969..a21503e62f 100644 --- a/tests/recipe/test_binding_failure_reachability.py +++ b/tests/recipe/test_binding_failure_reachability.py @@ -7,7 +7,11 @@ from autoskillit.core import BindingFailureCode, Severity from autoskillit.recipe import make_validation_context from autoskillit.recipe._binding import bind_recipe -from autoskillit.recipe.registry import _RULE_REGISTRY, run_semantic_rules +from autoskillit.recipe.registry import ( + _RULE_REGISTRY, + declaration_scoped_rule_names, + run_semantic_rules, +) from autoskillit.recipe.schema import Recipe, RecipeStep from tests.recipe._binding_helpers import _manifest, _required_inputs @@ -154,6 +158,7 @@ def test_binding_reporters_are_error_and_declaration_scoped() -> None: "unresolved-skill-input-template", } rules = {rule.name: rule for rule in _RULE_REGISTRY} + assert declaration_scoped_rule_names() == expected_rules for name in expected_rules: rule = rules[name] From 7d7705f3fd28f9c4b6e9209d08058b4bd0706b77 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sat, 24 Oct 2026 17:45:21 -0700 Subject: [PATCH 08/17] test(review): cover valid tool invocation binding diagnostics --- .../test_binding_failure_reachability.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/recipe/test_binding_failure_reachability.py b/tests/recipe/test_binding_failure_reachability.py index a21503e62f..7bc16b18f2 100644 --- a/tests/recipe/test_binding_failure_reachability.py +++ b/tests/recipe/test_binding_failure_reachability.py @@ -152,6 +152,25 @@ def test_code_reaches_expected_rule(code: BindingFailureCode) -> None: assert _EXPECTED_PARAMETER[code] in finding.message +def test_complete_tool_invocation_has_no_binding_error() -> None: + recipe = Recipe( + name="r", + description="d", + steps={ + "s": RecipeStep( + name="s", + tool="commit_files", + with_args={"paths": ["plan.md"], "message": "Save plan", "cwd": "/r"}, + ) + }, + ) + projection = bind_recipe(recipe, manifest=_manifest()) + assert projection.for_step("s").is_valid + + findings = run_semantic_rules(make_validation_context(recipe, binding_projection=projection)) + assert not any(finding.rule == "invalid-tool-invocation-binding" for finding in findings) + + def test_binding_reporters_are_error_and_declaration_scoped() -> None: expected_rules = set(_EXPECTED_RULE.values()) | { "step-identity-mismatch", From 605dfbb0e10ed948c2850d9321f39681186aa50a Mon Sep 17 00:00:00 2001 From: Trecek Date: Sat, 24 Oct 2026 17:46:00 -0700 Subject: [PATCH 09/17] test(review): pin successful bem-wrapper run_bem admission --- tests/server/test_attested_admission_replay.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/server/test_attested_admission_replay.py b/tests/server/test_attested_admission_replay.py index 169afa90ef..e21fb65bbc 100644 --- a/tests/server/test_attested_admission_replay.py +++ b/tests/server/test_attested_admission_replay.py @@ -39,7 +39,11 @@ pytestmark = [pytest.mark.layer("server"), pytest.mark.medium] REPO_ROOT = Path(__file__).resolve().parents[2] -_PINNED: dict[tuple[str, str], Any] = {} +_PINNED: dict[tuple[str, str], Any] = { + ("bem-wrapper", "run_bem"): pytest.param( + "bem-wrapper", "run_bem", None, None, id="bem-wrapper-run_bem" + ), +} @cache From 1f4f5bb70b291017b60682c646c77eaa26c8bd33 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sat, 24 Oct 2026 18:33:12 -0700 Subject: [PATCH 10/17] test(review): compare binding wire types with live MCP schemas --- tests/server/test_tool_registry_parity.py | 33 +++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/server/test_tool_registry_parity.py b/tests/server/test_tool_registry_parity.py index a236dd02b0..8ecb6a83fc 100644 --- a/tests/server/test_tool_registry_parity.py +++ b/tests/server/test_tool_registry_parity.py @@ -3,10 +3,12 @@ from __future__ import annotations import ast +from collections.abc import Callable from dataclasses import replace from pathlib import Path import pytest +from fastmcp.tools import FunctionTool import autoskillit.core.tool_registry as tool_registry from autoskillit.core import ( @@ -18,6 +20,13 @@ ToolWireType, compute_tool_contract_identity, ) +from autoskillit.server.tools import ( + tools_issue_headless, + tools_kitchen, + tools_pipeline_tracker, + tools_status, + tools_workspace, +) pytestmark = [pytest.mark.layer("server"), pytest.mark.small] @@ -254,6 +263,30 @@ def test_registry_preserves_typed_handler_wire_contracts() -> None: assert param.wire_type is wire_type +@pytest.mark.parametrize( + ("handler", "parameter_names"), + [ + (tools_workspace.commit_files, ("paths",)), + (tools_issue_headless.prepare_issue, ("labels",)), + (tools_kitchen.lock_ingredients, ("locked", "unlock")), + (tools_pipeline_tracker.record_pipeline_step, ("dependencies",)), + (tools_status.read_db, ("params",)), + ], + ids=lambda value: value.__name__ if callable(value) else None, +) +def test_binding_wire_types_match_handler_schemas( + handler: Callable[..., object], parameter_names: tuple[str, ...] +) -> None: + properties = FunctionTool.from_function(handler).parameters["properties"] + registry_params = {param.name: param for param in TOOL_REGISTRY[handler.__name__].params} + + for name in parameter_names: + schema = properties[name] + variants = schema.get("anyOf", [schema]) + wire_types = {variant["type"] for variant in variants} - {"null"} + assert wire_types == {registry_params[name].wire_type.value}, f"{handler.__name__}.{name}" + + def test_evidence_reader_registry_parameter_contracts_are_exact() -> None: expected = { "delegate_evidence_reader": (("role", True), ("role_data", True)), From 3569a97c1ab2aeb38f94fa61cccb7262183d716b Mon Sep 17 00:00:00 2001 From: Trecek Date: Sat, 24 Oct 2026 19:36:28 -0700 Subject: [PATCH 11/17] fix(review): share the tool binding rule name --- src/autoskillit/recipe/rules/rules_tools.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/autoskillit/recipe/rules/rules_tools.py b/src/autoskillit/recipe/rules/rules_tools.py index 66d55ae74b..0fb06e48fc 100644 --- a/src/autoskillit/recipe/rules/rules_tools.py +++ b/src/autoskillit/recipe/rules/rules_tools.py @@ -27,6 +27,7 @@ logger = get_logger(__name__) _ALL_TOOLS: frozenset[str] = frozenset(TOOL_REGISTRY) +INVALID_TOOL_INVOCATION_BINDING_RULE = "invalid-tool-invocation-binding" STEP_IDENTITY_MISMATCH_RULE = "step-identity-mismatch" # Registry of context variables that are captured upstream and MUST be forwarded @@ -207,7 +208,7 @@ def _check_dead_with_params(ctx: ValidationContext) -> list[RuleFinding]: @semantic_rule( - name="invalid-tool-invocation-binding", + name=INVALID_TOOL_INVOCATION_BINDING_RULE, description=( "Recipe tool steps must supply every required tool parameter with a value " "of the registered wire type." @@ -238,7 +239,7 @@ def _check_invalid_tool_invocation_binding(ctx: ValidationContext) -> list[RuleF ) findings.append( make_finding( - rule_name="invalid-tool-invocation-binding", + rule_name=INVALID_TOOL_INVOCATION_BINDING_RULE, step_name=step_name, message=message, ) From 549cff508910ba2ce271b9f1cc953ecbdc290c4f Mon Sep 17 00:00:00 2001 From: Trecek Date: Sat, 24 Oct 2026 19:37:42 -0700 Subject: [PATCH 12/17] refactor(review): reuse the fixed tool binding failure codes --- src/autoskillit/recipe/rules/rules_tools.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/autoskillit/recipe/rules/rules_tools.py b/src/autoskillit/recipe/rules/rules_tools.py index 0fb06e48fc..550d6c1ec5 100644 --- a/src/autoskillit/recipe/rules/rules_tools.py +++ b/src/autoskillit/recipe/rules/rules_tools.py @@ -28,6 +28,12 @@ _ALL_TOOLS: frozenset[str] = frozenset(TOOL_REGISTRY) INVALID_TOOL_INVOCATION_BINDING_RULE = "invalid-tool-invocation-binding" +_TOOL_INVOCATION_BINDING_FAILURE_CODES = frozenset( + { + BindingFailureCode.MISSING_TOOL_PARAMETER, + BindingFailureCode.INVALID_TOOL_PARAMETER_TYPE, + } +) STEP_IDENTITY_MISMATCH_RULE = "step-identity-mismatch" # Registry of context variables that are captured upstream and MUST be forwarded @@ -218,10 +224,6 @@ def _check_dead_with_params(ctx: ValidationContext) -> list[RuleFinding]: ) def _check_invalid_tool_invocation_binding(ctx: ValidationContext) -> list[RuleFinding]: findings: list[RuleFinding] = [] - reported_codes = { - BindingFailureCode.MISSING_TOOL_PARAMETER, - BindingFailureCode.INVALID_TOOL_PARAMETER_TYPE, - } for step_name, step in ctx.recipe.steps.items(): if step.tool is None: continue @@ -229,7 +231,7 @@ def _check_invalid_tool_invocation_binding(ctx: ValidationContext) -> list[RuleF if invocation is None: continue for failure in invocation.failures: - if failure.code not in reported_codes: + if failure.code not in _TOOL_INVOCATION_BINDING_FAILURE_CODES: continue message = f"step '{step_name}': tool '{step.tool}' {failure.message}" if failure.code is BindingFailureCode.MISSING_TOOL_PARAMETER and failure.name == "cwd": From 501fa36d874198cb5f2eef3446421d8412b69952 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sat, 24 Oct 2026 19:38:38 -0700 Subject: [PATCH 13/17] docs(review): declare the migration cwd ingredient requirement --- src/autoskillit/migrations/0.0.0-to-0.10.1293.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/autoskillit/migrations/0.0.0-to-0.10.1293.yaml b/src/autoskillit/migrations/0.0.0-to-0.10.1293.yaml index 355746df9c..a1d591f13c 100644 --- a/src/autoskillit/migrations/0.0.0-to-0.10.1293.yaml +++ b/src/autoskillit/migrations/0.0.0-to-0.10.1293.yaml @@ -13,7 +13,10 @@ changes: instruction: >- Write integer and boolean literals without quotes. Add cwd (usually "${{ inputs.source_dir }}") under with: for every run_skill and run_cmd step, - and callable for every run_python step. Supply actual lists or mappings for + and callable for every run_python step. When using inputs.source_dir, declare + source_dir as a required ingredient for the repository root path; alternatively, + bind cwd to another declared input containing the intended worktree path. + Supply actual lists or mappings for ARRAY and OBJECT parameters, then run validate_recipe. example_before: | run: From 468e63bf7d236c3a04b7d6655665a110454874b2 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sat, 24 Oct 2026 19:39:27 -0700 Subject: [PATCH 14/17] docs(review): simplify the bem wrapper source path description --- src/autoskillit/recipes/bem-wrapper.json | 2 +- src/autoskillit/recipes/bem-wrapper.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/autoskillit/recipes/bem-wrapper.json b/src/autoskillit/recipes/bem-wrapper.json index fd6aeca505..4f9b08dfad 100644 --- a/src/autoskillit/recipes/bem-wrapper.json +++ b/src/autoskillit/recipes/bem-wrapper.json @@ -22,7 +22,7 @@ "authority": "config" }, "source_dir": { - "description": "Absolute path to the repository root. Required — no default prevents silent cwd inheritance from the MCP server when an empty string is passed.", + "description": "Absolute path to the repository root", "required": true } }, diff --git a/src/autoskillit/recipes/bem-wrapper.yaml b/src/autoskillit/recipes/bem-wrapper.yaml index d1e1e486ea..e7139802e7 100644 --- a/src/autoskillit/recipes/bem-wrapper.yaml +++ b/src/autoskillit/recipes/bem-wrapper.yaml @@ -26,7 +26,7 @@ ingredients: default: "develop" authority: config source_dir: - description: "Absolute path to the repository root. Required — no default prevents silent cwd inheritance from the MCP server when an empty string is passed." + description: "Absolute path to the repository root" required: true steps: From 68e0cde13bc869cef65c3232592994ca9f4cb699 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sat, 24 Oct 2026 19:40:43 -0700 Subject: [PATCH 15/17] test(review): match every binding failure to its diagnostic --- .../recipe/test_binding_failure_reachability.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/recipe/test_binding_failure_reachability.py b/tests/recipe/test_binding_failure_reachability.py index 7bc16b18f2..20efc3abb3 100644 --- a/tests/recipe/test_binding_failure_reachability.py +++ b/tests/recipe/test_binding_failure_reachability.py @@ -105,10 +105,18 @@ BindingFailureCode.INVALID_SKILL_COMMAND: "invalid-skill-invocation-binding", } -_EXPECTED_PARAMETER = { +_EXPECTED_MESSAGE = { + BindingFailureCode.UNKNOWN_TOOL: "no_such_tool", BindingFailureCode.UNKNOWN_TOOL_PARAMETER: "bogus", BindingFailureCode.MISSING_TOOL_PARAMETER: "cwd", BindingFailureCode.INVALID_TOOL_PARAMETER_TYPE: "issue_numbers", + BindingFailureCode.UNKNOWN_SKILL: "selected skill has no canonical contract", + BindingFailureCode.UNKNOWN_SKILL_INPUT: "unknown_child", + BindingFailureCode.MISSING_SKILL_INPUT: "plan_path", + BindingFailureCode.AMBIGUOUS_SKILL_INPUT: "may not mix inline arguments", + BindingFailureCode.DEAD_SKILL_INPUT: "more positional values than the contract", + BindingFailureCode.INVALID_SKILL_INPUT_TYPE: "round", + BindingFailureCode.INVALID_SKILL_COMMAND: "effective skill arguments do not align", } @@ -125,7 +133,9 @@ def _recipe_for(code: BindingFailureCode) -> Recipe: def test_tables_cover_every_code() -> None: assert len(BindingFailureCode.__members__) == len(BindingFailureCode) - assert set(_TRIGGERS) == set(_EXPECTED_RULE) == set(BindingFailureCode) + assert ( + set(_TRIGGERS) == set(_EXPECTED_RULE) == set(_EXPECTED_MESSAGE) == set(BindingFailureCode) + ) @pytest.mark.parametrize("code", list(_TRIGGERS), ids=lambda code: code.name.lower()) @@ -144,12 +154,11 @@ def test_code_reaches_expected_rule(code: BindingFailureCode) -> None: if candidate.rule == _EXPECTED_RULE[code] and candidate.step_name == "s" and candidate.severity is Severity.ERROR + and _EXPECTED_MESSAGE[code] in candidate.message ), None, ) assert finding is not None - if code in _EXPECTED_PARAMETER: - assert _EXPECTED_PARAMETER[code] in finding.message def test_complete_tool_invocation_has_no_binding_error() -> None: From 98c65b05b33d9c71a5b84e28831f7b57204bf543 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sat, 24 Oct 2026 19:41:21 -0700 Subject: [PATCH 16/17] test(review): identify divergent handler schema parameters --- tests/server/test_tool_registry_parity.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/server/test_tool_registry_parity.py b/tests/server/test_tool_registry_parity.py index 8ecb6a83fc..820e74434f 100644 --- a/tests/server/test_tool_registry_parity.py +++ b/tests/server/test_tool_registry_parity.py @@ -281,10 +281,13 @@ def test_binding_wire_types_match_handler_schemas( registry_params = {param.name: param for param in TOOL_REGISTRY[handler.__name__].params} for name in parameter_names: - schema = properties[name] + schema = properties.get(name) + assert schema is not None, f"{handler.__name__}.{name}: handler parameter missing" + registry_param = registry_params.get(name) + assert registry_param is not None, f"{handler.__name__}.{name}: registry parameter missing" variants = schema.get("anyOf", [schema]) - wire_types = {variant["type"] for variant in variants} - {"null"} - assert wire_types == {registry_params[name].wire_type.value}, f"{handler.__name__}.{name}" + wire_types = {variant.get("type") for variant in variants} - {"null"} + assert wire_types == {registry_param.wire_type.value}, f"{handler.__name__}.{name}" def test_evidence_reader_registry_parameter_contracts_are_exact() -> None: From 845d3f029107d5cfa2d3b8fb8d79e90e9b01b72e Mon Sep 17 00:00:00 2001 From: Trecek Date: Sat, 24 Oct 2026 20:31:41 -0700 Subject: [PATCH 17/17] test: cover invalid dispatch items in binding failure reachability --- tests/recipe/test_binding_failure_reachability.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/recipe/test_binding_failure_reachability.py b/tests/recipe/test_binding_failure_reachability.py index 20efc3abb3..d21c8298cb 100644 --- a/tests/recipe/test_binding_failure_reachability.py +++ b/tests/recipe/test_binding_failure_reachability.py @@ -89,6 +89,15 @@ "cwd": "/r", }, ), + BindingFailureCode.INVALID_DISPATCH_ITEMS: ( + "run_skill", + { + "skill_command": _DRY_WALKTHROUGH, + "cwd": "/r", + "skill_inputs": dict(_REQUIRED_INPUTS), + "dispatch_items": "literal_items", + }, + ), } _EXPECTED_RULE: dict[BindingFailureCode, str] = { @@ -103,6 +112,7 @@ BindingFailureCode.DEAD_SKILL_INPUT: "invalid-skill-invocation-binding", BindingFailureCode.INVALID_SKILL_INPUT_TYPE: "invalid-skill-invocation-binding", BindingFailureCode.INVALID_SKILL_COMMAND: "invalid-skill-invocation-binding", + BindingFailureCode.INVALID_DISPATCH_ITEMS: "invalid-skill-invocation-binding", } _EXPECTED_MESSAGE = { @@ -117,6 +127,7 @@ BindingFailureCode.DEAD_SKILL_INPUT: "more positional values than the contract", BindingFailureCode.INVALID_SKILL_INPUT_TYPE: "round", BindingFailureCode.INVALID_SKILL_COMMAND: "effective skill arguments do not align", + BindingFailureCode.INVALID_DISPATCH_ITEMS: "dispatch_items producer", }