diff --git a/src/autoskillit/core/_tool_registry_builders.py b/src/autoskillit/core/_tool_registry_builders.py index 239063eda2..0a289af0a4 100644 --- a/src/autoskillit/core/_tool_registry_builders.py +++ b/src/autoskillit/core/_tool_registry_builders.py @@ -306,11 +306,10 @@ def _verify_review_receipt() -> ToolDef: ) -# The single classification authority for what each run_skill parameter is -# for. Every attestation-relevant surface (the runtime gate's always-admit -# set, the actual-kwargs assembly, the execution-tuning fallback table, and -# the frozen ledger in tests/contracts/test_run_skill_kwarg_ledger.py) is -# derived from this mapping — see ToolParamRole for what each role means. +# Classification authority for run_skill runtime admission and forwarding prose. +# The gate's always-admit set, execution-tuning fallback table, and frozen ledger +# derive from this mapping; kwarg membership is ToolDef.mcp_kwarg_names. +# See ToolParamRole for what each role means. _RUN_SKILL_PARAM_ROLES: Mapping[str, ToolParamRole] = MappingProxyType( { "skill_command": ToolParamRole.CHILD_INPUT, diff --git a/src/autoskillit/core/types/recipe/_type_recipe_binding.py b/src/autoskillit/core/types/recipe/_type_recipe_binding.py index 9fd98e6648..988e1249ef 100644 --- a/src/autoskillit/core/types/recipe/_type_recipe_binding.py +++ b/src/autoskillit/core/types/recipe/_type_recipe_binding.py @@ -90,11 +90,11 @@ class ToolInitializationOperation(StrEnum): class ToolParamRole(StrEnum): - """The single classification authority for what a tool parameter is for. + """The classification authority for what a tool parameter is for. - The runtime attestation gate's always-admit set, the actual-kwargs - assembly, and the server-side ``RecipeStep`` fallback obligation for - execution-tuning parameters are all derived from this field. The + The runtime gate's always-admit set and ``RecipeStep`` execution-tuning + fallback obligations derive from this field; kwarg membership comes + from ``ToolDef.mcp_kwarg_names``. The per-role runtime-admission *policy* is declared once, alongside this enum, in ``RUNTIME_ADMISSION_BY_ROLE`` — see ``runtime_exempt_param_names`` (core/tool_registry.py) and ``bind_runtime_skill_invocation`` @@ -152,7 +152,11 @@ def _assert_admission_policy_total( @dataclass(frozen=True, slots=True) class ToolParamDef: - """Static definition of one public MCP handler parameter.""" + """Static definition of one recipe-authorable tool parameter. + + ``handler_parameter=False`` denotes an orchestrator directive never sent + to the handler; ``structured_skill_inputs`` uses the separate child-input channel. + """ name: str wire_type: ToolWireType = ToolWireType.SCALAR @@ -255,6 +259,13 @@ def param_set(self) -> frozenset[str]: def handler_param_set(self) -> frozenset[str]: return frozenset(param.name for param in self.params if param.handler_parameter) + @property + def mcp_kwarg_names(self) -> frozenset[str]: + """Sole membership authority for compiled and runtime ``mcp_kwargs``.""" + return self.handler_param_set - frozenset( + param.name for param in self.params if param.structured_skill_inputs + ) + def param_def(self, name: str) -> ToolParamDef | None: return next((param for param in self.params if param.name == name), None) diff --git a/src/autoskillit/recipe/_binding.py b/src/autoskillit/recipe/_binding.py index b0e762e0dc..aaf43e9e07 100644 --- a/src/autoskillit/recipe/_binding.py +++ b/src/autoskillit/recipe/_binding.py @@ -174,7 +174,7 @@ def _compile_tool_parameters( hidden_inputs: frozenset[str], ingredient_values: Mapping[str, BoundScalar] | None, ) -> tuple[list[BoundValue], list[BindingFailure]]: - """Compile declared tool parameters in canonical registry order.""" + """Compile declared tool parameters in registry order; return only handler kwargs.""" failures: list[BindingFailure] = [] undeclared_effective_params = frozenset(effective_with) - frozenset(declared_with) for name in sorted(undeclared_effective_params): @@ -295,7 +295,7 @@ def _compile_tool_parameters( f"tool parameter {param.name!r} expects {param.wire_type.value!r}", ) ) - return mcp_kwargs, failures + return [value for value in mcp_kwargs if value.name in tool_def.mcp_kwarg_names], failures def _bind_run_skill_inputs( diff --git a/src/autoskillit/server/tools/tools_execution/_run_skill_admission.py b/src/autoskillit/server/tools/tools_execution/_run_skill_admission.py index cfa58664e0..9073bb4e9a 100644 --- a/src/autoskillit/server/tools/tools_execution/_run_skill_admission.py +++ b/src/autoskillit/server/tools/tools_execution/_run_skill_admission.py @@ -187,39 +187,32 @@ def _build_actual_mcp_kwargs( ) -> dict[str, BoundScalar]: """Assemble the actual runtime kwargs the attestation gate checks. - Iterates ``tool_def``'s handler params only — a param declared - ``handler_parameter=False`` (e.g. ``dispatch_items``) or - ``structured_skill_inputs=True`` (``skill_inputs``, which has its own - dedicated channel) is excluded — and pulls each remaining param's - runtime value from ``values``, keyed by param name, which the caller - builds from the handler's locals. + Uses ``tool_def.mcp_kwarg_names`` and pulls each member's runtime value + from ``values``, keyed by param name, which the caller builds from the + handler's locals. A param missing from ``values``, or a ``values`` key absent from the - filtered param set, is a loud construction-time bug: adding a + MCP kwarg set, is a loud construction-time bug: adding a ``ToolParamDef`` without wiring its runtime value (or vice versa) fails the first attested call instead of silently drifting. ``None`` values are omitted — the vacancy sentinel for optional params such as ``stale_threshold``/``idle_output_timeout`` (subsumes the prior hand-typed ``if ... is not None`` special case). """ - handler_param_names = frozenset( - param.name - for param in tool_def.params - if param.handler_parameter and not param.structured_skill_inputs - ) - missing = handler_param_names - values.keys() + mcp_kwarg_names = tool_def.mcp_kwarg_names + missing = mcp_kwarg_names - values.keys() if missing: raise ValueError( f"{tool_def.name}: no runtime value supplied for handler params: {sorted(missing)!r}" ) - unknown = values.keys() - handler_param_names + unknown = values.keys() - mcp_kwarg_names if unknown: raise ValueError( f"{tool_def.name}: runtime values supplied for unknown handler params: " f"{sorted(unknown)!r}" ) result: dict[str, BoundScalar] = {} - for name in handler_param_names: + for name in mcp_kwarg_names: value = values[name] if value is not None: result[name] = value diff --git a/src/autoskillit/skill_resources/arch-constraint-catalog.md b/src/autoskillit/skill_resources/arch-constraint-catalog.md index f3e01d15d7..0c22361b63 100644 --- a/src/autoskillit/skill_resources/arch-constraint-catalog.md +++ b/src/autoskillit/skill_resources/arch-constraint-catalog.md @@ -73,7 +73,7 @@ classified `REJECT` with `category: "arch_violation"`. | Env-var-set constant consumption | `test_canonical_constant_consumption.py` | `*_ENV_FORWARD_VARS` or `*_REQUIRED_ENV` constant with zero production importers — every canonical env-var-set must be consumed | | MCP env forward coverage | `test_mcp_env_forward_coverage.py` | `mcp_env_forward_vars` values missing from `CmdSpec.env` in any cmd-builder (skill, food-truck, headless, resume, interactive) | | Rule severity consistency | `test_rule_severity_consistency.py` | Direct `RuleFinding()` construction in `@semantic_rule`/`@block_rule` bodies — must use `make_finding()`/`make_block_finding()`; `_KNOWN_NON_CONFORMING_RULES` entries without `# tracking: #NNNN` comments | -| Xfail bridge policy | `test_xfail_bridge_policy.py` | `xfail(strict=True)` decorators and `pytest.param` marks whose `reason` string does not cite a `#NNNN` tracking issue; `_XFAIL_POLICY_EXEMPT_FILES` entries without `# permanent:` or `# tracking:` rationale comments | +| Xfail bridge policy | `tests/arch/test_xfail_bridge_policy.py`; `tests/server/test_attested_admission_replay.py` (`_assert_admitted`) | `xfail(strict=True)` decorators and `pytest.param` marks must cite a `#NNNN` tracking issue and declare a specific `raises=` exception type (excluding `Exception`, `BaseException`, and `AssertionError`, including tuple members and `builtins`-qualified forms); class-valued matching includes subclasses, while `_assert_admitted` checks the exact rejection code and a pinned detail substring; `_XFAIL_POLICY_EXEMPT_FILES` entries need `# permanent:` or `# tracking:` rationale comments | | No direct swap_labels in fleet | `test_swap_labels_guard.py` | Direct `swap_labels` calls in `fleet/` outside `_label_cleanup.py` — must route through `cleanup_orphaned_labels` | | Issue URL extraction guard | `test_issue_url_extraction_guard.py` | Raw `.get("issue_url")` / `.get("issue_urls")` in `fleet/` outside `_issue_url_helpers.py` and `state_records.py` — must use `extract_issue_urls()`; `fleet_claim_guard.py` must retain both key variants | | Pipeline ordering | `test_pipeline_ordering.py` | Moving `run_semantic_rules` before `_prune_skipped_steps` in `load_and_validate` — semantic rules must run on post-prune recipe | diff --git a/tests/_run_skill_caller_oracle.py b/tests/_run_skill_caller_oracle.py new file mode 100644 index 0000000000..d55b1a1009 --- /dev/null +++ b/tests/_run_skill_caller_oracle.py @@ -0,0 +1,233 @@ +"""Caller payload oracle and admission assertion protocol for ``run_skill`` tests. + +Models the delivered ``with:`` block, the ``recipe_execution`` credential, +and the sous-chef's forwarding rules. It must never derive actual values from +the compiled template. Replay xfails use ``raises=AdmissionDefect`` to match +the class and its subclasses; the marker alone does not check a rejection's +code or detail. ``tests.server.test_attested_admission_replay._assert_admitted`` +checks the exact code and pinned detail substring before raising +``AdmissionDefect``; other failures remain plain ``AssertionError`` instances. +""" + +from __future__ import annotations + +import re +from collections.abc import Mapping +from inspect import Parameter, signature +from typing import cast + +from autoskillit.core import BoundScalar +from autoskillit.server.tools import tools_execution + + +class AdmissionDefect(AssertionError): + """Raised only for a rejection matching both the pinned code and detail.""" + + +ORCHESTRATOR_SAMPLES: Mapping[str, Mapping[str, BoundScalar]] = { + "context": {"pr_number": 7, "review_pr_number": 7}, + "inputs": {"pr_number": 7, "review_pr_number": 7}, + "item": { + "phase_id": "P1", + "context_path": ".autoskillit/temp/planner/run-test/assignments/ctx_P1.json", + "pr_number": "7", + "complexity": "simple", + }, +} + +_CONTEXT_INPUT_REF = re.compile(r"\$\{\{\s*(context|inputs)\.([^{}\s.]+)\s*\}\}") +_SINGLE_BRACE_TOKEN = re.compile(r"(? BoundScalar: + """Return a pinned sample or a stable path for an otherwise unknown name.""" + values = samples.get(namespace) + if values is not None and name in values: + return values[name] + return f".autoskillit/temp/sample/{name}" + + +def _substitute_scalar( + value: BoundScalar, samples: Mapping[str, Mapping[str, BoundScalar]] +) -> BoundScalar: + """Apply supported typed refs, then isolated item tokens, to one scalar.""" + if not isinstance(value, str): + return value + + exact_ref = _CONTEXT_INPUT_REF.fullmatch(value) + if exact_ref is not None: + return _sample_for(samples, exact_ref.group(1), exact_ref.group(2)) + + def replace_context_input(match: re.Match[str]) -> str: + return str(_sample_for(samples, match.group(1), match.group(2))) + + substituted = _CONTEXT_INPUT_REF.sub(replace_context_input, value) + + def replace_item(match: re.Match[str]) -> str: + return str(_sample_for(samples, "item", match.group(1).strip())) + + return _SINGLE_BRACE_TOKEN.sub(replace_item, substituted) + + +def _declared_skill_inputs(value: object) -> dict[str, BoundScalar]: + """Validate the scalar map declared by ``with.skill_inputs``.""" + if not isinstance(value, Mapping): + raise TypeError("run_skill skill_inputs must be a mapping") + declared: dict[str, BoundScalar] = {} + for key, raw_value in value.items(): + if not isinstance(key, str): + raise TypeError("run_skill skill_inputs keys must be strings") + if not isinstance(raw_value, (str, int, bool)): + raise TypeError(f"run_skill skill_inputs value for {key!r} must be a scalar") + declared[key] = raw_value + return declared + + +def _skill_input_shape( + step_key: str, credential: Mapping[str, object] +) -> tuple[list[str], Mapping[str, object]]: + """Read ordered input keys and absence values from the caller credential.""" + shapes_value = credential.get("skill_input_shapes") + if not isinstance(shapes_value, Mapping): + raise TypeError("recipe_execution skill_input_shapes must be a mapping") + shape_value = shapes_value.get(step_key) + if not isinstance(shape_value, Mapping): + raise AssertionError(f"step {step_key!r} has no skill input shape") + keys_value = shape_value.get("keys") + absence_values = shape_value.get("absence_values") + if not isinstance(keys_value, list): + raise TypeError(f"step {step_key!r} skill input shape keys must be strings") + if not isinstance(absence_values, Mapping): + raise TypeError(f"step {step_key!r} absence_values must be a mapping") + keys: list[str] = [] + for key in keys_value: + if not isinstance(key, str): + raise TypeError(f"step {step_key!r} skill input shape keys must be strings") + keys.append(key) + return keys, absence_values + + +def _absence_value(step_key: str, key: str, absence_values: Mapping[str, object]) -> BoundScalar: + """Return one advertised scalar absence value.""" + value = absence_values[key] + if not isinstance(value, (str, int, bool)): + raise TypeError(f"step {step_key!r} absence value for {key!r} must be scalar") + return value + + +def _skill_input_payload( + step_key: str, + declared_value: object, + credential: Mapping[str, object], + samples: Mapping[str, Mapping[str, BoundScalar]], +) -> dict[str, BoundScalar]: + """Build declared inputs in credential order with advertised absence values.""" + declared = _declared_skill_inputs(declared_value) + keys, absence_values = _skill_input_shape(step_key, credential) + substituted = {key: _substitute_scalar(value, samples) for key, value in declared.items()} + result: dict[str, BoundScalar] = {} + for key in keys: + if key in substituted: + result[key] = substituted[key] + elif key in absence_values: + result[key] = _absence_value(step_key, key, absence_values) + else: + raise AssertionError(f"step {step_key!r} skill input {key!r} has no caller value") + for key, value in substituted.items(): + if key not in result: + result[key] = value + return result + + +def build_run_skill_payload( + step_key: str, + step_body: Mapping[str, object], + credential: Mapping[str, object], + *, + cwd: str, + samples: Mapping[str, Mapping[str, BoundScalar]] = ORCHESTRATOR_SAMPLES, +) -> dict[str, object]: + """Build one caller payload after orchestrator placeholder substitution.""" + payload = literal_run_skill_payload(step_key, step_body, credential, cwd=cwd) + with_values = cast(Mapping[str, object], step_body.get("with", {})) + for name, value in with_values.items(): + if name == "skill_inputs": + payload["skill_inputs"] = _skill_input_payload(step_key, value, credential, samples) + elif name in payload: + if value is None: + payload[name] = None + elif isinstance(value, (str, int, bool)): + payload[name] = _substitute_scalar(value, samples) + payload["cwd"] = cwd + return payload + + +def literal_run_skill_payload( + step_key: str, + step_body: Mapping[str, object], + credential: Mapping[str, object], + *, + cwd: str, +) -> dict[str, object]: + """Build delivered ``with:`` kwargs verbatim, without template substitution.""" + with_value = step_body.get("with", {}) + if not isinstance(with_value, Mapping): + raise TypeError("recipe step 'with' must be a mapping") + with_values: Mapping[str, object] = with_value + + parameters = signature(tools_execution.run_skill).parameters + payload: dict[str, object] = {} + for name, value in with_values.items(): + if name in parameters and name != "skill_inputs": + if value is not None and not isinstance(value, (str, int, bool)): + raise TypeError(f"run_skill payload value {name!r} must be a scalar") + payload[name] = value + + step_name_value = with_values.get("step_name", step_key) + if not isinstance(step_name_value, str): + raise TypeError("run_skill step_name must be a string") + execution_id = credential.get("execution_id") + if not isinstance(execution_id, str): + raise TypeError("recipe_execution execution_id must be a string") + digests_value = credential.get("invocation_template_digests") + if not isinstance(digests_value, Mapping): + raise TypeError("recipe_execution invocation_template_digests must be a mapping") + digest = digests_value.get(step_key) + if not isinstance(digest, str): + raise TypeError("recipe_execution step digest must be a string") + + payload["step_name"] = step_name_value + payload["recipe_execution_id"] = execution_id + payload["invocation_template_digest"] = digest + payload["cwd"] = cwd + return payload + + +def run_skill_actual_mcp_kwargs( + payload: Mapping[str, object], +) -> dict[str, BoundScalar]: + """Project payload values and signature defaults through the real MCP builder.""" + parameters = signature(tools_execution.run_skill).parameters + values: dict[str, BoundScalar | None] = {} + for name, parameter in parameters.items(): + if name in {"ctx", "skill_inputs"}: + continue + if name in payload: + value = payload[name] + elif parameter.default is not Parameter.empty: + value = parameter.default + else: + continue + if value is None: + values[name] = None + elif isinstance(value, (str, int, bool)): + values[name] = value + else: + raise TypeError(f"run_skill argument {name!r} must be a scalar") + + tool_def = tools_execution.get_tool_def("run_skill") + if tool_def is None: + raise RuntimeError("run_skill must be a registered ToolDef") + return tools_execution._build_actual_mcp_kwargs(tool_def, values) diff --git a/tests/_test_filter.py b/tests/_test_filter.py index 31ceff6fb7..bd1ea92c8f 100644 --- a/tests/_test_filter.py +++ b/tests/_test_filter.py @@ -1355,6 +1355,7 @@ class SourceMapRejection(enum.StrEnum): "server/test_serve_idempotence.py", "server/test_attestation_delivery_reachability.py", "server/test_recipe_execution_receipt_completeness.py", + "server/test_attested_admission_replay.py", "server/test_recipe_checkpoint_delivery_regressions.py", "server/test_tools_status_kitchen.py", "server/test_open_kitchen_deferred_recall.py", diff --git a/tests/arch/test_xfail_bridge_policy.py b/tests/arch/test_xfail_bridge_policy.py index 355badde23..2c7cb52319 100644 --- a/tests/arch/test_xfail_bridge_policy.py +++ b/tests/arch/test_xfail_bridge_policy.py @@ -1,8 +1,12 @@ -"""Architectural guard: xfail(strict=True) bridges must cite a tracking issue. +"""Architectural guard: strict xfail bridges must be specific and cite an issue. Every ``pytest.mark.xfail(strict=True, reason=...)`` decorator — both on function definitions and inside ``pytest.param(..., marks=...)`` — must include a ``#NNNN`` -issue reference in its ``reason`` string so bridge exit conditions are trackable. +issue reference in its ``reason`` string and a specific ``raises=`` exception type. +Bare ``Exception``, ``BaseException``, and ``AssertionError`` do not identify the +expected failure precisely. Class-valued ``raises=`` matching includes subclasses; +``tests.server.test_attested_admission_replay._assert_admitted`` checks the exact +rejection code and pinned detail substring. Files in ``_XFAIL_POLICY_EXEMPT_FILES`` are excluded when they have an alternative self-governance mechanism (e.g. a companion meta-test that forces the count monotonically non-increasing). @@ -99,11 +103,30 @@ def _collect_xfail_strict_true_nodes(tree: ast.Module) -> list[ast.Call]: results.extend(_decorator_xfail_nodes(decorator)) if isinstance(node, ast.Call): results.extend(_param_mark_xfail_nodes(node)) - return results + return list({id(node): node for node in results}.values()) + + +def _strict_xfail_raises_issue(node: ast.Call) -> str | None: + raises = next((kw.value for kw in node.keywords if kw.arg == "raises"), None) + if raises is None: + return "requires a specific raises= exception type" + for exception in ast.walk(raises): + name = None + if isinstance(exception, ast.Name): + name = exception.id + elif ( + isinstance(exception, ast.Attribute) + and isinstance(exception.value, ast.Name) + and exception.value.id == "builtins" + ): + name = exception.attr + if name in {"Exception", "BaseException", "AssertionError"}: + return f"raises={name} is too broad" + return None def test_strict_xfail_reasons_cite_tracking_issue() -> None: - """Every xfail(strict=True) reason must contain a #NNNN issue reference.""" + """Strict xfail bridges must name an expected type and cite a tracking issue.""" violations: list[str] = [] for py_file in sorted(_TESTS_ROOT.rglob("test_*.py")): rel = py_file.relative_to(_TESTS_ROOT).as_posix() @@ -114,6 +137,9 @@ def test_strict_xfail_reasons_cite_tracking_issue() -> None: except (SyntaxError, UnicodeDecodeError, OSError): continue for xfail_node in _collect_xfail_strict_true_nodes(tree): + raises_issue = _strict_xfail_raises_issue(xfail_node) + if raises_issue is not None: + violations.append(f"{rel}:{xfail_node.lineno} — {raises_issue}") reason = _extract_reason(xfail_node) if reason is None: violations.append( @@ -125,11 +151,44 @@ def test_strict_xfail_reasons_cite_tracking_issue() -> None: f"{rel}:{xfail_node.lineno} — reason={reason!r} does not cite " "a tracking issue; add #NNNN or resolve the defect and remove the xfail" ) - assert not violations, "xfail(strict=True) without tracking issue reference:\n" + "\n".join( + assert not violations, "strict xfail bridge policy violations:\n" + "\n".join( f" {v}" for v in violations ) +@pytest.mark.parametrize( + ("raises_clause", "expected_issue"), + [ + pytest.param("", True, id="missing"), + pytest.param(", raises=Exception", True, id="exception"), + pytest.param(", raises=BaseException", True, id="base-exception"), + pytest.param(", raises=AssertionError", True, id="assertion-error"), + pytest.param(", raises=(ValueError, Exception)", True, id="broad-tuple"), + pytest.param(", raises=builtins.Exception", True, id="qualified-broad-type"), + pytest.param(", raises=(ValueError, LookupError)", False, id="specific-tuple"), + pytest.param(", raises=builtins.LookupError", False, id="qualified-specific-type"), + pytest.param(", raises=LookupError", False, id="specific-type"), + ], +) +def test_strict_xfail_raises_policy_on_synthetic_source( + raises_clause: str, expected_issue: bool +) -> None: + """Synthetic param marks enforce raises precision and are collected once.""" + tree = ast.parse( + "import pytest\n" + "@pytest.mark.parametrize('value', [\n" + " pytest.param(1, marks=pytest.mark.xfail(\n" + " strict=True, reason='tracked #1234'" + f"{raises_clause})),\n" + "])\n" + "def test_example(value):\n" + " pass\n" + ) + nodes = _collect_xfail_strict_true_nodes(tree) + assert len(nodes) == 1 + assert (_strict_xfail_raises_issue(nodes[0]) is not None) is expected_issue + + def _collect_imperative_xfail_nodes(tree: ast.Module) -> list[ast.Call]: """Find imperative ``pytest.xfail(...)``/``xfail(...)`` calls used as bare statements. diff --git a/tests/contracts/test_tool_param_roles.py b/tests/contracts/test_tool_param_roles.py index be9e865341..a5a7e473cd 100644 --- a/tests/contracts/test_tool_param_roles.py +++ b/tests/contracts/test_tool_param_roles.py @@ -14,6 +14,7 @@ from dataclasses import fields as dataclass_fields from dataclasses import replace +from inspect import signature import pytest @@ -24,12 +25,15 @@ RUNTIME_ADMISSION_BY_ROLE, RuntimeAdmission, ToolParamRole, + ToolWireType, compute_tool_contract_identity, get_tool_def, runtime_exempt_param_names, ) +from autoskillit.recipe._binding import bind_step_invocation +from autoskillit.recipe._contracts_manifest import load_bundled_manifest from autoskillit.recipe.schema import RecipeStep -from autoskillit.server.tools.tools_execution import _build_actual_mcp_kwargs +from autoskillit.server.tools.tools_execution import _build_actual_mcp_kwargs, run_skill pytestmark = [pytest.mark.layer("contracts"), pytest.mark.small] @@ -113,20 +117,13 @@ def test_execution_tuning_fallback_tables_cover_role_exactly() -> None: def test_build_actual_mcp_kwargs_fails_fast_on_drift() -> None: - """The kwargs-assembly helper must raise on a missing handler param and on an - unknown values key, and must NOT demand values for dispatch_items - (handler_parameter=False) or skill_inputs (structured_skill_inputs=True) — - pinning the filter that keeps assembly aligned with the real handler surface.""" + """The kwargs-assembly helper must follow the ToolDef wire-kwarg view.""" tool_def = _run_skill_tool_def() - handler_param_names = frozenset( - param.name - for param in tool_def.params - if param.handler_parameter and not param.structured_skill_inputs - ) - assert "dispatch_items" not in handler_param_names - assert "skill_inputs" not in handler_param_names + mcp_kwarg_names = tool_def.mcp_kwarg_names + assert "dispatch_items" not in mcp_kwarg_names + assert "skill_inputs" not in mcp_kwarg_names - complete_values: dict[str, object] = dict.fromkeys(handler_param_names, "") + complete_values: dict[str, object] = dict.fromkeys(mcp_kwarg_names, "") incomplete = dict(complete_values) del incomplete["order_id"] @@ -134,12 +131,45 @@ def test_build_actual_mcp_kwargs_fails_fast_on_drift() -> None: _build_actual_mcp_kwargs(tool_def, incomplete) excess = dict(complete_values) - excess["not_a_real_param"] = "" - with pytest.raises(ValueError, match="not_a_real_param"): + excess["dispatch_items"] = "" + with pytest.raises(ValueError, match="dispatch_items"): _build_actual_mcp_kwargs(tool_def, excess) result = _build_actual_mcp_kwargs(tool_def, complete_values) - assert set(result) == handler_param_names + assert set(result) == mcp_kwarg_names + + +def test_compiled_mcp_kwargs_are_exactly_the_kwarg_view() -> None: + tool_def = get_tool_def("run_skill") + assert tool_def is not None + with_args = { + param.name: ( + "/autoskillit:dry-walkthrough" + if param.name == "skill_command" + else "/repo" + if param.name == "cwd" + else "verify" + if param.name == "step_name" + else 2400 + if param.wire_type is ToolWireType.INTEGER + else {"plan_path": "/tmp/plan.md"} + if param.wire_type is ToolWireType.OBJECT + else "value" + ) + for param in tool_def.params + } + step = RecipeStep(name="verify", tool="run_skill", with_args=with_args) + + invocation = bind_step_invocation( + "verify", + step, + manifest=load_bundled_manifest(), + ) + + assert invocation.is_valid, invocation.failures + expected_names = set(signature(run_skill).parameters) - {"ctx", "skill_inputs"} + assert tool_def.mcp_kwarg_names == expected_names + assert {value.name for value in invocation.mcp_kwargs} == expected_names def test_run_skill_attestation_params_is_subset_of_protocol_role() -> None: diff --git a/tests/recipe/test_runtime_skill_invocation_binding.py b/tests/recipe/test_runtime_skill_invocation_binding.py index b2364c3f5b..4c41a967ba 100644 --- a/tests/recipe/test_runtime_skill_invocation_binding.py +++ b/tests/recipe/test_runtime_skill_invocation_binding.py @@ -125,13 +125,11 @@ def _bind( skill_inputs: Mapping[str, BoundScalar] | None | object = _DEFAULT, **overrides: BoundScalar, ): - """Bind against ``template``, matching production's shape: the caller - always supplies actual values for every compiled (with:-declared) - param — see ``_build_actual_mcp_kwargs`` in tools_execution/, which - never produces a sparse dict. ``skill_command``/``cwd``/``step_name`` - are always compiled (every ``_template()`` step declares them); pass - extra kwargs to override or to supply values for anything else under - test.""" + """Bind values for each compiled MCP kwarg (``ToolDef.mcp_kwarg_names`` + intersected with the step's ``with:`` declarations), matching production. + ``skill_command``/``cwd``/``step_name`` are always compiled (each + ``_template()`` step declares them); extra kwargs override or supply + values for other compiled kwargs under test.""" assert template.invocation.skill_name is not None skill_command = f"/autoskillit:{template.invocation.skill_name}" actual_mcp_kwargs: dict[str, BoundScalar] = { @@ -230,6 +228,13 @@ def test_runtime_binding_admits_explicit_advertised_default_without_auto_fill() assert bound == (("plan_path", "/tmp/plan.md"), ("audit_cycle_path", "")) +def test_non_handler_param_is_authorable_but_not_a_compiled_mcp_kwarg() -> None: + template = _template(dispatch_items="P1,P2") + + assert "dispatch_items" not in {value.name for value in template.invocation.mcp_kwargs} + assert _bind(template) == (("plan_path", "/tmp/plan.md"),) + + @pytest.mark.parametrize( ("input_name", "wire", "expected"), [ diff --git a/tests/server/test_attestation_delivery_reachability.py b/tests/server/test_attestation_delivery_reachability.py index 79b2ccad65..50941aed3f 100644 --- a/tests/server/test_attestation_delivery_reachability.py +++ b/tests/server/test_attestation_delivery_reachability.py @@ -330,20 +330,7 @@ async def test_ready_recipe_load_uses_worker_cache_without_sharing_context( "task_description": "test task", }, ), - pytest.param( - ("planner", "elaborate_phases", {"task": "test task", "source_dir": "."}), - marks=pytest.mark.xfail( - strict=True, - reason=( - "Blocked by #4775: elaborate_phases declares dispatch_items, " - "which is compiled into mcp_kwargs but filtered out of the " - "runtime actual-kwargs assembly, so the genuine-digest half " - "is denied at preflight:recipe_execution. The fixture itself " - "is sound — only the attested call is unreachable. Independent " - "of #4707; goes green automatically once #4775 lands." - ), - ), - ), + ("planner", "elaborate_phases", {"task": "test task", "source_dir": "."}), ], indirect=True, ) diff --git a/tests/server/test_attested_admission_replay.py b/tests/server/test_attested_admission_replay.py new file mode 100644 index 0000000000..28bce5bbb3 --- /dev/null +++ b/tests/server/test_attested_admission_replay.py @@ -0,0 +1,403 @@ +"""Replay statically resolved built-in run_skill steps through admission. + +Dynamic skill commands require runtime resolution and are explicitly skipped. +""" + +from __future__ import annotations + +from functools import cache +from pathlib import Path +from types import SimpleNamespace +from typing import Any, NoReturn, cast + +import pytest + +from autoskillit.core import ( + InstalledRecipeExecution, + RecipeSource, + build_recipe_execution_credential, +) +from autoskillit.core.io import load_yaml +from autoskillit.recipe import load_and_validate +from autoskillit.recipe.schema import RecipeKind +from autoskillit.server.recipe._recipe_execution import ( + RecipeExecutionAdmissionError, + bind_attested_runtime_invocation, + build_recipe_execution_snapshot, +) +from tests._run_skill_caller_oracle import ( + AdmissionDefect, + build_run_skill_payload, + run_skill_actual_mcp_kwargs, +) +from tests._tracked_recipes import tracked_recipe_load_result + +pytestmark = [pytest.mark.layer("server"), pytest.mark.medium] + +REPO_ROOT = Path(__file__).resolve().parents[2] +_PINNED: dict[tuple[str, str], Any] = { + ("planner", "elaborate_phases"): pytest.param( + "planner", + "elaborate_phases", + "recipe_execution_static_input_mismatch", + "static skill input 'target_phase_id'", + id="planner-elaborate_phases", + marks=pytest.mark.xfail( + strict=True, + raises=AdmissionDefect, + reason="Blocked by #5307: planner item placeholders compile as static inputs", + ), + ), + ("planner", "elaborate_assignments"): pytest.param( + "planner", + "elaborate_assignments", + "recipe_execution_static_input_mismatch", + "static skill input 'context_file'", + id="planner-elaborate_assignments", + marks=pytest.mark.xfail( + strict=True, + raises=AdmissionDefect, + reason="Blocked by #5307: planner item placeholders compile as static inputs", + ), + ), + ("planner", "refine_assignments"): pytest.param( + "planner", + "refine_assignments", + "recipe_execution_static_input_mismatch", + "static skill input 'context_file'", + id="planner-refine_assignments", + marks=pytest.mark.xfail( + strict=True, + raises=AdmissionDefect, + reason="Blocked by #5307: planner item placeholders compile as static inputs", + ), + ), + ("planner", "elaborate_wps"): pytest.param( + "planner", + "elaborate_wps", + "recipe_execution_static_input_mismatch", + "static skill input 'context_file'", + id="planner-elaborate_wps", + marks=pytest.mark.xfail( + strict=True, + raises=AdmissionDefect, + reason="Blocked by #5307: planner item placeholders compile as static inputs", + ), + ), + ("planner", "refine_wps"): pytest.param( + "planner", + "refine_wps", + "recipe_execution_static_input_mismatch", + "static skill input 'context_file'", + id="planner-refine_wps", + marks=pytest.mark.xfail( + strict=True, + raises=AdmissionDefect, + reason="Blocked by #5307: planner item placeholders compile as static inputs", + ), + ), + ("merge-prs", "merge_pr"): pytest.param( + "merge-prs", + "merge_pr", + "recipe_execution_static_input_mismatch", + "static skill input 'pr_number'", + id="merge-prs-merge_pr", + marks=pytest.mark.xfail( + strict=True, + raises=AdmissionDefect, + reason="Blocked by #5306: merge item placeholders compile as static inputs", + ), + ), + ("research-implement", "decompose_phases"): pytest.param( + "research-implement", + "decompose_phases", + "recipe_execution_step_mismatch", + "step is not present in the active compiled execution", + id="research-implement-decompose_phases", + marks=pytest.mark.xfail( + strict=True, + raises=AdmissionDefect, + reason="Blocked by #5305: delivered step_name differs from the recipe step key", + ), + ), + ("research-implement", "adjust_experiment"): pytest.param( + "research-implement", + "adjust_experiment", + "recipe_execution_step_mismatch", + "step is not present in the active compiled execution", + id="research-implement-adjust_experiment", + marks=pytest.mark.xfail( + strict=True, + raises=AdmissionDefect, + reason="Blocked by #5305: delivered step_name differs from the recipe step key", + ), + ), + ("research-implement", "fix_tests"): pytest.param( + "research-implement", + "fix_tests", + "recipe_execution_step_mismatch", + "step is not present in the active compiled execution", + id="research-implement-fix_tests", + marks=pytest.mark.xfail( + strict=True, + raises=AdmissionDefect, + reason="Blocked by #5305: delivered step_name differs from the recipe step key", + ), + ), + ("bem-wrapper", "run_bem"): pytest.param( + "bem-wrapper", + "run_bem", + "step_not_installed", + "missing_tool_parameter: cwd", + id="bem-wrapper-run_bem", + marks=pytest.mark.xfail( + strict=True, + raises=AdmissionDefect, + reason="Blocked by #5310: missing required cwd silently prevents step installation", + ), + ), +} + + +@cache +def _load_recipe( + recipe_name: str, +) -> tuple[Any, InstalledRecipeExecution, dict[str, Any], dict[str, Any]]: + loaded = load_and_validate( + recipe_name, + project_dir=REPO_ROOT, + include_finalized_projection=True, + ) + if not loaded["valid"]: + raise AssertionError(f"invalid built-in recipe {recipe_name!r}: {loaded['errors']}") + + projection = loaded["_finalized_projection"].binding_projection + snapshot = build_recipe_execution_snapshot( + recipe_name=recipe_name, + content_hash=loaded["content_hash"], + composite_hash=loaded["composite_hash"], + projection=projection, + ) + installed = cast(InstalledRecipeExecution, SimpleNamespace(snapshot=snapshot)) + credential = build_recipe_execution_credential(snapshot).as_wire_block() + recipe_document = cast(dict[str, Any], load_yaml(loaded["content"])) + steps = cast(dict[str, Any], recipe_document["steps"]) + return loaded, installed, credential, steps + + +def _replay_parameters() -> list[Any]: + parameters: list[Any] = [] + seen_pins: set[tuple[str, str]] = set() + for info in tracked_recipe_load_result(REPO_ROOT).items: + if info.source is not RecipeSource.BUILTIN or info.kind is RecipeKind.CAMPAIGN: + continue + loaded, _, _, steps = _load_recipe(info.name) + post_prune_steps = loaded["post_prune_step_names"] + for step_name, step_body in steps.items(): + if step_name not in post_prune_steps or step_body.get("tool") != "run_skill": + continue + key = (info.name, step_name) + seen_pins.add(key) + parameters.append( + _PINNED[key] + if key in _PINNED + else pytest.param(info.name, step_name, None, None, id=f"{info.name}-{step_name}") + ) + + parameters.extend(pin for key, pin in _PINNED.items() if key not in seen_pins) + return parameters + + +def _dynamic_step_outcome(step_name: str) -> NoReturn: + pytest.skip( + f"{step_name}: dynamic skill command requires runtime resolution before admission replay" + ) + + +def _uninstalled_step_outcome(projection: Any, step_name: str) -> tuple[str, str]: + failures = projection.invocations[step_name].failures + detail = "; ".join(f"{failure.code.value}: {failure.name}" for failure in failures) + return "step_not_installed", detail + + +def _templated_step_outcome( + step_name: str, + step_body: dict[str, Any], + installed: InstalledRecipeExecution, + credential: dict[str, Any], + cwd: str, +) -> tuple[str, str] | None: + payload = build_run_skill_payload( + step_name, + step_body, + credential, + cwd=cwd, + ) + try: + bind_attested_runtime_invocation( + installed, + execution_id=payload["recipe_execution_id"], + step_name=payload["step_name"], + template_digest=payload["invocation_template_digest"], + skill_command=payload["skill_command"], + skill_inputs=payload.get("skill_inputs"), + actual_mcp_kwargs=run_skill_actual_mcp_kwargs(payload), + ) + except RecipeExecutionAdmissionError as exc: + return exc.code, str(exc) + return None + + +def _step_outcome( + recipe_name: str, + step_name: str, + tmp_path: Path, +) -> tuple[str, str] | None: + loaded, installed, credential, steps = _load_recipe(recipe_name) + if ( + step_name not in loaded["post_prune_step_names"] + or steps.get(step_name, {}).get("tool") != "run_skill" + ): + raise AssertionError( + f"stale admission replay pin {recipe_name}.{step_name}: " + "no delivered run_skill step matches this pin" + ) + snapshot = installed.snapshot + if step_name in snapshot.dynamic_skill_step_names: + _dynamic_step_outcome(step_name) + if step_name not in snapshot.templates: + projection = loaded["_finalized_projection"].binding_projection + return _uninstalled_step_outcome(projection, step_name) + return _templated_step_outcome( + step_name, + steps[step_name], + installed, + credential, + str(tmp_path), + ) + + +def test_dynamic_steps_are_explicitly_skipped() -> None: + with pytest.raises(pytest.skip.Exception, match="dynamic.*runtime resolution"): + _dynamic_step_outcome("dispatch") + + +def _assert_admitted( + outcome: tuple[str, str] | None, + pin_code: str | None, + pin_detail: str | None, + diagnostic: str, +) -> None: + if outcome is None: + return + code, detail = outcome + if ( + pin_code is not None + and pin_detail is not None + and code == pin_code + and pin_detail in detail + ): + raise AdmissionDefect(f"{diagnostic}: {code}: {detail}") + raise AssertionError(f"{diagnostic}: unexpected admission rejection {code!r}: {detail}") + + +@pytest.mark.parametrize( + ("outcome", "pin_code", "pin_detail", "expected_type"), + [ + pytest.param(None, None, None, None, id="admitted"), + pytest.param( + ("rejected", "expected detail is present"), + "rejected", + "expected detail", + AdmissionDefect, + id="exact-pin", + ), + pytest.param( + ("different", "expected detail is present"), + "rejected", + "expected detail", + AssertionError, + id="different-code", + ), + pytest.param( + ("rejected", "other detail"), + "rejected", + "expected detail", + AssertionError, + id="different-detail", + ), + ], +) +def test_assert_admitted_only_accepts_the_exact_pin( + outcome: tuple[str, str] | None, + pin_code: str | None, + pin_detail: str | None, + expected_type: type[BaseException] | None, +) -> None: + if expected_type is None: + assert _assert_admitted(outcome, pin_code, pin_detail, "recipe.step") is None + return + with pytest.raises(expected_type) as exc_info: + _assert_admitted(outcome, pin_code, pin_detail, "recipe.step") + assert type(exc_info.value) is expected_type + + +@pytest.mark.parametrize( + ("command", "expected"), + [ + ("run {phase_id}", "run P1"), + ("run {{AUTOSKILLIT_TEMP}}", "run {{AUTOSKILLIT_TEMP}}"), + ("run ${{ env.NAME }}", "run ${{ env.NAME }}"), + ], +) +def test_payload_substitutes_only_supported_tokens(command: str, expected: str) -> None: + payload = build_run_skill_payload( + "P1", + {"with": {"skill_command": command}}, + {"execution_id": "exec", "invocation_template_digests": {"P1": "digest"}}, + cwd="/repo", + ) + assert payload["skill_command"] == expected + + +@pytest.mark.parametrize("missing_required", [False, True]) +def test_payload_uses_credential_input_order_and_requires_caller_values( + missing_required: bool, +) -> None: + step = { + "with": { + "cwd": "${{ context.repo }}", + "skill_inputs": {} if missing_required else {"required": "${{ context.pr_number }}"}, + } + } + credential = { + "execution_id": "exec", + "invocation_template_digests": {"verify": "digest"}, + "skill_input_shapes": { + "verify": {"keys": ["required", "optional"], "absence_values": {"optional": ""}} + }, + } + if missing_required: + with pytest.raises(AssertionError, match="verify.*required.*no caller value"): + build_run_skill_payload("verify", step, credential, cwd="/repo") + return + payload = build_run_skill_payload("verify", step, credential, cwd="/repo") + assert payload["cwd"] == "/repo" + skill_inputs = payload["skill_inputs"] + assert isinstance(skill_inputs, dict) + assert list(skill_inputs.items()) == [("required", 7), ("optional", "")] + + +@pytest.mark.parametrize( + ("recipe", "step", "pin_code", "pin_detail"), + _replay_parameters(), +) +def test_run_skill_step_is_admitted_for_its_orchestrator_payload( + recipe: str, + step: str, + pin_code: str | None, + pin_detail: str | None, + tmp_path: Path, +) -> None: + outcome = _step_outcome(recipe, step, tmp_path) + _assert_admitted(outcome, pin_code, pin_detail, f"{recipe}.{step}") diff --git a/tests/server/test_attested_param_admission_conformance.py b/tests/server/test_attested_param_admission_conformance.py index 23e8d792d3..87d8acb0a2 100644 --- a/tests/server/test_attested_param_admission_conformance.py +++ b/tests/server/test_attested_param_admission_conformance.py @@ -185,18 +185,6 @@ def record_step_provider(*args, **kwargs): assert resolved_step_providers == ["anthropic"] -@pytest.mark.xfail( - strict=True, - reason=( - "Blocked by #4775: dispatch_items is filtered out of the runtime " - "actual-kwargs assembly (handler_parameter=False) but not out of the " - "compile-time mcp_kwargs binding, and _binding.py:1028 demands every " - "compiled kwarg be present — so no fan-out step can pass attestation. " - "Every bundled recipe step with a literal non-templated model: is a " - "dispatch_items fan-out step, so this is the only reachable vehicle. " - "Independent of #4707; goes green automatically once #4775 lands." - ), -) @pytest.mark.parametrize( "tool_ctx_ready_recipe", [("planner", "elaborate_phases", _PLANNER_OVERRIDES)], @@ -216,9 +204,8 @@ async def test_model_delivered_from_recipe_step_without_caller_forwarding( engages) would reproduce the exact coverage gap this plan exists to close — a green test that is no evidence at all about attestation. - So it is bridged rather than weakened. See the xfail reason for why it - cannot pass today; the assertions below are the real contract and are - what will be checked once #4775 is fixed. + The assertions below check that the model declared by the recipe reaches + the headless executor through the attested admission path. """ from autoskillit.server.tools.tools_execution import run_skill from tests.fakes import InMemoryHeadlessExecutor @@ -233,7 +220,8 @@ async def test_model_delivered_from_recipe_step_without_caller_forwarding( # elaborate_phases declares no skill_inputs: sub-key — its child inputs come # from parsing skill_command's placeholders. Omitting the parameter lets the - # runtime binder derive them (_binding.py:965-990) rather than guessing keys. + # runtime binder derive them in _normalize_runtime_skill_inputs rather + # than guessing keys. declared_inputs = with_args.get("skill_inputs") optional_inputs = ( {"skill_inputs": {name: "probe value" for name in declared_inputs}} @@ -246,7 +234,8 @@ async def test_model_delivered_from_recipe_step_without_caller_forwarding( with_args["skill_command"], str(work_dir), step_name=ready.step_name, - output_dir=with_args["output_dir"], + # Stands in for context.planner_dir as returned by create_run_dir. + output_dir=".autoskillit/temp/planner/run-test", recipe_execution_id=ready.credential["execution_id"], invocation_template_digest=ready.template_digest, **optional_inputs, diff --git a/tests/server/test_recipe_execution_receipt_completeness.py b/tests/server/test_recipe_execution_receipt_completeness.py index 87ac652408..cf675c53e9 100644 --- a/tests/server/test_recipe_execution_receipt_completeness.py +++ b/tests/server/test_recipe_execution_receipt_completeness.py @@ -20,6 +20,10 @@ build_bound_child_prompt, get_recipe_execution, ) +from tests._run_skill_caller_oracle import ( + literal_run_skill_payload, + run_skill_actual_mcp_kwargs, +) from tests.server._recipe_fixture_parameters import ( IMPLEMENTATION_OVERRIDES, IMPLEMENTATION_READY_PARAMETERS, @@ -214,6 +218,14 @@ async def test_following_the_recovery_yields_an_admitted_digest( receipt = json.loads(await complete_recipe_initialization(initialization_id=recovery_id)) assert receipt["success"] is True, receipt credential = receipt["recipe_execution"] + payload = literal_run_skill_payload( + step_name, + {"with": ready.tool_ctx.active_recipe_steps[step_name].with_args}, + credential, + cwd=str(git_checkout), + ) + payload["skill_command"] = _skill_command(ready.tool_ctx, step_name) + assert payload["step_name"] == step_name expected_template = installed.snapshot.templates[step_name] invocation = expected_template.invocation contract = get_skill_contract(invocation.skill_name, load_bundled_manifest()) @@ -237,14 +249,9 @@ async def test_following_the_recovery_yields_an_admitted_digest( execution_id=credential["execution_id"], step_name=step_name, template_digest=credential["invocation_template_digests"][step_name], - skill_command=_skill_command(ready.tool_ctx, step_name), + skill_command=payload["skill_command"], skill_inputs=skill_inputs, - actual_mcp_kwargs={ - **{value.name: value.effective_value for value in invocation.mcp_kwargs}, - "step_name": step_name, - "recipe_execution_id": credential["execution_id"], - "invocation_template_digest": credential["invocation_template_digests"][step_name], - }, + actual_mcp_kwargs=run_skill_actual_mcp_kwargs(payload), ) assert admitted_template is expected_template assert dict(bound_inputs) == skill_inputs