diff --git a/src/autoskillit/core/types/constants/_type_constants_registries.py b/src/autoskillit/core/types/constants/_type_constants_registries.py index 763ff4f61c..a692acc10a 100644 --- a/src/autoskillit/core/types/constants/_type_constants_registries.py +++ b/src/autoskillit/core/types/constants/_type_constants_registries.py @@ -398,7 +398,7 @@ class ExecutionInstallSiteDef(NamedTuple): { "inline_delivery": ExecutionInstallSiteDef( installer_module="src/autoskillit/server/recipe/_recipe_delivery/_completion.py", - installer_symbol="complete_finalized_recipe_response", + installer_symbol="_activate_recipe_initialization", delivering_module="src/autoskillit/server/recipe/_recipe_artifact.py", delivering_symbol="build_canonical_recipe_artifact_payload", delivery_surface="open_kitchen", diff --git a/tests/_ambient_env_surface.py b/tests/_ambient_env_surface.py index 59fc25e3ee..788edbec6c 100644 --- a/tests/_ambient_env_surface.py +++ b/tests/_ambient_env_surface.py @@ -953,124 +953,144 @@ class AmbientEnvDisposition: DYNAMIC_READ_EXEMPTIONS: dict[str, str] = { - "execution/evidence/reader/_contract.py:146": ( + ("execution/evidence/reader/_contract.py::evidence_reader_provider_environment"): ( "Dict/generator-comprehension key bound by `for name in _PROVIDER_ENV`; this scanner does" "not trace comprehension-bound names back through their iterable's members." ), - "server/lifecycle/_guards.py:418": ( + ("server/lifecycle/_guards.py::_profile_to_env"): ( "`profile.api_key_env` is a per-provider-profile instance attribute resolved at runtime" "from config, not a module-level constant this AST scanner can resolve." ), - "server/lifecycle/_lifespan/_session_boots.py:524": ( + ("server/lifecycle/_lifespan/_session_boots.py::_evidence_reader_auto_gate_boot"): ( "Dict-comprehension key bound by `for name in EVIDENCE_READER_ENV_FORWARD_VARS`; the three" "forwarded names are already captured directly via that collection's own R4 scan." ), - "server/lifecycle/_session_type.py:49": ( + "server/lifecycle/_session_type.py::_evidence_reader_binding_state": ( "Set-comprehension key bound by `for key in EVIDENCE_READER_ENV_FORWARD_VARS`; the" "forwarded names are already captured directly via that collection's own R4 scan." - ), - "server/lifecycle/_session_type.py:54": ( + "\n\n" "Generator-expression key bound by `for key in EVIDENCE_READER_ENV_FORWARD_VARS`; the" "forwarded names are already captured directly via that collection's own R4 scan." ), - "server/lifecycle/_session_type.py:61": ( + ("server/lifecycle/_session_type.py::_has_explorer_binding_env"): ( "Dict-comprehension key bound by `for key in _EXPLORER_BINDING_ENV_KEYS`; the four" "explorer-binding names are already captured directly via that collection's own R4 scan." ), - "server/tools/tools_evidence_reader.py:174": ( + ("server/tools/tools_evidence_reader.py::_private_environment"): ( "Dict-comprehension key bound by `for name in EVIDENCE_READER_ENV_FORWARD_VARS`; the three" "forwarded names are already captured directly via that collection's own R4 scan." ), - "server/tools/tools_execution/_candidate_policy.py:191": ( + ("server/tools/tools_execution/_candidate_policy.py::resolve_candidate_policy"): ( "`definition.api_key_env` is a per-provider-profile instance attribute resolved at " "runtime from config, not a module-level constant this AST scanner can resolve." ), - "server/tools/tools_execution/_run_skill_prepare.py:123": ( + ("server/tools/tools_execution/_run_skill_prepare.py::_resolve_provider_binding"): ( "`definition.api_key_env` is a per-provider-profile instance attribute resolved at " "runtime from the persisted launch contract, not a statically resolvable name." ), } +DYNAMIC_READ_EXEMPTION_COUNTS: dict[str, int] = { + "execution/evidence/reader/_contract.py::evidence_reader_provider_environment": 2, + "server/lifecycle/_session_type.py::_evidence_reader_binding_state": 2, + "server/tools/tools_evidence_reader.py::_private_environment": 2, +} + FORWARDING_SITES: dict[str, str] = { - "cli/install/_marketplace.py:358": ( + ("cli/install/_marketplace.py::install"): ( "Unfiltered dict(os.environ) snapshot (`ambient_env`) used as the base for an" "install/publish subprocess env; intentional wholesale forward for a maintenance-style" "operation." ), - "cli/app.py:315": ( + ("cli/app.py::install"): ( "Bare os.environ passed as `child_env` to the maintenance installer, which itself applies" "an explicit allowlist (build_maintenance_env) before spawning; unfiltered by design here." ), - "cli/session/_session_launch.py:161": ( + ("cli/session/_session_launch.py::prepare_interactive_launch"): ( "Unfiltered dict(os.environ) used only to probe an exact executable path before the real" "session env is sealed by build_agent_env elsewhere; not the launched child's env." ), - "cli/update/_transaction.py:440": ( + ("cli/update/_transaction.py::run_update_transaction"): ( "Unfiltered dict(os.environ if base_env is None else base_env) snapshot captured for" "update-transaction diagnostics/rollback comparison, not for a spawned child process." ), - "cli/update/_update_checks_source.py:190": ( + ("cli/update/_update_checks_source.py::_git_ls_remote_sha"): ( "Bare os.environ passed as `env=` to a read-only `git ls-remote` probe subprocess;" "unfiltered wholesale forward for a local diagnostic command." ), - "execution/backends/_codex_probes.py:411": ( + ("execution/backends/_codex_probes.py::_validate_generated_codex_home"): ( "Unfiltered dict(os.environ) base for the global-Codex-home MCP-inventory validation" "probe subprocess, with CODEX_RESERVED_HOME_ENV_VARS overridden to the source home." ), - "execution/backends/claude.py:341": ( + ("execution/backends/claude.py::ClaudeCodeBackend._prepare_interactive_environment"): ( "Excludes _INTERACTIVE_ENV_EXCLUSIONS (TERM/NO_COLOR headless-hardening keys) when" "building the interactive Claude Code base env." ), - "execution/backends/_claude/session_commands.py:108": ( + ( + "execution/backends/_claude/session_commands.py::" + "ClaudeSessionCommandMixin._skill_session_environment_inputs" + ): ( "Excludes _HEADLESS_EXCLUSIVE_VARS before build_agent_env layers extras back in for a" "headless Claude Code skill-session launch." ), - "execution/backends/_claude/session_commands.py:326": ( + ( + "execution/backends/_claude/session_commands.py::" + "ClaudeSessionCommandMixin.build_food_truck_cmd" + ): ( "Excludes _HEADLESS_EXCLUSIVE_VARS before build_agent_env layers extras back in for a" "headless Claude Code food-truck orchestrator-session launch." ), - "execution/backends/_codex/headless_commands.py:65": ( + ( + "execution/backends/_codex/headless_commands.py::" + "CodexOrdinaryHeadlessCommandMixin.build_headless_cmd" + ): ( "Excludes _HEADLESS_EXCLUSIVE_VARS before build_env layers extras back in for a headless" " Codex generic-prompt launch; the finalized generated home is layered by the caller." ), - "execution/backends/_codex/session_commands.py:176": ( + ( + "execution/backends/_codex/session_commands.py::CodexCommandMixin." + "_prepare_skill_session_environment" + ): ( "Excludes _HEADLESS_EXCLUSIVE_VARS before build_env layers extras back in for a headless" "Codex skill-session launch." ), - "execution/backends/_codex/session_commands.py:494": ( + ("execution/backends/_codex/session_commands.py::CodexCommandMixin.build_food_truck_cmd"): ( "Excludes _HEADLESS_EXCLUSIVE_VARS before build_env layers extras back in for a headless" "Codex food-truck orchestrator-session launch." ), - "execution/backends/_codex/session_commands.py:194": ( + ( + "execution/backends/_codex/session_commands.py::CodexCommandMixin." + "_prepare_interactive_environment" + ): ( "Excludes _HEADLESS_EXCLUSIVE_VARS before extras merge for a Codex interactive-session" "launch base env." ), - "execution/backends/_codex/session_commands.py:707": ( + ("execution/backends/_codex/session_commands.py::CodexCommandMixin.build_resume_cmd"): ( "Excludes _HEADLESS_EXCLUSIVE_VARS before build_env layers extras back in for a headless" "Codex resume-session launch." ), - "execution/merge_queue/_merge_queue_group_ci.py:109": ( + ("execution/merge_queue/_merge_queue_group_ci.py::_query_merge_group_ci"): ( "Unfiltered {**os.environ} base for a local `gh run list` CI-status probe subprocess;" "GH_TOKEN is the only key ever added on top." ), - "execution/runtime/testing.py:50": ( + ("execution/runtime/testing.py::build_sanitized_env"): ( "Excludes AUTOSKILLIT_PRIVATE_ENV_VARS before handing the sanitized env to a user-code" "`pytest` subprocess spawned by test_check." ), - "exploration/snapshot/_capture.py:92": ( + ("exploration/snapshot/_capture.py::_git"): ( "Unfiltered dict(os.environ) base for a local read-only `git` subprocess (snapshot" "diffing); only GIT_OPTIONAL_LOCKS is added on top." ), - "hooks/_capture_spawn.py:298": ( + ("hooks/_capture_spawn.py::_scrubbed_user_environment"): ( "dict(os.environ) followed by an explicit per-key .pop() loop over" "PROTECTED_CAPTURE_ENV_VARS; the exclusion happens outside the single expression this" "scanner inspects." ), - "hooks/_dispatch.py:84": ( + ("hooks/_dispatch.py::main"): ( "Unfiltered dict(os.environ) base for a same-host hook-script subprocess; only" "PYTHONDONTWRITEBYTECODE is added on top." ), - "smoke_utils/_cross_interpreter_upgrade.py:337": ( + ("smoke_utils/_cross_interpreter_upgrade.py::run_cross_interpreter_upgrade_smoke"): ( "Unfiltered dict(os.environ) base for a smoke-test cross-interpreter upgrade subprocess" "sandboxed to a scratch HOME/XDG tree overridden immediately after." ), diff --git a/tests/contracts/_ast_helpers.py b/tests/_ast_helpers.py similarity index 52% rename from tests/contracts/_ast_helpers.py rename to tests/_ast_helpers.py index 54f94b615c..2ab50cc497 100644 --- a/tests/contracts/_ast_helpers.py +++ b/tests/_ast_helpers.py @@ -1,11 +1,26 @@ -"""Shared AST helpers for call-site extraction tests.""" +"""Shared AST helpers for call-site extraction tests across test packages.""" from __future__ import annotations import ast from collections import Counter -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from pathlib import Path +from typing import TypeVar + +MODULE_SCOPE = "" +_K = TypeVar("_K") + + +def count_differences( + expected: Mapping[_K, int], observed: Mapping[_K, int] +) -> list[tuple[_K, int, int]]: + """Return mismatched registry keys with their expected and observed counts.""" + return [ + (key, expected.get(key, 0), observed.get(key, 0)) + for key in sorted(expected.keys() | observed.keys(), key=str) + if expected.get(key, 0) != observed.get(key, 0) + ] def call_name(node: ast.Call) -> str | None: @@ -17,6 +32,27 @@ def call_name(node: ast.Call) -> str | None: return None +def scope_qualnames(tree: ast.AST) -> dict[ast.AST, str]: + """Map nodes to their enclosing named-definition AST chain. + + Scopes are ``ClassDef``/``FunctionDef``/``AsyncFunctionDef`` nodes joined with ``."`` + (``Outer.method``); a def or class node maps to its own qualname, and nodes outside + every def and class map to ``MODULE_SCOPE``. Definition headers inherit that + definition's key, including decorators, defaults, annotations and class bases. + This describes AST ancestry, not Python execution scopes. Keys omit runtime + ``__qualname__``'s ```` component: a nested function is ``outer.inner``. + """ + owners: dict[ast.AST, str] = {} + stack: list[tuple[ast.AST, str]] = [(tree, MODULE_SCOPE)] + while stack: + node, scope = stack.pop() + if isinstance(node, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)): + scope = node.name if scope == MODULE_SCOPE else f"{scope}.{node.name}" + owners[node] = scope + stack.extend((child, scope) for child in ast.iter_child_nodes(node)) + return owners + + def callers_by_function( src_root: Path, *, @@ -59,4 +95,11 @@ def count_callers(src_root: Path, symbol: str) -> int: return sum(callers_by_function(src_root, symbol=symbol).values()) -__all__: Iterable[str] = ("call_name", "callers_by_function", "count_callers") +__all__: Iterable[str] = ( + "MODULE_SCOPE", + "call_name", + "callers_by_function", + "count_callers", + "count_differences", + "scope_qualnames", +) diff --git a/tests/_test_filter.py b/tests/_test_filter.py index b42ca77da0..31ceff6fb7 100644 --- a/tests/_test_filter.py +++ b/tests/_test_filter.py @@ -196,9 +196,13 @@ class SourceMapRejection(enum.StrEnum): # skip the test and let a drift pass silently. test_docker_image.py # likewise compares the Dockerfile's pins with code constants under src/, which no infra # trigger covers. +# Whole-source-tree scanners read src/ by path, so neither an import edge nor coverage +# selects them; tests/arch/test_infra_source_scanner_selection.py requires every such +# infra or hooks scanner to be listed. _INFRA_UNCONDITIONAL_FILES: frozenset[str] = frozenset( { "test_docker_image.py", + "test_canonical_json_producer_convention.py", "test_manifest_completeness.py", "test_manifest_directory_completeness.py", "test_guard_coverage.py", @@ -211,6 +215,8 @@ class SourceMapRejection(enum.StrEnum): "test_plugin_source_ratchets.py", "test_lint_gate_selection.py", "test_typecheck_gate.py", + "test_schema_read_convention.py", + "test_session_ttl_single_source.py", } ) @@ -221,6 +227,9 @@ class SourceMapRejection(enum.StrEnum): "test_hook_executability.py", "test_hook_registration_coverage.py", "test_hook_registry.py", + "test_hook_scope_authority.py", + "test_join_enforcement_wiring.py", + "test_write_target_scan_authority.py", } ) diff --git a/tests/arch/_acceptance_policy_surfaces.py b/tests/arch/_acceptance_policy_surfaces.py index 22d85cd0a6..ddadfd06a5 100644 --- a/tests/arch/_acceptance_policy_surfaces.py +++ b/tests/arch/_acceptance_policy_surfaces.py @@ -49,6 +49,18 @@ class PolicyRelaxationApproval: POLICY_SURFACES: tuple[PolicySurface, ...] = ( + PolicySurface( + "tests/infra/test_schema_version_convention.py", + "_LEGACY_JSON_WRITE_COUNTS", + "int_map", + default=0, + ), + PolicySurface( + "tests/_ambient_env_surface.py", + "DYNAMIC_READ_EXEMPTION_COUNTS", + "int_map", + default=1, + ), PolicySurface( "tests/arch/_subpackage_isolation_line_limits.py", "_LINE_LIMIT_EXEMPTIONS", diff --git a/tests/arch/test_admission_evidence_explicit_guard.py b/tests/arch/test_admission_evidence_explicit_guard.py index 781c184f02..eb635b5cca 100644 --- a/tests/arch/test_admission_evidence_explicit_guard.py +++ b/tests/arch/test_admission_evidence_explicit_guard.py @@ -20,7 +20,7 @@ import pytest -from tests.contracts._ast_helpers import call_name +from tests._ast_helpers import call_name pytestmark = [pytest.mark.layer("arch"), pytest.mark.medium] diff --git a/tests/arch/test_execution_install_delivery.py b/tests/arch/test_execution_install_delivery.py index 8c2d904074..7ca1451900 100644 --- a/tests/arch/test_execution_install_delivery.py +++ b/tests/arch/test_execution_install_delivery.py @@ -1,20 +1,22 @@ -"""Install-site registry ratchet — every install_recipe_execution site is registered. +"""Install-site registry ratchet — every install_recipe_execution call is registered. Modelled on `tests/infra/test_canonical_json_producer_convention.py`. The registry binds -each install site to the response builder that delivers its credential. The scanner finds -real call sites in `src/autoskillit/`; the test fails when the two diverge in either -direction (added or removed site). A meta-test injects a fake site to confirm the -guard fires. +each install site to the response builder that delivers its credential. The scanner counts +real calls per named scope in `src/autoskillit/`, preserving multiple calls on one line. +The test fails when registered and scanned scope counts diverge in either direction. A meta-test +injects a fake site to confirm the guard fires. """ from __future__ import annotations import ast +from collections import Counter from pathlib import Path import pytest from autoskillit.core import RECIPE_EXECUTION_INSTALL_SITE_REGISTRY, all_tool_names +from tests._ast_helpers import MODULE_SCOPE, count_differences, scope_qualnames pytestmark = [pytest.mark.layer("arch"), pytest.mark.medium] @@ -31,27 +33,37 @@ def _is_install_recipe_execution_call(node: ast.Call) -> bool: return False -def _scan_install_recipe_execution_sites() -> set[tuple[str, int]]: +def _scan_install_recipe_execution_sites() -> list[tuple[str, str]]: """AST-scan src/autoskillit/ for install_recipe_execution() call sites. - Returns set of (relative_path, line_number) tuples. + Returns one (relative_path, qualified_scope) tuple per call, including same-line calls. """ - sites: set[tuple[str, int]] = set() + sites: list[tuple[str, str]] = [] for py_file in _SRC_ROOT.rglob("*.py"): - try: - tree = ast.parse(py_file.read_text(), filename=str(py_file)) - except SyntaxError: - continue + tree = ast.parse(py_file.read_text(), filename=str(py_file)) + scopes = scope_qualnames(tree) for node in ast.walk(tree): if not isinstance(node, ast.Call): continue if not _is_install_recipe_execution_call(node): continue rel = str(py_file.relative_to(_SRC_ROOT.parent.parent)) - sites.add((rel, node.lineno)) + sites.append((rel, scopes[node])) return sites +def _install_registry_problems( + registered: Counter[tuple[str, str]], observed: Counter[tuple[str, str]] +) -> list[str]: + """Report every scope whose registered and scanned call counts differ.""" + return [ + f"{module}::{scope}: registered {registered_count}, scanned {observed_count}" + for (module, scope), registered_count, observed_count in count_differences( + registered, observed + ) + ] + + def _function_calls_function( func_node: ast.FunctionDef | ast.AsyncFunctionDef, target_name: str ) -> bool: @@ -67,19 +79,93 @@ def _function_calls_function( class TestExecutionInstallDelivery: - """AST ratchet: every install_recipe_execution site must be registered.""" + """AST ratchet: registered and scanned install calls must match by scope.""" def test_every_install_site_is_registered(self) -> None: - registered_modules = { - entry.installer_module for entry in RECIPE_EXECUTION_INSTALL_SITE_REGISTRY.values() - } scanned = _scan_install_recipe_execution_sites() assert scanned, "scanner returned no install_recipe_execution sites" - scanner_modules = {site[0] for site in scanned} - missing = scanner_modules - registered_modules - assert not missing, ( - f"install_recipe_execution sites without registered entries: {sorted(missing)}" + registered = Counter( + (entry.installer_module, entry.installer_symbol) + for entry in RECIPE_EXECUTION_INSTALL_SITE_REGISTRY.values() ) + observed = Counter(scanned) + problems = _install_registry_problems(registered, observed) + assert not problems, f"install_recipe_execution registry mismatch: {problems}" + + @pytest.mark.parametrize( + ("registered", "observed", "expected"), + [ + pytest.param( + Counter({("a.py", "install"): 1, ("b.py", "install"): 1}), + Counter({("a.py", "install"): 1}), + ["b.py::install: registered 1, scanned 0"], + id="stale-registered-module", + ), + pytest.param( + Counter({("a.py", "install"): 1}), + Counter({("a.py", "install"): 2}), + ["a.py::install: registered 1, scanned 2"], + id="extra-scanned-call", + ), + pytest.param( + Counter({("a.py", "install"): 2}), + Counter({("a.py", "install"): 1}), + ["a.py::install: registered 2, scanned 1"], + id="lost-scanned-call", + ), + pytest.param( + Counter({("a.py", "install"): 1}), + Counter({("a.py", "install"): 1}), + [], + id="equal-counts", + ), + pytest.param( + Counter({("a.py", "install"): 1}), + Counter({("a.py", "other"): 1}), + ["a.py::install: registered 1, scanned 0", "a.py::other: registered 0, scanned 1"], + id="same-module-moved-scope", + ), + ], + ) + def test_install_registry_diff_is_bidirectional_and_counted( + self, + registered: Counter[tuple[str, str]], + observed: Counter[tuple[str, str]], + expected: list[str], + ) -> None: + assert _install_registry_problems(registered, observed) == expected + + @pytest.mark.parametrize( + ("source", "scope"), + [ + ("install_recipe_execution(); install_recipe_execution()\n", MODULE_SCOPE), + ( + "def install():\n install_recipe_execution(); install_recipe_execution()\n", + "install", + ), + ], + ) + def test_install_scanner_preserves_same_line_multiplicity( + self, tmp_path: Path, monkeypatch, source: str, scope: str + ) -> None: + source_root = tmp_path / "src" / "autoskillit" + source_root.mkdir(parents=True) + (source_root / "same_line.py").write_text(source, encoding="utf-8") + monkeypatch.setattr("tests.arch.test_execution_install_delivery._SRC_ROOT", source_root) + + sites = _scan_install_recipe_execution_sites() + assert sites == [ + ("src/autoskillit/same_line.py", scope), + ("src/autoskillit/same_line.py", scope), + ] + + def test_install_scanner_reports_parse_failure(self, tmp_path: Path, monkeypatch) -> None: + broken = tmp_path / "broken.py" + broken.write_text("def broken(:\n", encoding="utf-8") + monkeypatch.setattr("tests.arch.test_execution_install_delivery._SRC_ROOT", tmp_path) + with pytest.raises(SyntaxError) as error: + _scan_install_recipe_execution_sites() + assert error.value.filename == str(broken) def test_registered_delivering_symbols_call_the_sanctioned_producer(self) -> None: for entry in RECIPE_EXECUTION_INSTALL_SITE_REGISTRY.values(): @@ -107,9 +193,9 @@ def test_unregistered_install_site_fails_the_ratchet(self, monkeypatch) -> None: """Meta-test: a fake install site should cause the ratchet to fail.""" original_scan = _scan_install_recipe_execution_sites - def patched_scan() -> set[tuple[str, int]]: + def patched_scan() -> list[tuple[str, str]]: sites = original_scan() - sites.add(("src/autoskillit/fake_install_module.py", 1)) + sites.append(("src/autoskillit/fake_install_module.py", MODULE_SCOPE)) return sites monkeypatch.setattr( diff --git a/tests/arch/test_generated_home_writer_inventory.py b/tests/arch/test_generated_home_writer_inventory.py index 3cfce9b6c0..49f8709220 100644 --- a/tests/arch/test_generated_home_writer_inventory.py +++ b/tests/arch/test_generated_home_writer_inventory.py @@ -8,7 +8,7 @@ import pytest -from tests.contracts._ast_helpers import call_name, callers_by_function +from tests._ast_helpers import call_name, callers_by_function pytestmark = [pytest.mark.layer("arch"), pytest.mark.small] diff --git a/tests/arch/test_git_ref_qualification.py b/tests/arch/test_git_ref_qualification.py index 0fc2e7ffe9..3dd0623ad1 100644 --- a/tests/arch/test_git_ref_qualification.py +++ b/tests/arch/test_git_ref_qualification.py @@ -9,11 +9,13 @@ import ast import re +from collections import Counter from dataclasses import dataclass from typing import Final import pytest +from tests._ast_helpers import MODULE_SCOPE, scope_qualnames from tests.arch._deferred_debt import ( TrackedDeferral, assert_deferrals_have_regression_tests, @@ -82,147 +84,110 @@ r"^(?:FETCH_HEAD|HEAD|MERGE_HEAD|ORIG_HEAD|@)(?:(?:\^)|(?:~\d+))*$" ) _SHA_PATTERN: Final = re.compile(r"^(?:[0-9a-fA-F]{40}|[0-9a-fA-F]{64})$") -_CLONE_GUARD_PATH: Final = "execution/runtime/clone_guard.py" -_CMD_RPC_MERGE_PATH: Final = "recipe/cmd_rpc/_cmd_rpc_merge.py" - -# (relative path, enclosing function, line number, subcommand, classification) -_EXPECTED_GIT_REF_SITES: tuple[tuple[str, str, int, str, str], ...] = ( - ("cli/update/_update_checks_source.py", "_git_ls_remote_sha", 186, "ls-remote", "BARE"), - ("core/git/git_refs.py", "verify_qualified_ref_sync", 61, "rev-parse", "BARE"), - ("core/install/cmd_runner.py", "run_git", 62, "", "UNRESOLVABLE"), + +# (relative path, enclosing class/function, subcommand, classification) +# Repeated tuples represent distinct call sites with the same line-free identity. +_EXPECTED_GIT_REF_SITES: tuple[tuple[str, str, str, str], ...] = ( + ("cli/update/_update_checks_source.py", "_git_ls_remote_sha", "ls-remote", "BARE"), + ("core/git/git_refs.py", "verify_qualified_ref_sync", "rev-parse", "BARE"), + ("core/install/cmd_runner.py", "run_git", "", "UNRESOLVABLE"), + ("core/install/install_detect.py", "_git", "", "UNRESOLVABLE"), ( "execution/headless/_headless_git.py", "_capture_main_checkout_state", - 86, "for-each-ref", "UNRESOLVABLE", ), - ("execution/headless/_headless_git.py", "_compute_loc_changed", 380, "diff", "BARE"), - (_CLONE_GUARD_PATH, "_prune_stash_overflow", 379, "stash", "UNRESOLVABLE"), - (_CLONE_GUARD_PATH, "_prune_stash_overflow", 401, "stash", "UNRESOLVABLE"), - (_CLONE_GUARD_PATH, "_stash_and_clean", 326, "stash", "UNRESOLVABLE"), - (_CLONE_GUARD_PATH, "revert_contamination", 505, "reset", "BARE"), - (_CLONE_GUARD_PATH, "revert_contamination", 524, "reset", "BARE"), - (_CLONE_GUARD_PATH, "revert_contamination", 539, "reset", "BARE"), - (_CLONE_GUARD_PATH, "validate_pre_session_index", 236, "reset", "BARE"), - (_CLONE_GUARD_PATH, "validate_pre_session_index", 267, "rm", "UNRESOLVABLE"), - ("exploration/identity.py", "_git_value", 112, "", "UNRESOLVABLE"), - ("exploration/snapshot/_capture.py", "_git", 94, "", "UNRESOLVABLE"), - ("fleet/_reset.py", "_delete_reset_branches", 258, "push", "BARE"), + ("execution/headless/_headless_git.py", "_compute_loc_changed", "diff", "BARE"), + ("execution/runtime/clone_guard.py", "_prune_stash_overflow", "stash", "UNRESOLVABLE"), + ("execution/runtime/clone_guard.py", "_prune_stash_overflow", "stash", "UNRESOLVABLE"), + ("execution/runtime/clone_guard.py", "_stash_and_clean", "stash", "UNRESOLVABLE"), + ("execution/runtime/clone_guard.py", "revert_contamination", "reset", "BARE"), + ("execution/runtime/clone_guard.py", "revert_contamination", "reset", "BARE"), + ("execution/runtime/clone_guard.py", "revert_contamination", "reset", "BARE"), + ("execution/runtime/clone_guard.py", "validate_pre_session_index", "reset", "BARE"), + ("execution/runtime/clone_guard.py", "validate_pre_session_index", "rm", "UNRESOLVABLE"), + ("exploration/identity.py", "_git_value", "", "UNRESOLVABLE"), + ("exploration/snapshot/_capture.py", "_git", "", "UNRESOLVABLE"), + ("fleet/_reset.py", "_delete_reset_branches", "push", "BARE"), ( "hooks/_runtime/_git_command_classification.py", "_git_result", - 224, - "", - "UNRESOLVABLE", - ), - ("hooks/guards/remove_clone_guard.py", "_git", 57, "", "UNRESOLVABLE"), - ("recipe/cmd_rpc/_cmd_rpc_guards.py", "_check_regression", 543, "merge-base", "BARE"), - ( - "recipe/cmd_rpc/_cmd_rpc_guards.py", - "_check_regression", - 548, - "", - "UNRESOLVABLE", - ), - ( - "recipe/cmd_rpc/_cmd_rpc_guards.py", - "_check_regression", - 558, "", "UNRESOLVABLE", ), - ("recipe/cmd_rpc/_cmd_rpc_guards.py", "commit_guard", 648, "", "UNRESOLVABLE"), - ("recipe/cmd_rpc/_cmd_rpc_guards.py", "main_repo_guard", 167, "stash", "UNRESOLVABLE"), - (_CMD_RPC_MERGE_PATH, "_attempt_rebase_with_conflict_report", 211, "fetch", "BARE"), - (_CMD_RPC_MERGE_PATH, "_attempt_rebase_with_conflict_report", 212, "fetch", "BARE"), + ("hooks/guards/remove_clone_guard.py", "_git", "", "UNRESOLVABLE"), + ("recipe/cmd_rpc/_cmd_rpc_guards.py", "_check_regression", "", "UNRESOLVABLE"), + ("recipe/cmd_rpc/_cmd_rpc_guards.py", "_check_regression", "", "UNRESOLVABLE"), + ("recipe/cmd_rpc/_cmd_rpc_guards.py", "_check_regression", "merge-base", "BARE"), + ("recipe/cmd_rpc/_cmd_rpc_guards.py", "commit_guard", "", "UNRESOLVABLE"), + ("recipe/cmd_rpc/_cmd_rpc_guards.py", "main_repo_guard", "stash", "UNRESOLVABLE"), ( - _CMD_RPC_MERGE_PATH, + "recipe/cmd_rpc/_cmd_rpc_merge.py", "_attempt_rebase_with_conflict_report", - 215, "", "UNRESOLVABLE", ), - (_CMD_RPC_MERGE_PATH, "_attempt_rebase_with_conflict_report", 216, "rebase", "BARE"), - (_CMD_RPC_MERGE_PATH, "create_persistent_integration", 271, "checkout", "BARE"), - (_CMD_RPC_MERGE_PATH, "create_persistent_integration", 272, "pull", "UNRESOLVABLE"), - (_CMD_RPC_MERGE_PATH, "create_persistent_integration", 274, "push", "BARE"), - (_CMD_RPC_MERGE_PATH, "force_push_and_wait_mergeability", 293, "push", "BARE"), - (_CMD_RPC_MERGE_PATH, "queue_ejected_fix", 123, "fetch", "BARE"), - (_CMD_RPC_MERGE_PATH, "queue_ejected_fix", 126, "rebase", "BARE"), - ("server/_self_revert.py", "_scan_commit", 147, "rev-list", "BARE"), - ("server/_self_revert.py", "_scan_commit", 147, "rev-list", "BARE"), - ("server/_self_revert.py", "_scan_commit", 163, "show", "BARE"), - ("server/_self_revert.py", "detect_self_reverts", 236, "rev-list", "BARE"), - ("server/git.py", "perform_merge", 322, "", "UNRESOLVABLE"), - ("server/git.py", "perform_merge", 327, "", "UNRESOLVABLE"), - ("server/git.py", "perform_merge", 445, "log", "BARE"), - ("server/git.py", "perform_merge", 468, "rebase", "BARE"), - ("server/git.py", "perform_merge", 548, "rev-parse", "BARE"), - ("server/git.py", "perform_merge", 565, "merge-base", "BARE"), - ("server/git.py", "perform_merge", 565, "merge-base", "BARE"), - ("server/git.py", "perform_merge", 617, "merge", "BARE"), - ("core/install/install_detect.py", "_git", 64, "", "UNRESOLVABLE"), - ("server/tools/tools_ci_watch.py", "_auto_trigger_ci", 408, "push", "BARE"), - ("server/tools/tools_git.py", "classify_fix", 177, "fetch", "BARE"), - ("server/tools/tools_git.py", "classify_fix", 196, "diff", "BARE"), + ("recipe/cmd_rpc/_cmd_rpc_merge.py", "_attempt_rebase_with_conflict_report", "fetch", "BARE"), + ("recipe/cmd_rpc/_cmd_rpc_merge.py", "_attempt_rebase_with_conflict_report", "fetch", "BARE"), + ("recipe/cmd_rpc/_cmd_rpc_merge.py", "_attempt_rebase_with_conflict_report", "rebase", "BARE"), + ("recipe/cmd_rpc/_cmd_rpc_merge.py", "create_persistent_integration", "checkout", "BARE"), + ("recipe/cmd_rpc/_cmd_rpc_merge.py", "create_persistent_integration", "pull", "UNRESOLVABLE"), + ("recipe/cmd_rpc/_cmd_rpc_merge.py", "create_persistent_integration", "push", "BARE"), + ("recipe/cmd_rpc/_cmd_rpc_merge.py", "force_push_and_wait_mergeability", "push", "BARE"), + ("recipe/cmd_rpc/_cmd_rpc_merge.py", "queue_ejected_fix", "fetch", "BARE"), + ("recipe/cmd_rpc/_cmd_rpc_merge.py", "queue_ejected_fix", "rebase", "BARE"), + ("server/_self_revert.py", "_scan_commit", "rev-list", "BARE"), + ("server/_self_revert.py", "_scan_commit", "rev-list", "BARE"), + ("server/_self_revert.py", "_scan_commit", "show", "BARE"), + ("server/_self_revert.py", "detect_self_reverts", "rev-list", "BARE"), + ("server/git.py", "perform_merge", "", "UNRESOLVABLE"), + ("server/git.py", "perform_merge", "", "UNRESOLVABLE"), + ("server/git.py", "perform_merge", "log", "BARE"), + ("server/git.py", "perform_merge", "merge", "BARE"), + ("server/git.py", "perform_merge", "merge-base", "BARE"), + ("server/git.py", "perform_merge", "merge-base", "BARE"), + ("server/git.py", "perform_merge", "rebase", "BARE"), + ("server/git.py", "perform_merge", "rev-parse", "BARE"), ( "server/tools/_pre_commit_transaction.py", "_run_pre_commit_transaction", - 67, "", "UNRESOLVABLE", ), - ("server/tools/_self_revert.py", "validate_self_revert_base", 26, "rev-parse", "BARE"), - ("server/tools/_self_revert.py", "validate_self_revert_base", 39, "merge-base", "BARE"), - ( - "server/tools/tools_workspace.py", - "_run_commit_transaction", - 63, - "", - "UNRESOLVABLE", - ), - ("smoke_utils/_git.py", "check_commits_ahead", 179, "rev-list", "BARE"), - ("smoke_utils/_git.py", "check_ref_state", 235, "ls-remote", "BARE"), - ("smoke_utils/_git.py", "check_ref_state", 259, "merge-base", "BARE"), - ("smoke_utils/_git.py", "check_ref_state", 259, "merge-base", "BARE"), - ("smoke_utils/_git.py", "compute_domain_partitions", 45, "diff", "BARE"), - ("smoke_utils/_git.py", "detect_zero_changes", 135, "rev-list", "BARE"), - ("smoke_utils/_review.py", "annotate_pr_diff", 297, "merge-base", "BARE"), - ("smoke_utils/_review.py", "annotate_pr_diff", 297, "merge-base", "BARE"), - ("smoke_utils/_review.py", "annotate_pr_diff", 311, "diff", "BARE"), - ("smoke_utils/_review.py", "annotate_pr_diff", 312, "diff", "BARE"), - ( - "smoke_utils/_review.py", - "annotate_pr_diff._ensure_provider_objects", - 242, - "cat-file", - "BARE", - ), - ( - "smoke_utils/_review.py", - "annotate_pr_diff._ensure_provider_objects", - 272, - "fetch", - "BARE", - ), + ("server/tools/_self_revert.py", "validate_self_revert_base", "merge-base", "BARE"), + ("server/tools/_self_revert.py", "validate_self_revert_base", "rev-parse", "BARE"), + ("server/tools/tools_ci_watch.py", "_auto_trigger_ci", "push", "BARE"), + ("server/tools/tools_git.py", "classify_fix", "diff", "BARE"), + ("server/tools/tools_git.py", "classify_fix", "fetch", "BARE"), + ("server/tools/tools_workspace.py", "_run_commit_transaction", "", "UNRESOLVABLE"), + ("smoke_utils/_git.py", "check_commits_ahead", "rev-list", "BARE"), + ("smoke_utils/_git.py", "check_ref_state", "ls-remote", "BARE"), + ("smoke_utils/_git.py", "check_ref_state", "merge-base", "BARE"), + ("smoke_utils/_git.py", "check_ref_state", "merge-base", "BARE"), + ("smoke_utils/_git.py", "compute_domain_partitions", "diff", "BARE"), + ("smoke_utils/_git.py", "detect_zero_changes", "rev-list", "BARE"), + ("smoke_utils/_review.py", "annotate_pr_diff", "diff", "BARE"), + ("smoke_utils/_review.py", "annotate_pr_diff", "diff", "BARE"), + ("smoke_utils/_review.py", "annotate_pr_diff", "merge-base", "BARE"), + ("smoke_utils/_review.py", "annotate_pr_diff", "merge-base", "BARE"), + ("smoke_utils/_review.py", "annotate_pr_diff._ensure_provider_objects", "cat-file", "BARE"), + ("smoke_utils/_review.py", "annotate_pr_diff._ensure_provider_objects", "fetch", "BARE"), ( "workspace/clone/__init__.py", "_decontaminate_generated_files", - 170, "", "UNRESOLVABLE", ), ( "workspace/clone/__init__.py", "_decontaminate_generated_files", - 178, "", "UNRESOLVABLE", ), - ("workspace/clone/__init__.py", "push_to_remote", 523, "push", "BARE"), + ("workspace/clone/__init__.py", "push_to_remote", "push", "BARE"), ) -_DEFERRED_REF_SITES: dict[tuple[str, str, int, str, str], TrackedDeferral] = {} +_DEFERRED_REF_SITES: dict[tuple[str, str, str, str], TrackedDeferral] = {} @dataclass(frozen=True) @@ -355,9 +320,9 @@ def _is_string_join_call(node: ast.Call) -> bool: class _GitRefCollector(ast.NodeVisitor): - def __init__(self, path: str) -> None: + def __init__(self, path: str, tree: ast.AST) -> None: self._path = path - self._functions: list[str] = [] + self._scope_names = scope_qualnames(tree) self._assignments: list[dict[str, ast.expr | None]] = [{}] self.sites: list[_GitRefSite] = [] @@ -365,27 +330,14 @@ def __init__(self, path: str) -> None: def _scope(self) -> dict[str, ast.expr | None]: return self._assignments[-1] - @property - def _function(self) -> str: - return ".".join(self._functions) or "" - def _visit_scope(self, node: ast.AST) -> None: self._assignments.append({}) self.generic_visit(node) self._assignments.pop() - def visit_FunctionDef(self, node: ast.FunctionDef) -> None: - self._functions.append(node.name) - self._visit_scope(node) - self._functions.pop() - - def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None: - self._functions.append(node.name) - self._visit_scope(node) - self._functions.pop() - - def visit_ClassDef(self, node: ast.ClassDef) -> None: - self._visit_scope(node) + visit_FunctionDef = _visit_scope + visit_AsyncFunctionDef = _visit_scope + visit_ClassDef = _visit_scope def visit_DictComp(self, node: ast.DictComp) -> None: self._visit_scope(node) @@ -528,7 +480,7 @@ def _collect_subcommand_site(self, call: ast.Call, arguments: tuple[ast.expr, .. self.sites.append( _GitRefSite( path=self._path, - function=self._function, + function=self._scope_names[call], lineno=getattr(argument, "lineno", call.lineno), subcommand=subcommand, classification=classification, @@ -560,7 +512,7 @@ def _record_unresolvable(self, call: ast.Call, subcommand: str) -> None: self.sites.append( _GitRefSite( path=self._path, - function=self._function, + function=self._scope_names[call], lineno=call.lineno, subcommand=subcommand, classification="UNRESOLVABLE", @@ -569,7 +521,7 @@ def _record_unresolvable(self, call: ast.Call, subcommand: str) -> None: def _scan_tree(tree: ast.AST, path: str) -> tuple[_GitRefSite, ...]: - collector = _GitRefCollector(path) + collector = _GitRefCollector(path, tree) collector.visit(tree) return tuple(collector.sites) @@ -583,8 +535,8 @@ def _scan_source() -> tuple[_GitRefSite, ...]: return tuple(sites) -def _inventory_key(site: _GitRefSite) -> tuple[str, str, int, str, str]: - return site.path, site.function, site.lineno, site.subcommand, site.classification +def _inventory_key(site: _GitRefSite) -> tuple[str, str, str, str]: + return site.path, site.function, site.subcommand, site.classification def _live_violations() -> tuple[_GitRefSite, ...]: @@ -592,10 +544,10 @@ def _live_violations() -> tuple[_GitRefSite, ...]: def _assert_inventory_matches( - observed: tuple[tuple[str, str, int, str, str], ...], - expected: tuple[tuple[str, str, int, str, str], ...], + observed: tuple[tuple[str, str, str, str], ...], + expected: tuple[tuple[str, str, str, str], ...], ) -> None: - assert sorted(observed) == sorted(expected) + assert Counter(observed) == Counter(expected) def test_no_unqualified_ref_site_outside_inventory() -> None: @@ -772,8 +724,38 @@ def test_path_separator_stops_revision_scan() -> None: assert sites == () +def test_inventory_key_is_line_free() -> None: + source = 'def push_it(base_branch):\n subprocess.run(["git", "rev-parse", base_branch])\n' + original = _scan_tree(ast.parse(source), "synthetic.py") + shifted = _scan_tree(ast.parse("\n" * 25 + source), "synthetic.py") + + assert len(original) == len(shifted) == 1 + assert [site.classification for site in original] == ["BARE"] + assert [site.classification for site in shifted] == ["BARE"] + assert shifted[0].lineno == original[0].lineno + 25 + assert _inventory_key(original[0]) == _inventory_key(shifted[0]) + assert len(_inventory_key(original[0])) == 4 + + +def test_same_named_methods_in_different_classes_have_distinct_keys() -> None: + source = ( + "class A:\n" + " def run(self, base):\n" + ' subprocess.run(["git", "rev-parse", base])\n' + "class B:\n" + " def run(self, base):\n" + ' subprocess.run(["git", "rev-parse", base])\n' + ) + sites = _scan_tree(ast.parse(source), "synthetic.py") + + assert len(sites) == 2 + keys = [_inventory_key(site) for site in sites] + assert [site.function for site in sites] == ["A.run", "B.run"] + assert keys[0][1] != keys[1][1] + + def test_missing_expected_site_is_caught() -> None: - observed = (("synthetic.py", "", 1, "diff", "BARE"),) + observed = (("synthetic.py", MODULE_SCOPE, "diff", "BARE"),) with pytest.raises(AssertionError): _assert_inventory_matches(observed, ()) with pytest.raises(AssertionError): diff --git a/tests/arch/test_infra_source_scanner_selection.py b/tests/arch/test_infra_source_scanner_selection.py new file mode 100644 index 0000000000..697e4f09ca --- /dev/null +++ b/tests/arch/test_infra_source_scanner_selection.py @@ -0,0 +1,245 @@ +"""Selection contract: an infra or hooks test that scans the source package or hooks reads +code by path, so no import edge selects it; it must be named in the matching +unconditional set. Detects `.rglob` on an imported HOOKS_DIR or a `Path(__file__)`-rooted +chain, directly or via up to four name bindings. Conservative syntax matching may +over-select fixture roots or names rebound in another scope. +""" + +from __future__ import annotations + +import ast +import copy +import textwrap +from pathlib import Path + +import pytest + +from tests._test_filter import _HOOKS_UNCONDITIONAL_FILES, _INFRA_UNCONDITIONAL_FILES + +pytestmark = [pytest.mark.layer("arch"), pytest.mark.small] + + +def _bindings(tree: ast.AST) -> dict[str, list[ast.expr]]: + bindings: dict[str, list[ast.expr]] = {} + for node in ast.walk(tree): + if isinstance(node, ast.Assign) and len(node.targets) == 1: + target = node.targets[0] + if isinstance(target, ast.Name): + bindings.setdefault(target.id, []).append(node.value) + elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + if node.value is not None: + bindings.setdefault(node.target.id, []).append(node.value) + return bindings + + +def _receiver_forms(receiver: ast.expr, bindings: dict[str, list[ast.expr]]) -> list[ast.expr]: + forms = [receiver] + frontier = [receiver] + seen = {ast.dump(receiver)} + for _ in range(4): + next_frontier: list[ast.expr] = [] + for form in frontier: + names = sorted({node.id for node in ast.walk(form) if isinstance(node, ast.Name)}) + for name in names: + for value in bindings.get(name, []): + + class _ReplaceName(ast.NodeTransformer): + def visit_Name(self, node: ast.Name) -> ast.AST: + if node.id == name: + return copy.deepcopy(value) + return node + + expanded = _ReplaceName().visit(copy.deepcopy(form)) + key = ast.dump(expanded) + if key not in seen: + seen.add(key) + next_frontier.append(expanded) + if not next_frontier: + break + forms.extend(next_frontier) + frontier = next_frontier + return forms + + +def _is_package_root(expression: ast.expr, hook_root_names: set[str]) -> bool: + if isinstance(expression, ast.Name) and expression.id in hook_root_names: + return True + suffix: list[str] = [] + base = expression + while isinstance(base, ast.BinOp) and isinstance(base.op, ast.Div): + if not isinstance(base.right, ast.Constant) or not isinstance(base.right.value, str): + break + suffix.append(base.right.value) + base = base.left + return any( + isinstance(node, ast.Name) and node.id == "__file__" for node in ast.walk(base) + ) and tuple(reversed(suffix)) in { + ("src",), + ("src", "autoskillit"), + ("src", "autoskillit", "hooks"), + } + + +def _root_scan_lines(tree: ast.AST) -> list[int]: + bindings = _bindings(tree) + hook_root_names = { + alias.asname or alias.name + for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom) and node.module == "autoskillit.hook_registry" + for alias in node.names + if alias.name == "HOOKS_DIR" + } + lines: list[int] = [] + for call in ast.walk(tree): + if ( + isinstance(call, ast.Call) + and isinstance(call.func, ast.Attribute) + and call.func.attr == "rglob" + and any( + _is_package_root(form, hook_root_names) + for form in _receiver_forms(call.func.value, bindings) + ) + ): + lines.append(call.lineno) + return sorted(lines) + + +@pytest.mark.parametrize( + ("source", "expected_hits"), + [ + pytest.param( + """ + from pathlib import Path + TESTS_ROOT = Path(__file__).parent + SRC = Path(__file__).resolve().parents[2] / "src" / "autoskillit" + TESTS_ROOT.rglob("test_*.py") + SRC.rglob("*.py") + """, + 1, + id="bound-source-package-root", + ), + pytest.param( + """ + from pathlib import Path + REPO_ROOT = Path(__file__).resolve().parents[2] + (REPO_ROOT / "src").rglob("*.py") + """, + 1, + id="repository-root-plus-src", + ), + pytest.param( + """ + from pathlib import Path + def scan(): + root = Path(__file__).parents[2] / "src" / "autoskillit" + return list(root.rglob("*.py")) + """, + 1, + id="function-local-binding", + ), + pytest.param( + """ + from pathlib import Path + _REPO_ROOT = Path(__file__).resolve().parents[2] + _SRC_ROOT = _REPO_ROOT / "src" / "autoskillit" + _SRC_ROOT.rglob("*.py") + """, + 1, + id="two-hop-binding", + ), + pytest.param( + """ + from pathlib import Path + SRC = Path(__file__).resolve().parents[2] / "src" / "autoskillit" + (SRC / "hooks").rglob("*.py") + """, + 1, + id="hooks-source-root", + ), + pytest.param( + """ + from autoskillit.hook_registry import HOOKS_DIR as hooks_root + hooks_root.rglob("*.py") + """, + 1, + id="imported-hooks-root", + ), + pytest.param( + """ + from pathlib import Path + SRC = Path(__file__).resolve().parents[2] / "src" / "autoskillit" + (SRC / "server").rglob("*.py") + """, + 0, + id="non-root-subdirectory", + ), + pytest.param( + """ + from pathlib import Path + SRC = Path(__file__).resolve().parents[2] / "src" / "autoskillit" + SRC.glob("*.py") + """, + 0, + id="glob-is-not-rglob", + ), + pytest.param( + """ + from pathlib import Path + TESTS_ROOT = Path(__file__).parent + SRC = Path(__file__).resolve().parents[2] / "src" / "autoskillit" + TESTS_ROOT.rglob("test_*.py") + """, + 0, + id="tests-root-does-not-match", + ), + pytest.param( + """ + from pathlib import Path + SRC = Path("__file__").parents[2] / "src" / "autoskillit" + SRC.rglob("*.py") + """, + 0, + id="file-spelling-is-not-file-name", + ), + ], +) +def test_detector_classifies_receivers(source: str, expected_hits: int) -> None: + tree = ast.parse(textwrap.dedent(source)) + assert len(_root_scan_lines(tree)) == expected_hits + + +def test_infra_and_hooks_source_scanners_are_unconditional() -> None: + tests_root = Path(__file__).resolve().parents[1] + scanner_sets = ( + (tests_root / "infra", _INFRA_UNCONDITIONAL_FILES), + (tests_root / "hooks", _HOOKS_UNCONDITIONAL_FILES), + ) + scanner_names: dict[str, set[str]] = {} + violations: list[str] = [] + for directory, unconditional_files in scanner_sets: + hits: set[str] = set() + for path in sorted(directory.glob("test_*.py")): + try: + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + except SyntaxError as error: + pytest.fail(f"Could not parse {path}: {error}") + if _root_scan_lines(tree): + hits.add(path.name) + scanner_names[directory.name] = hits + set_name = ( + "_INFRA_UNCONDITIONAL_FILES" + if directory.name == "infra" + else "_HOOKS_UNCONDITIONAL_FILES" + ) + violations.extend( + f"add `{name}` to `{set_name}` in tests/_test_filter.py" + for name in sorted(hits - unconditional_files) + ) + + required_infra_scanners = { + "test_schema_version_convention.py", + "test_plugin_source_ratchets.py", + } + assert required_infra_scanners <= scanner_names["infra"] + assert "test_hook_registration_coverage.py" in scanner_names["hooks"] + assert not violations, "\n".join(violations) diff --git a/tests/arch/test_managed_join_authority_reachability.py b/tests/arch/test_managed_join_authority_reachability.py index a322b68435..6946fc3ace 100644 --- a/tests/arch/test_managed_join_authority_reachability.py +++ b/tests/arch/test_managed_join_authority_reachability.py @@ -8,7 +8,7 @@ import pytest -from tests.contracts._ast_helpers import call_name +from tests._ast_helpers import call_name pytestmark = [pytest.mark.layer("arch"), pytest.mark.small] diff --git a/tests/contracts/test_ambient_env_surface.py b/tests/contracts/test_ambient_env_surface.py index b137afb113..208e28e357 100644 --- a/tests/contracts/test_ambient_env_surface.py +++ b/tests/contracts/test_ambient_env_surface.py @@ -14,8 +14,9 @@ from __future__ import annotations import ast -from collections.abc import Iterator -from functools import lru_cache +from collections import Counter +from collections.abc import Iterator, Mapping +from functools import cache, lru_cache from pathlib import Path import pytest @@ -25,26 +26,81 @@ from tests import _ambient_env_surface as ambient_env_surface from tests._ambient_env_surface import ( AMBIENT_ENV_DISPOSITIONS, + DYNAMIC_READ_EXEMPTION_COUNTS, DYNAMIC_READ_EXEMPTIONS, FORWARDING_SITES, production_env_read_surface, production_env_write_surface, ) +from tests._ast_helpers import MODULE_SCOPE, count_differences, scope_qualnames from tests.arch._helpers import SRC_ROOT pytestmark = [pytest.mark.small] -@pytest.fixture +@pytest.fixture(autouse=True) def cleared_ambient_env_scan_caches() -> Iterator[None]: - """Clear both ambient-environment scan caches around each requesting test.""" + """Clear ambient-environment and scope caches around each test.""" ambient_env_surface._cached_production_env_read_surface.cache_clear() _cached_ambient_env_marker_names.cache_clear() + _scope_owners.cache_clear() try: yield finally: ambient_env_surface._cached_production_env_read_surface.cache_clear() _cached_ambient_env_marker_names.cache_clear() + _scope_owners.cache_clear() + + +@cache +def _scope_owners(root: Path, file: str) -> dict[ast.AST, str]: + path = root / file + tree = ast.parse(path.read_text(encoding="utf-8"), filename=file) + return scope_qualnames(tree) + + +def _site_scope(file: str, line: int, src_root: Path) -> str: + candidates: list[tuple[int, str]] = [] + for node, scope in _scope_owners(src_root, file).items(): + if not isinstance(node, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)): + continue + start = min([node.lineno, *(decorator.lineno for decorator in node.decorator_list)]) + end = node.end_lineno if node.end_lineno is not None else node.lineno + if start <= line <= end: + candidates.append((scope.count("."), scope)) + if not candidates: + return MODULE_SCOPE + return max(candidates, key=lambda candidate: candidate[0])[1] + + +def _site_key( + record: ambient_env_surface.UnresolvedRead | ambient_env_surface.ForwardingSite, + src_root: Path, +) -> str: + return f"{record.file}::{_site_scope(record.file, record.line, src_root)}" + + +def _site_registry_problems(live: Counter[str], expected: Mapping[str, int]) -> list[str]: + problems = [ + f"{key}: count must be at least 1 (got {count})" + for key, count in expected.items() + if count < 1 + ] + for key, expected_count, live_count in count_differences(expected, live): + if key in expected and expected_count < 1: + continue + if live_count > expected_count: + problems.append( + f"{key}: undeclared count {live_count} exceeds expected {expected_count}; " + "add the key or raise its count with justification" + ) + elif live_count < expected_count: + problems.append( + f"{key}: stale count {live_count} is below expected {expected_count}; " + "if moved or renamed, update the key; " + "if removed, delete the entry or lower its count" + ) + return sorted(problems) def test_production_read_surface_caches_only_canonical_root( @@ -121,6 +177,78 @@ def test_scanner_reports_unresolved_dynamic_reads(tmp_path: Path) -> None: assert matches[0].expression == "some_runtime_name" +def test_registry_key_survives_line_shift( + tmp_path: Path, + cleared_ambient_env_scan_caches: None, +) -> None: + source = "import os\n\n\ndef read(name: str) -> str | None:\n return os.environ.get(name)\n" + original_root = tmp_path / "original" + shifted_root = tmp_path / "shifted" + original_root.mkdir() + shifted_root.mkdir() + (original_root / "synthetic_shift.py").write_text(source, encoding="utf-8") + (shifted_root / "synthetic_shift.py").write_text("\n" * 30 + source, encoding="utf-8") + + original = production_env_read_surface(original_root).unresolved[0] + shifted = production_env_read_surface(shifted_root).unresolved[0] + + assert _site_key(original, original_root) == "synthetic_shift.py::read" + assert shifted.line == original.line + 30 + assert _site_key(shifted, shifted_root) == _site_key(original, original_root) + + +def test_site_scope_is_class_qualified( + tmp_path: Path, + cleared_ambient_env_scan_caches: None, +) -> None: + source = ( + "import os\nimport subprocess\n\n" + "def decorate(value: object) -> object:\n return value\n\n" + "class Runner:\n" + " @decorate(os.getenv(env_name))\n" + " def spawn(self) -> None:\n" + ' subprocess.run(["true"], env=os.environ)\n' + ) + (tmp_path / "synthetic_fw.py").write_text(source, encoding="utf-8") + surface = production_env_read_surface(tmp_path) + forwarding = [site for site in surface.forwarding_sites if site.file == "synthetic_fw.py"] + unresolved = [site for site in surface.unresolved if site.file == "synthetic_fw.py"] + + assert forwarding + assert unresolved + assert _site_key(forwarding[0], tmp_path) == "synthetic_fw.py::Runner.spawn" + assert _site_key(unresolved[0], tmp_path) == "synthetic_fw.py::Runner.spawn" + + +@pytest.mark.parametrize( + ("live_items", "expected", "expected_key", "expected_message"), + [ + ({"a": 2}, {"a": 1}, "a", "undeclared"), + ({"a": 1, "b": 1}, {"a": 1}, "b", "undeclared"), + ({"a": 1}, {"a": 1, "b": 1}, "b", "stale"), + ({"a": 1}, {"a": 2}, "a", "stale"), + ({}, {"a": 0}, "a", "count must be at least 1"), + ({}, {"a": -1}, "a", "count must be at least 1"), + ({"a": 1}, {"a": 1}, "", ""), + ], + ids=["extra-count", "extra-key", "missing-key", "missing-count", "zero", "negative", "equal"], +) +def test_site_registry_diff_is_per_key_and_counted( + live_items: dict[str, int], + expected: Mapping[str, int], + expected_key: str, + expected_message: str, +) -> None: + problems = _site_registry_problems(Counter(live_items), expected) + + if not expected_message: + assert problems == [] + return + assert len(problems) == 1 + assert expected_key in problems[0] + assert expected_message in problems[0] + + _KEYWORD_ONLY_READ_CASES: tuple[tuple[str, str, str], ...] = ( ( "getenv_kwonly", @@ -338,36 +466,27 @@ def test_scanner_reports_wholesale_forwarding_sites( def test_every_forwarding_site_is_declared() -> None: surface = production_env_read_surface(SRC_ROOT) - undeclared = {f"{s.file}:{s.line}" for s in surface.forwarding_sites} - set(FORWARDING_SITES) - assert not undeclared, ( - f"Forwarding sites missing a FORWARDING_SITES entry: {sorted(undeclared)}" - ) - - -def test_no_orphan_forwarding_sites() -> None: - surface = production_env_read_surface(SRC_ROOT) - live = {f"{s.file}:{s.line}" for s in surface.forwarding_sites} - orphans = set(FORWARDING_SITES) - live - assert not orphans, ( - f"Stale FORWARDING_SITES entries no longer in the surface: {sorted(orphans)}" - ) + live = Counter(_site_key(site, SRC_ROOT) for site in surface.forwarding_sites) + expected = {key: 1 for key in FORWARDING_SITES} + problems = _site_registry_problems(live, expected) + assert not problems, f"FORWARDING_SITES registry mismatch: {problems}" def test_every_unresolved_read_is_declared() -> None: surface = production_env_read_surface(SRC_ROOT) - undeclared = {f"{u.file}:{u.line}" for u in surface.unresolved} - set(DYNAMIC_READ_EXEMPTIONS) - assert not undeclared, ( - f"Unresolved dynamic reads missing a DYNAMIC_READ_EXEMPTIONS entry: {sorted(undeclared)}" - ) + live = Counter(_site_key(site, SRC_ROOT) for site in surface.unresolved) + expected = {key: DYNAMIC_READ_EXEMPTION_COUNTS.get(key, 1) for key in DYNAMIC_READ_EXEMPTIONS} + problems = _site_registry_problems(live, expected) + assert not problems, f"DYNAMIC_READ_EXEMPTIONS registry mismatch: {problems}" -def test_no_orphan_dynamic_read_exemptions() -> None: - surface = production_env_read_surface(SRC_ROOT) - live = {f"{u.file}:{u.line}" for u in surface.unresolved} - orphans = set(DYNAMIC_READ_EXEMPTIONS) - live - assert not orphans, ( - f"Stale DYNAMIC_READ_EXEMPTIONS entries no longer in the surface: {sorted(orphans)}" +def test_dynamic_read_exemption_counts_are_wellformed() -> None: + orphaned = set(DYNAMIC_READ_EXEMPTION_COUNTS) - set(DYNAMIC_READ_EXEMPTIONS) + assert not orphaned, ( + f"Count overrides lack DYNAMIC_READ_EXEMPTIONS entries: {sorted(orphaned)}" ) + invalid = {key: count for key, count in DYNAMIC_READ_EXEMPTION_COUNTS.items() if count < 2} + assert not invalid, f"Count overrides must be at least 2: {invalid}" def test_production_env_surface_scan_parses_all_source_files() -> None: diff --git a/tests/contracts/test_managed_route_pairing.py b/tests/contracts/test_managed_route_pairing.py index 181f4f013e..6fdf29e435 100644 --- a/tests/contracts/test_managed_route_pairing.py +++ b/tests/contracts/test_managed_route_pairing.py @@ -11,7 +11,7 @@ from autoskillit.execution.backends import BACKEND_REGISTRY from autoskillit.execution.backends._codex_hooks import managed_codex_route_digest from autoskillit.hook_registry import HOOK_REGISTRY_HASH -from tests.contracts._ast_helpers import callers_by_function +from tests._ast_helpers import callers_by_function pytestmark = [pytest.mark.layer("contracts"), pytest.mark.small] diff --git a/tests/core/test_type_constants_response_backstop.py b/tests/core/test_type_constants_response_backstop.py index 149bf97d5c..f4725a82e3 100644 --- a/tests/core/test_type_constants_response_backstop.py +++ b/tests/core/test_type_constants_response_backstop.py @@ -103,5 +103,5 @@ def test_recipe_execution_install_site_registry_digest_is_canonical() -> None: ) assert ( RECIPE_EXECUTION_INSTALL_SITE_REGISTRY_DIGEST - == "23e3043b50bafb0aeb8c60296b2d44a6f7bb5e92abf78b92d1f1685603b045e6" + == "30297bfa2028fc244c740490f91737fe12fe52b7928c579cb2209cba3d3db985" ) diff --git a/tests/execution/test_process_stop_hook_join.py b/tests/execution/test_process_stop_hook_join.py index c80832dee5..95ab6d34d7 100644 --- a/tests/execution/test_process_stop_hook_join.py +++ b/tests/execution/test_process_stop_hook_join.py @@ -410,14 +410,14 @@ async def test_schedule_wakeup_defers_release_from_native_terminal(tmp_path, sto emit_stdout({"type": "assistant", "message": {"content": [ {"type": "tool_use", "id": "wake", "name": "ScheduleWakeup", "input": {}} ]}}) - touch("a_at") + (root / "a_at").write_text(str(time.monotonic())) emit_stdout({"type": "result", "result": "done\\n" + marker, "session_id": sid}) time.sleep(30) """, child_deferral_ceiling=2.0, natural_exit_grace_seconds=0.2, ) - ended = time.time() + ended = time.monotonic() assert result.schedule_wakeup_violation is True assert result.completion_ceiling_expired is True diff --git a/tests/infra/test_canonical_json_producer_convention.py b/tests/infra/test_canonical_json_producer_convention.py index 5d9ed71ada..20cf87821d 100644 --- a/tests/infra/test_canonical_json_producer_convention.py +++ b/tests/infra/test_canonical_json_producer_convention.py @@ -1,68 +1,74 @@ -"""Producer ratchet: enforce that every strict-canonical JSON consumer has a -registered, verified server-side producer. - -Scans src/autoskillit/ for decode_versioned_json_bytes(..., require_canonical=True) -call sites. Each such site must be registered in _CANONICAL_JSON_ARTIFACT_REGISTRY, -pointing at a producer function that either calls write_canonical_versioned_json or -exclusively writes already-canonical bytes, and at any child-facing SKILL.md section -that names the producer's MCP tool by symbol. This closes the -gap #4406 exhibited: a Python consumer demanding canonical bytes with no mechanical -guarantee that its LLM-agent producer emits them. +"""Ratchet strict canonical JSON consumers to their registered producers. + +Consumer identity is the source path plus its dotted enclosing definition scope, +independent of line numbers. A decode call is strict when require_canonical is true, +not a literal false, or supplied through **kwargs. Each strict consumer scope must be +registered in _CANONICAL_JSON_ARTIFACT_REGISTRY, pointing at a producer function that +either calls write_canonical_versioned_json or exclusively writes already-canonical +bytes, and at any child-facing SKILL.md section that names the producer's MCP tool by +symbol. This closes the gap #4406 exhibited: a Python consumer demanding canonical +bytes with no mechanical guarantee that its LLM-agent producer emits them. """ from __future__ import annotations import ast +from collections import Counter +from collections.abc import Mapping from pathlib import Path from typing import NamedTuple import pytest +from tests._ast_helpers import MODULE_SCOPE, call_name, count_differences, scope_qualnames + pytestmark = [pytest.mark.layer("infra"), pytest.mark.medium] class CanonicalArtifactDef(NamedTuple): - consumer_site: tuple[str, int | tuple[int, ...]] + consumer_path: str + consumer_function: str producer_symbol: str producer_path: str producer_function: str skill_md_refs: tuple[tuple[str, int, int], ...] -def _is_literal_true(node: ast.expr) -> bool: - return isinstance(node, ast.Constant) and node.value is True +_REPO_ROOT = Path(__file__).resolve().parents[2] +_SRC_ROOT = _REPO_ROOT / "src" / "autoskillit" -def _scan_require_canonical_consumer_sites() -> set[tuple[str, int]]: - """AST-scan src/autoskillit/ for decode_versioned_json_bytes(require_canonical=True). +def _is_strict(call: ast.Call) -> bool: + """Return whether a decode call may require canonical bytes.""" + if any(keyword.arg is None for keyword in call.keywords): + return True + for keyword in call.keywords: + if keyword.arg == "require_canonical": + return not (isinstance(keyword.value, ast.Constant) and keyword.value.value is False) + return False - Returns set of (relative_path, line_number) for call sites that pass - require_canonical=True as a keyword argument. - """ - src_root = Path(__file__).resolve().parents[2] / "src" / "autoskillit" - sites: set[tuple[str, int]] = set() - for py_file in src_root.rglob("*.py"): - try: - tree = ast.parse(py_file.read_text(), filename=str(py_file)) - except SyntaxError: - continue +def _decode_call_sites(tree: ast.AST, relpath: str) -> Counter[tuple[str, str, bool]]: + """Count decode calls by relative path, enclosing scope, and strictness.""" + scopes = scope_qualnames(tree) + sites: Counter[tuple[str, str, bool]] = Counter() + for node in ast.walk(tree): + if isinstance(node, ast.Call) and call_name(node) == "decode_versioned_json_bytes": + sites[(relpath, scopes[node], _is_strict(node))] += 1 + return sites - for node in ast.walk(tree): - if not isinstance(node, ast.Call): - continue - func = node.func - is_decode_call = ( - isinstance(func, ast.Name) and func.id == "decode_versioned_json_bytes" - ) or (isinstance(func, ast.Attribute) and func.attr == "decode_versioned_json_bytes") - if not is_decode_call: - continue - for kw in node.keywords: - if kw.arg == "require_canonical" and _is_literal_true(kw.value): - rel = str(py_file.relative_to(src_root.parent.parent)) - sites.add((rel, node.lineno)) - break +def _scan_decode_call_sites() -> Counter[tuple[str, str, bool]]: + """Scan source decode calls; an unparseable file fails here with its filename. + + This check is independent of lint and typecheck and does not rely on their presence + or execution order. + """ + sites: Counter[tuple[str, str, bool]] = Counter() + for path in sorted(_SRC_ROOT.rglob("*.py")): + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + relative_path = str(path.relative_to(_REPO_ROOT)) + sites.update(_decode_call_sites(tree, relative_path)) return sites @@ -75,27 +81,13 @@ def _find_function_by_name( return None -def _find_call_at_line(tree: ast.Module, lineno: int, func_name: str) -> ast.Call | None: - for node in ast.walk(tree): - if not isinstance(node, ast.Call) or node.lineno != lineno: - continue - func = node.func - matches = (isinstance(func, ast.Name) and func.id == func_name) or ( - isinstance(func, ast.Attribute) and func.attr == func_name - ) - if matches: - return node - return None - - def _scan_child_tool_authority_producer_sites() -> set[tuple[str, int, str]]: """Find authority construction or canonicalization in child-facing MCP tools.""" - repo_root = Path(__file__).resolve().parents[2] - tools_root = repo_root / "src" / "autoskillit" / "server" / "tools" + tools_root = _SRC_ROOT / "server" / "tools" sites: set[tuple[str, int, str]] = set() for source_path in tools_root.glob("tools_*.py"): tree = ast.parse(source_path.read_text(), filename=str(source_path)) - relative_path = str(source_path.relative_to(repo_root)) + relative_path = str(source_path.relative_to(_REPO_ROOT)) for node in ast.walk(tree): if isinstance(node, ast.Call): func = node.func @@ -121,42 +113,48 @@ def _scan_child_tool_authority_producer_sites() -> set[tuple[str, int, str]]: _CANONICAL_JSON_ARTIFACT_REGISTRY: dict[str, CanonicalArtifactDef] = { "authority": CanonicalArtifactDef( - consumer_site=("src/autoskillit/core/audit/audit_cycle_verifier.py", (504,)), + consumer_path="src/autoskillit/core/audit/audit_cycle_verifier.py", + consumer_function="AuditCycleVerifier.decode_authority", producer_symbol="_write_or_verify", producer_path=_MATERIALIZER_PRODUCER_PATH, producer_function="_write_or_verify", skill_md_refs=(), ), "disposition_report": CanonicalArtifactDef( - consumer_site=("src/autoskillit/core/audit/audit_cycle_verifier.py", (521,)), + consumer_path="src/autoskillit/core/audit/audit_cycle_verifier.py", + consumer_function="AuditCycleVerifier.load_report", producer_symbol="write_audit_disposition_bundle", producer_path=_TYPED_PRODUCER_MODULE, producer_function="_write_disposition_report", skill_md_refs=(("src/autoskillit/skills_extended/make-plan/SKILL.md", 386, 398),), ), "inventory": CanonicalArtifactDef( - consumer_site=("src/autoskillit/core/audit/audit_cycle_verifier.py", (660,)), + consumer_path="src/autoskillit/core/audit/audit_cycle_verifier.py", + consumer_function="AuditCycleVerifier._decode_inventory_requirement_ids", producer_symbol="_write_or_verify", producer_path=_MATERIALIZER_PRODUCER_PATH, producer_function="_write_or_verify", skill_md_refs=(), ), "plan_association": CanonicalArtifactDef( - consumer_site=("src/autoskillit/recipe/cmd_rpc/_cmd_rpc_guards.py", 271), + consumer_path="src/autoskillit/recipe/cmd_rpc/_cmd_rpc_guards.py", + consumer_function="_load_associated_disposition", producer_symbol="write_audit_disposition_bundle", producer_path=_TYPED_PRODUCER_MODULE, producer_function="_write_plan_association", skill_md_refs=(("src/autoskillit/skills_extended/make-plan/SKILL.md", 386, 398),), ), "audit_semantic_result": CanonicalArtifactDef( - consumer_site=("src/autoskillit/core/audit/audit_semantic_codec.py", 423), + consumer_path="src/autoskillit/core/audit/audit_semantic_codec.py", + consumer_function="load_audit_semantic_result", producer_symbol="write_audit_semantic_result", producer_path=_TYPED_PRODUCER_MODULE, producer_function="_write_semantic_result", skill_md_refs=(("src/autoskillit/skills_extended/audit-impl/SKILL.md", 103, 117),), ), "standalone_audit_evidence": CanonicalArtifactDef( - consumer_site=("src/autoskillit/core/audit/audit_semantic_codec.py", 470), + consumer_path="src/autoskillit/core/audit/audit_semantic_codec.py", + consumer_function="load_standalone_audit_evidence", producer_symbol="write_standalone_audit_evidence", producer_path=_TYPED_PRODUCER_MODULE, producer_function="_write_standalone_evidence", @@ -165,12 +163,14 @@ def _scan_child_tool_authority_producer_sites() -> set[tuple[str, int, str]]: } -# Documented exception: consumer sites that intentionally decode non-canonical -# versioned JSON. Paired with the writer that produces them so a future accidental -# flip to require_canonical=True is caught by test_non_canonical_exceptions_below. -_NON_CANONICAL_JSON_EXCEPTIONS: dict[str, tuple[tuple[str, int], str]] = { +# Documented exception: scopes that intentionally decode non-canonical versioned JSON. +# Their reasons cover every non-canonical read in the scope. +_NON_CANONICAL_JSON_EXCEPTIONS: dict[str, tuple[tuple[str, str], str]] = { "closure_report.json": ( - ("src/autoskillit/core/audit/closure_verifier.py", 93), + ( + "src/autoskillit/core/audit/closure_verifier.py", + "verify_closure_report", + ), "Closure reports are written by write_versioned_json (see the " "_write_report fixture in tests/core/test_closure_verifier.py), not " "write_canonical_versioned_json — no content-addressed tamper-evidence " @@ -179,6 +179,81 @@ def _scan_child_tool_authority_producer_sites() -> set[tuple[str, int, str]]: } +def _consumer_registry_problems( + registry: Mapping[str, CanonicalArtifactDef], + sites: Counter[tuple[str, str, bool]], +) -> list[str]: + """Compare strict call counts to registered path-and-scope consumer counts.""" + strict = Counter( + { + (path, qualname): count + for (path, qualname, is_strict), count in sites.items() + if is_strict + } + ) + expected = Counter( + (entry.consumer_path, entry.consumer_function) for entry in registry.values() + ) + problems: list[str] = [] + + for (path, qualname), expected_count, observed_count in count_differences(expected, strict): + if expected_count > observed_count: + kinds = sorted( + kind + for kind, entry in registry.items() + if (entry.consumer_path, entry.consumer_function) == (path, qualname) + ) + problems.append( + f"{', '.join(kinds)}: {path}::{qualname} is missing count: " + f"{expected_count - observed_count}; " + "if the consumer moved or was renamed, update consumer_path/consumer_function; " + "if it no longer exists, delete the entry" + ) + else: + problems.append( + f"{path}::{qualname} has unregistered count: {observed_count - expected_count}; " + "register a CanonicalArtifactDef in _CANONICAL_JSON_ARTIFACT_REGISTRY " + "naming its producer" + ) + return sorted(problems) + + +def _non_canonical_exception_problems( + exceptions: Mapping[str, tuple[tuple[str, str], str]], + sites: Counter[tuple[str, str, bool]], +) -> list[str]: + """Check scope-wide exceptions for a reason, decode call, and no strict calls. + + An exception requires at least one decode call and rejects any strict call in its + scope. It does not associate individual non-strict calls with separate artifact + names or enforce their multiplicity; the reason must cover the scope's non-canonical + reads. + """ + problems: list[str] = [] + for artifact_name, ((path, qualname), reason) in exceptions.items(): + if not reason: + problems.append(f"{artifact_name} has an empty exception reason") + scoped_calls = [ + (is_strict, count) + for (site_path, site_qualname, is_strict), count in sites.items() + if (site_path, site_qualname) == (path, qualname) and count + ] + if not scoped_calls: + problems.append( + f"{artifact_name}: no decode_versioned_json_bytes call in " + f"{path}::{qualname} — update the anchor if it moved, delete the entry if removed" + ) + elif any(is_strict for is_strict, _count in scoped_calls): + problems.append( + f"{artifact_name}: {path}::{qualname} now counts as strict " + "(require_canonical=True, a nonliteral argument, or **kwargs) — this pairing " + "is documented as non-canonical in _NON_CANONICAL_JSON_EXCEPTIONS; " + "register its producer in _CANONICAL_JSON_ARTIFACT_REGISTRY if strictness is " + "intended, otherwise restore an explicitly non-canonical call" + ) + return sorted(problems) + + class TestCanonicalJsonProducerConvention: def test_authority_has_one_materializer_owned_producer(self): authority = _CANONICAL_JSON_ARTIFACT_REGISTRY["authority"] @@ -188,48 +263,11 @@ def test_authority_has_one_materializer_owned_producer(self): assert _scan_child_tool_authority_producer_sites() == set() def test_require_canonical_consumers_have_registered_producers(self): - """Every require_canonical=True consumer site must have a registry entry. - - consumer_site may be ``(file, line: int)`` or ``(file, lines: tuple[int, ...])``. - The tuple form lets a single entry span the line numbers of multiple - development branches whose refactors shift the call site but do not - change its meaning — the test then asserts the scanner finds at least - one acceptable line per registered file and vice versa. - """ - current = _scan_require_canonical_consumer_sites() - - registered_per_file: dict[str, set[int]] = {} - for entry in _CANONICAL_JSON_ARTIFACT_REGISTRY.values(): - file, lines = entry.consumer_site - if isinstance(lines, int): - registered_per_file.setdefault(file, set()).add(lines) - else: - registered_per_file.setdefault(file, set()).update(lines) - - current_per_file: dict[str, set[int]] = {} - for file, line in current: - current_per_file.setdefault(file, set()).add(line) - - added_files = sorted(set(current_per_file) - set(registered_per_file)) - stale = [ - file - for file, acceptable in registered_per_file.items() - if not (acceptable & current_per_file.get(file, set())) - ] - - msg_parts = [] - if added_files: - msg_parts.append( - "New require_canonical=True consumer files found (register a " - "CanonicalArtifactDef in _CANONICAL_JSON_ARTIFACT_REGISTRY):\n" - + "\n".join(f" + {f}" for f in added_files) - ) - if stale: - msg_parts.append( - "Registered consumer sites no longer found (remove from " - "_CANONICAL_JSON_ARTIFACT_REGISTRY):\n" + "\n".join(f" - {f}" for f in stale) - ) - assert not added_files and not stale, "\n\n".join(msg_parts) + """Every strict consumer scope matches one registry entry in both directions.""" + problems = _consumer_registry_problems( + _CANONICAL_JSON_ARTIFACT_REGISTRY, _scan_decode_call_sites() + ) + assert not problems, "\n\n".join(problems) def test_registered_producers_are_sanctioned_canonical_writers(self): """Every registered producer writes canonical objects or prepared canonical bytes.""" @@ -273,42 +311,240 @@ def test_registered_skill_md_refs_name_the_producer(self): f"{entry.producer_symbol!r}" ) - def test_new_require_canonical_consumer_without_registered_producer_fails(self, monkeypatch): + def test_new_require_canonical_consumer_without_registered_producer_fails(self): """Meta-test: a fake extra consumer site should cause the ratchet to fail.""" - original_scan = _scan_require_canonical_consumer_sites + sites = _scan_decode_call_sites() + sites[("src/autoskillit/fake_canonical_module.py", "load", True)] += 1 + problems = _consumer_registry_problems(_CANONICAL_JSON_ARTIFACT_REGISTRY, sites) + assert any("fake_canonical_module" in problem for problem in problems) + + def test_non_canonical_exceptions_still_resolve_to_non_canonical_reads(self): + """Each exception scope has non-strict decode calls and a reason.""" + sites = Counter() + exception_paths = sorted( + {path for (path, _qualname), _reason in _NON_CANONICAL_JSON_EXCEPTIONS.values()} + ) + for relative_path in exception_paths: + source_path = _REPO_ROOT / relative_path + tree = ast.parse(source_path.read_text(), filename=str(source_path)) + sites.update(_decode_call_sites(tree, relative_path)) + problems = _non_canonical_exception_problems(_NON_CANONICAL_JSON_EXCEPTIONS, sites) + assert not problems, "\n\n".join(problems) + + +def _def(path: str, function: str) -> CanonicalArtifactDef: + """Build a synthetic consumer entry; producer fields are ignored by this diff.""" + return CanonicalArtifactDef( + consumer_path=path, + consumer_function=function, + producer_symbol="write_placeholder", + producer_path="pkg/producer.py", + producer_function="write_placeholder", + skill_md_refs=(), + ) + + +class TestCanonicalConsumerGuardBehavior: + def test_consumer_identity_is_line_free_and_scope_qualified(self): + source = """ +def load(): + decode_versioned_json_bytes(payload, require_canonical=True) - def patched_scan(): - sites = original_scan() - sites.add(("src/autoskillit/fake_canonical_module.py", 999)) - return sites +class Verifier: + def load(self): + decode_versioned_json_bytes(payload, require_canonical=True) + async def aload(self): + decode_versioned_json_bytes(payload, require_canonical=True) + + decode_versioned_json_bytes(payload, require_canonical=True) + +def outer(): + def inner(): + decode_versioned_json_bytes(payload, require_canonical=True) + +decode_versioned_json_bytes(payload, require_canonical=True) + +def HeaderOwner(value=header_probe()): + pass + +class ClassHeaderOwner(header_probe()): + pass +""" + tree = ast.parse(source) + expected = Counter( + { + ("pkg/mod.py", "load", True): 1, + ("pkg/mod.py", "Verifier.load", True): 1, + ("pkg/mod.py", "Verifier.aload", True): 1, + ("pkg/mod.py", "Verifier", True): 1, + ("pkg/mod.py", "outer.inner", True): 1, + ("pkg/mod.py", MODULE_SCOPE, True): 1, + } + ) + assert _decode_call_sites(tree, "pkg/mod.py") == expected + + owners = scope_qualnames(tree) + header_calls = [ + node + for node in ast.walk(tree) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "header_probe" + ] + assert {owners[node] for node in header_calls} == {"HeaderOwner", "ClassHeaderOwner"} + + shifted_tree = ast.parse("\n" * 40 + "# shifted line numbers\n" + source) + assert _decode_call_sites(shifted_tree, "pkg/mod.py") == expected + + @pytest.mark.parametrize( + "load_b_def", + [ + "", + " def load_b(self):\n" + " decode_versioned_json_bytes(payload, require_canonical=False)", + " def load_b(self):\n decode_versioned_json_bytes(payload)", + ], + ids=["removed", "non-canonical", "keyword-omitted"], + ) + def test_registered_consumer_missing_or_noncanonical_is_rejected_beside_valid_sibling( + self, load_b_def: str + ): + source = f""" +class Verifier: + def load_a(self): + decode_versioned_json_bytes(payload, require_canonical=True) +{load_b_def} +""" + registry = { + "alpha": _def("pkg/mod.py", "Verifier.load_a"), + "beta": _def("pkg/mod.py", "Verifier.load_b"), + } + problems = _consumer_registry_problems( + registry, _decode_call_sites(ast.parse(source), "pkg/mod.py") + ) + + assert len(problems) == 1 + problem = problems[0] + assert "beta" in problem + assert "pkg/mod.py::Verifier.load_b" in problem + assert "alpha" not in problem + assert "update consumer_path/consumer_function" in problem + assert "delete the entry" in problem + + def test_extra_strict_calls_are_rejected(self): + registry = {"alpha": _def("pkg/mod.py", "Verifier.load_a")} + extra_scope = ast.parse( + """ +class Verifier: + def load_a(self): + decode_versioned_json_bytes(payload, require_canonical=True) + +def load_c(): + decode_versioned_json_bytes(payload, require_canonical=True) +""" + ) + problems = _consumer_registry_problems( + registry, _decode_call_sites(extra_scope, "pkg/mod.py") + ) + assert len(problems) == 1 + assert "pkg/mod.py::load_c" in problems[0] + assert "register a CanonicalArtifactDef" in problems[0] + + duplicate_scope = ast.parse( + """ +class Verifier: + def load_a(self): + decode_versioned_json_bytes(first, require_canonical=True) + decode_versioned_json_bytes(second, require_canonical=True) +""" + ) + problems = _consumer_registry_problems( + registry, _decode_call_sites(duplicate_scope, "pkg/mod.py") + ) + assert len(problems) == 1 + assert "pkg/mod.py::Verifier.load_a" in problems[0] + assert "unregistered count: 1" in problems[0] + + def test_ambiguous_require_canonical_counts_as_strict(self): + tree = ast.parse( + """ +def load_x(flag): + decode_versioned_json_bytes(payload, require_canonical=flag) + +def load_y(opts): + decode_versioned_json_bytes(payload, **opts) + +def load_z(opts): + decode_versioned_json_bytes(payload, require_canonical=False, **opts) +""" + ) + sites = _decode_call_sites(tree, "pkg/mod.py") + assert sites == Counter( + { + ("pkg/mod.py", "load_x", True): 1, + ("pkg/mod.py", "load_y", True): 1, + ("pkg/mod.py", "load_z", True): 1, + } + ) + problems = _consumer_registry_problems({}, sites) + assert len(problems) == 3 + assert all("unregistered count: 1" in problem for problem in problems) + assert all( + f"pkg/mod.py::{function}" in "\n".join(problems) + for function in ("load_x", "load_y", "load_z") + ) + + def test_non_canonical_exception_detects_flip_and_disappearance(self): + exceptions = {"report.json": (("pkg/mod.py", "verify"), "reason text")} + non_strict = _decode_call_sites( + ast.parse( + """ +def verify(): + decode_versioned_json_bytes(payload, require_canonical=False) +""" + ), + "pkg/mod.py", + ) + assert _non_canonical_exception_problems(exceptions, non_strict) == [] + + strict = _decode_call_sites( + ast.parse( + """ +def verify(): + decode_versioned_json_bytes(payload, require_canonical=True) +""" + ), + "pkg/mod.py", + ) + problems = _non_canonical_exception_problems(exceptions, strict) + assert len(problems) == 1 + assert "report.json" in problems[0] + assert "counts as strict" in problems[0] + + missing = _decode_call_sites(ast.parse("def verify():\n pass\n"), "pkg/mod.py") + problems = _non_canonical_exception_problems(exceptions, missing) + assert len(problems) == 1 + assert "report.json" in problems[0] + assert "no decode_versioned_json_bytes call" in problems[0] + + empty_reason = {"report.json": (("pkg/mod.py", "verify"), "")} + problems = _non_canonical_exception_problems(empty_reason, non_strict) + assert len(problems) == 1 + assert "report.json" in problems[0] + assert "empty exception reason" in problems[0] + + def test_source_parse_failure_is_not_silently_skipped(self, tmp_path, monkeypatch): + source_root = tmp_path / "src" / "autoskillit" + source_root.mkdir(parents=True) + invalid_file = source_root / "broken.py" + invalid_file.write_text("def broken(:\n", encoding="utf-8") + monkeypatch.setattr( + "tests.infra.test_canonical_json_producer_convention._REPO_ROOT", tmp_path + ) monkeypatch.setattr( - "tests.infra.test_canonical_json_producer_convention." - "_scan_require_canonical_consumer_sites", - patched_scan, + "tests.infra.test_canonical_json_producer_convention._SRC_ROOT", source_root ) - with pytest.raises(AssertionError, match="fake_canonical_module"): - self.test_require_canonical_consumers_have_registered_producers() - def test_non_canonical_exceptions_still_resolve_to_non_canonical_reads(self): - """Documented non-canonical exceptions must not pass require_canonical=True.""" - repo_root = Path(__file__).resolve().parents[2] - for artifact_name, (site, reason) in _NON_CANONICAL_JSON_EXCEPTIONS.items(): - assert reason, f"{artifact_name} has an empty exception reason" - relative_path, lineno = site - source_path = repo_root / relative_path - tree = ast.parse(source_path.read_text(), filename=str(source_path)) - call = _find_call_at_line(tree, lineno, "decode_versioned_json_bytes") - assert call is not None, ( - f"{artifact_name}: no decode_versioned_json_bytes call found at " - f"{relative_path}:{lineno}" - ) - for kw in call.keywords: - if kw.arg == "require_canonical": - assert not _is_literal_true(kw.value), ( - f"{artifact_name}: {relative_path}:{lineno} now passes " - "require_canonical=True — this pairing is documented as " - "non-canonical in _NON_CANONICAL_JSON_EXCEPTIONS; either " - "register it in _CANONICAL_JSON_ARTIFACT_REGISTRY instead or " - "revert the flag" - ) + with pytest.raises(SyntaxError, match="broken.py"): + _scan_decode_call_sites() diff --git a/tests/infra/test_schema_version_convention.py b/tests/infra/test_schema_version_convention.py index 8ef2bc59b3..00c2691d1d 100644 --- a/tests/infra/test_schema_version_convention.py +++ b/tests/infra/test_schema_version_convention.py @@ -1,63 +1,60 @@ """Allowlist ratchet: enforce that new JSON dict write sites use write_versioned_json. Scans src/autoskillit/ for atomic_write calls whose second argument wraps json.dumps -of a dict payload. Sites in the _LEGACY_JSON_WRITES allowlist are grandfathered; +of a dict payload. Sites in the _LEGACY_JSON_WRITE_COUNTS allowlist are grandfathered; any new site must use write_versioned_json instead. """ from __future__ import annotations import ast +from collections import Counter +from collections.abc import Mapping from pathlib import Path import pytest +from tests._ast_helpers import call_name, count_differences, scope_qualnames + pytestmark = [pytest.mark.layer("infra"), pytest.mark.medium] -def _scan_atomic_write_json_dict_sites() -> set[tuple[str, int]]: - """AST-scan src/autoskillit/ for atomic_write(path, json.dumps({...})) calls. +def _legacy_json_write_sites(tree: ast.AST, relpath: str) -> Counter[str]: + """Return qualified legacy JSON write scopes from one parsed source file.""" + owners = scope_qualnames(tree) + sites: Counter[str] = Counter() - Returns set of (relative_path, line_number) for sites where the second - positional argument to atomic_write is (or wraps) json.dumps of a dict payload. - """ - src_root = Path(__file__).resolve().parents[2] / "src" / "autoskillit" - sites: set[tuple[str, int]] = set() + for node in ast.walk(tree): + if not isinstance(node, ast.Call) or call_name(node) != "atomic_write": + continue + if len(node.args) < 2: + continue - for py_file in src_root.rglob("*.py"): - try: - tree = ast.parse(py_file.read_text(), filename=str(py_file)) - except SyntaxError: + second_arg = node.args[1] + json_dumps_call = _extract_json_dumps(second_arg) + if json_dumps_call is None or not _is_dict_payload(json_dumps_call): + continue + if _is_yaml_dump(second_arg): continue - for node in ast.walk(tree): - if not isinstance(node, ast.Call): - continue - # Match atomic_write(path, json.dumps(...)) - func = node.func - is_atomic_write = (isinstance(func, ast.Name) and func.id == "atomic_write") or ( - isinstance(func, ast.Attribute) and func.attr == "atomic_write" - ) - if not is_atomic_write: - continue - if len(node.args) < 2: - continue + sites[f"{relpath}::{owners[node]}"] += 1 - second_arg = node.args[1] - json_dumps_call = _extract_json_dumps(second_arg) - if json_dumps_call is None: - continue + return sites - # Check if the json.dumps argument is a dict (or could be a dict) - if not _is_dict_payload(json_dumps_call): - continue - # Skip if it's a dump_yaml_str call - if _is_yaml_dump(second_arg): - continue +def _scan_atomic_write_json_dict_sites() -> Counter[str]: + """Scan src/autoskillit/ for atomic_write(path, json.dumps({...})) calls. - rel = str(py_file.relative_to(src_root.parent.parent)) - sites.add((rel, node.lineno)) + Source parsing fails directly with the filename for any syntax error, independently + of whether lint or type checking also reports that syntax error. + """ + src_root = Path(__file__).resolve().parents[2] / "src" / "autoskillit" + sites: Counter[str] = Counter() + + for py_file in sorted(src_root.rglob("*.py")): + tree = ast.parse(py_file.read_text(encoding="utf-8"), filename=str(py_file)) + relpath = str(py_file.relative_to(src_root.parent.parent)) + sites.update(_legacy_json_write_sites(tree, relpath)) return sites @@ -106,111 +103,167 @@ def _is_yaml_dump(node: ast.expr) -> bool: return False -# Hard-curated allowlist of existing atomic_write + json.dumps sites detected by AST scan. +def _legacy_write_problems(expected: Mapping[str, int], observed: Counter[str]) -> list[str]: + """Describe per-scope count differences between the registry and source scan.""" + problems = [f"{key}: count must be at least 1" for key, count in expected.items() if count < 1] + for key, expected_count, observed_count in count_differences(expected, observed): + if key in expected and expected_count < 1: + continue + if observed_count > expected_count: + problems.append( + f"{key}: use write_versioned_json instead (or, if a moved/renamed " + "allowlisted write, update its key)" + ) + elif observed_count < expected_count and observed_count == 0: + problems.append( + f"{key}: missing write; if the write moved or was renamed, update the key; " + "if it was removed or migrated to write_versioned_json, delete the entry" + ) + elif observed_count < expected_count: + problems.append(f"{key}: fewer write sites than expected; lower the count") + + return sorted(problems) + + +# Hard-curated counts of existing atomic_write + json.dumps sites detected by AST scan. # Any new site that writes a dict payload SHOULD use write_versioned_json. -_LEGACY_JSON_WRITES: set[tuple[str, int]] = { +_LEGACY_JSON_WRITE_COUNTS: dict[str, int] = { # migration/store.py — failure store dicts - ("src/autoskillit/migration/store.py", 54), - ("src/autoskillit/migration/store.py", 64), + "src/autoskillit/migration/store.py::FailureStore.record": 1, + "src/autoskillit/migration/store.py::FailureStore.clear": 1, # clone/_registry.py — clones dict (CloneRegistry.__exit__ atomic write-back) - ("src/autoskillit/workspace/clone/_registry.py", 97), + "src/autoskillit/workspace/clone/_registry.py::CloneRegistry.__exit__": 1, # staleness_cache.py — cache dict - ("src/autoskillit/recipe/contracts/staleness_cache.py", 67), + "src/autoskillit/recipe/contracts/staleness_cache.py::write_staleness_cache": 1, # _hook_repair.py — hooks.json self-heal for plugin cache and projection scopes, # triggered by the in-process repair primitive instead of startup drift detection - # (co-owned with Claude plugin system). The shared lease-held hook transaction at line 324 - # rewrites hooks.json for both cache and projection artifacts. Manifest refresh stays - # versioned separately. - ("src/autoskillit/workspace/_projected_artifact/_hook_repair.py", 359), + # (co-owned with Claude plugin system). The shared lease-held hook transaction in + # _repair_hook_payload_under_lease rewrites hooks.json for both cache and projection + # artifacts. Manifest refresh stays versioned separately. + ( + "src/autoskillit/workspace/_projected_artifact/_hook_repair.py::" + "_repair_hook_payload_under_lease" + ): 1, # tools_kitchen/_hook_config.py — hook config, quota guard, and git_ops_policy - ("src/autoskillit/server/tools/tools_kitchen/_hook_config.py", 101), - ("src/autoskillit/server/tools/tools_kitchen/_hook_config.py", 120), - ("src/autoskillit/server/tools/tools_kitchen/_hook_config.py", 152), + "src/autoskillit/server/tools/tools_kitchen/_hook_config.py::_write_hook_config": 1, + ( + "src/autoskillit/server/tools/tools_kitchen/_hook_config.py::" + "_update_hook_config_with_recipe" + ): 1, + ( + "src/autoskillit/server/tools/tools_kitchen/_hook_config.py::" + "_update_hook_config_with_git_ops_policy" + ): 1, # _overlay_state.py — session-scoped hook config overlay (not schema-versioned) - ("src/autoskillit/server/tools/_overlay_state.py", 132), + "src/autoskillit/server/tools/_overlay_state.py::write_overlay_locked": 1, # pipeline_tracker.py — locked tracker init/merge/mutation writes centralize the existing # unversioned tracker file schema. - ("src/autoskillit/core/pipeline/pipeline_tracker.py", 560), - ("src/autoskillit/core/pipeline/pipeline_tracker.py", 588), - ("src/autoskillit/core/pipeline/pipeline_tracker.py", 608), + "src/autoskillit/core/pipeline/pipeline_tracker.py::initialize_kitchen_tracker": 1, + "src/autoskillit/core/pipeline/pipeline_tracker.py::initialize_manual_tracker": 1, + "src/autoskillit/core/pipeline/pipeline_tracker.py::mutate_tracker": 1, # tools_status.py — mcp_data dict - ("src/autoskillit/server/tools/tools_status.py", 592), + "src/autoskillit/server/tools/tools_status.py::write_telemetry_files": 1, # tools_github.py — bug report dict (non-blocking report-bug status file) - ("src/autoskillit/server/tools/tools_github.py", 343), + "src/autoskillit/server/tools/tools_github.py::report_bug": 1, # _hooks.py — settings.json dict (co-owned with Claude CLI) - ("src/autoskillit/cli/_hooks.py", 29), + "src/autoskillit/cli/_hooks.py::_write_settings_data": 1, # _installed_plugins.py — installed_plugins.json (co-owned with Claude plugin system) - ("src/autoskillit/cli/install/_installed_plugins.py", 81), + "src/autoskillit/cli/install/_installed_plugins.py::InstalledPluginsFile.remove": 1, # _update_checks.py — dismissal state file - ("src/autoskillit/cli/update/_update_checks.py", 96), + "src/autoskillit/cli/update/_update_checks.py::_write_dismiss_state": 1, # _update_checks_fetch.py — fetch cache (extracted from _update_checks.py) - ("src/autoskillit/cli/update/_update_checks_fetch.py", 67), + "src/autoskillit/cli/update/_update_checks_fetch.py::_write_fetch_cache": 1, # smoke_utils/_review_design.py — atomic v1→v2 handoff enrichment and manifest - ("src/autoskillit/smoke_utils/_review_design.py", 248), - ("src/autoskillit/smoke_utils/_review_design.py", 350), + "src/autoskillit/smoke_utils/_review_design.py::enrich_diff_context": 1, + "src/autoskillit/smoke_utils/_review_design.py::select_review_dimensions": 1, # tools_pr_ops.py — GitHubReviewReceipt.to_dict() already carries schema_version - ("src/autoskillit/server/tools/tools_pr_ops.py", 507), + "src/autoskillit/server/tools/tools_pr_ops.py::post_pr_review": 1, # smoke_utils/_git.py — partitions, merge queue data - # Line 110 is a list-payload write site (dual membership: also in list_sites - # in test_allowlist_includes_list_payloads_as_documented). The AST scanner catches + # The fetch_merge_queue_data site is a list-payload write site (dual membership: also in + # list_sites in test_allowlist_includes_list_payloads_as_documented). The AST scanner catches # it because it cannot distinguish list vs dict return types — intentional. - ("src/autoskillit/smoke_utils/_git.py", 55), - ("src/autoskillit/smoke_utils/_git.py", 110), + "src/autoskillit/smoke_utils/_git.py::compute_domain_partitions": 1, + "src/autoskillit/smoke_utils/_git.py::fetch_merge_queue_data": 1, # smoke_utils/_eval.py — eval resolved/manifest/scorecard/eval_context, agent-eval - ("src/autoskillit/smoke_utils/_eval.py", 87), - ("src/autoskillit/smoke_utils/_eval.py", 98), - ("src/autoskillit/smoke_utils/_eval.py", 235), - ("src/autoskillit/smoke_utils/_eval.py", 246), - ("src/autoskillit/smoke_utils/_eval.py", 324), - ("src/autoskillit/smoke_utils/_eval.py", 495), + "src/autoskillit/smoke_utils/_eval.py::parse_eval_manifests": 2, + "src/autoskillit/smoke_utils/_eval.py::parse_agent_eval_manifests": 2, + "src/autoskillit/smoke_utils/_eval.py::_build_eval_context_common": 1, + "src/autoskillit/smoke_utils/_eval.py::compile_eval_scorecard": 1, # planner/consolidation.py — broken_cycle_edges.json (list payload; AST scanner catches it) - ("src/autoskillit/planner/consolidation.py", 362), - # planner/consolidation.py — broken_cycle_edges.json (list payload; AST scanner catches it) - ("src/autoskillit/planner/consolidation.py", 383), + "src/autoskillit/planner/consolidation.py::_persist_consolidation": 2, # planner/manifests.py — finalize_wp_manifest: wp_index.json rebuild (list payload) - ("src/autoskillit/planner/manifests.py", 334), + "src/autoskillit/planner/manifests.py::finalize_wp_manifest": 1, # _cmd_rpc_issues.py — emit_fallback_map: BEM fallback execution map (recipe-internal) - ("src/autoskillit/recipe/cmd_rpc/_cmd_rpc_issues.py", 78), + "src/autoskillit/recipe/cmd_rpc/_cmd_rpc_issues.py::emit_fallback_map": 1, # _probe_canary.py — CanaryState save() 3-field state file (intentionally unversioned) - ("src/autoskillit/_probe_canary.py", 51), + "src/autoskillit/_probe_canary.py::CanaryState.save": 1, } class TestSchemaVersionConvention: + def test_legacy_write_identity_survives_line_shift(self): + source = ( + 'class Store:\n def save(self):\n atomic_write(p, json.dumps({"a": 1}))\n' + ) + expected = Counter({"pkg/m.py::Store.save": 1}) + + original = _legacy_json_write_sites(ast.parse(source), "pkg/m.py") + shifted = _legacy_json_write_sites(ast.parse("\n" * 30 + source), "pkg/m.py") + + assert original == expected + assert shifted == expected + + @pytest.mark.parametrize( + ("expected", "observed", "key", "message"), + [ + ( + {"pkg/m.py::writer": 1}, + Counter({"pkg/m.py::writer": 2}), + "pkg/m.py::writer", + "use write_versioned_json", + ), + ( + {"pkg/m.py::present": 1, "pkg/m.py::missing": 1}, + Counter({"pkg/m.py::present": 1}), + "pkg/m.py::missing", + "update the key", + ), + ( + {"pkg/m.py::writer": 2}, + Counter({"pkg/m.py::writer": 1}), + "pkg/m.py::writer", + "lower the count", + ), + ({"pkg/m.py::writer": 0}, Counter(), "pkg/m.py::writer", "count must be at least 1"), + ({"pkg/m.py::writer": -1}, Counter(), "pkg/m.py::writer", "count must be at least 1"), + ], + ) + def test_legacy_write_registry_diff_is_per_key_and_counted( + self, expected, observed, key, message + ): + problems = _legacy_write_problems(expected, observed) + + assert len(problems) == 1 + assert key in problems[0] + assert message in problems[0] + if key == "pkg/m.py::missing": + assert "delete the entry" in problems[0] + assert "present" not in problems[0] + def test_current_json_write_sites_match_allowlist(self): - """All atomic_write+json.dumps(dict) sites must be in _LEGACY_JSON_WRITES.""" + """All atomic_write+json.dumps(dict) sites must match the scope/count registry.""" current = _scan_atomic_write_json_dict_sites() - added = current - _LEGACY_JSON_WRITES - removed = _LEGACY_JSON_WRITES - current - - msg_parts = [] - if added: - msg_parts.append( - "New json.dumps dict write sites found (use write_versioned_json instead):\n" - + "\n".join(f" + {f}:{ln}" for f, ln in sorted(added)) - ) - if removed: - msg_parts.append( - "Allowlisted sites no longer found (remove from _LEGACY_JSON_WRITES):\n" - + "\n".join(f" - {f}:{ln}" for f, ln in sorted(removed)) - ) - assert current == _LEGACY_JSON_WRITES, "\n\n".join(msg_parts) + problems = _legacy_write_problems(_LEGACY_JSON_WRITE_COUNTS, current) - def test_new_json_write_site_without_helper_fails(self, monkeypatch): - """Meta-test: a fake extra site should cause the ratchet to fail.""" - original_scan = _scan_atomic_write_json_dict_sites + assert not problems, "\n".join(problems) - def patched_scan(): - sites = original_scan() - sites.add(("src/autoskillit/fake_module.py", 999)) - return sites + def test_new_json_write_site_without_helper_fails(self): + """Meta-test: a fake extra site should cause the ratchet to fail.""" + sites = _scan_atomic_write_json_dict_sites() + sites["src/autoskillit/fake_module.py::fake_writer"] += 1 - monkeypatch.setattr( - "tests.infra.test_schema_version_convention._scan_atomic_write_json_dict_sites", - patched_scan, - ) - with pytest.raises(AssertionError, match="fake_module"): - self.test_current_json_write_sites_match_allowlist() + problems = _legacy_write_problems(_LEGACY_JSON_WRITE_COUNTS, sites) + assert any("fake_module" in problem for problem in problems) def test_allowlist_excludes_externally_co_owned_files(self): """Documented exclusions: settings.json, ~/.claude.json, installed_plugins.json, etc.""" @@ -219,39 +272,34 @@ def test_allowlist_excludes_externally_co_owned_files(self): "_init_helpers.py", "_marketplace.py", } - for path, _line in _LEGACY_JSON_WRITES: + for key in _LEGACY_JSON_WRITE_COUNTS: + path, _scope = key.rsplit("::", 1) basename = Path(path).name if basename in co_owned_paths: - assert isinstance(path, str) and path, ( - f"Co-owned allowlist entry has invalid path: {path!r}" - ) - assert isinstance(_line, int) and _line > 0, ( - f"Co-owned allowlist entry has invalid line number: {_line!r}" - ) + assert _scope, f"Co-owned allowlist entry has empty scope: {key!r}" def test_allowlist_includes_list_payloads_as_documented(self): """List-payload sites are included since the AST scanner can't distinguish return types.""" # These sites write list payloads through function calls but are caught by the scanner - list_sites = [ - ("src/autoskillit/smoke_utils/_git.py", 110), - ] - for site in list_sites: - assert site in _LEGACY_JSON_WRITES, ( - f"List-payload site {site} should be in _LEGACY_JSON_WRITES (scanner limitation)" - ) + site = "src/autoskillit/smoke_utils/_git.py::fetch_merge_queue_data" + assert site in _LEGACY_JSON_WRITE_COUNTS, ( + f"List-payload site {site} should be in _LEGACY_JSON_WRITE_COUNTS (scanner limitation)" + ) def test_allowlist_excludes_yaml_writes(self): """YAML write sites must not appear in the allowlist.""" - for path, _line in _LEGACY_JSON_WRITES: + for key in _LEGACY_JSON_WRITE_COUNTS: + path, _scope = key.rsplit("::", 1) assert "yaml" not in Path(path).stem.lower() or "staleness" in path, ( - f"YAML write file {path} should not be in _LEGACY_JSON_WRITES" + f"YAML write file {path} should not be in _LEGACY_JSON_WRITE_COUNTS" ) def test_allowlist_excludes_string_body_writes(self): """Gitignore, marker files, pre-rendered report text must not be in allowlist.""" string_body_keywords = ["gitignore", "marker", "report"] - for path, _line in _LEGACY_JSON_WRITES: + for key in _LEGACY_JSON_WRITE_COUNTS: + path, _scope = key.rsplit("::", 1) for kw in string_body_keywords: assert kw not in Path(path).stem.lower(), ( - f"String-body file {path} should not be in _LEGACY_JSON_WRITES" + f"String-body file {path} should not be in _LEGACY_JSON_WRITE_COUNTS" ) diff --git a/tests/test_test_filter_tiered_always_run.py b/tests/test_test_filter_tiered_always_run.py index 990855b92e..a407cce844 100644 --- a/tests/test_test_filter_tiered_always_run.py +++ b/tests/test_test_filter_tiered_always_run.py @@ -106,15 +106,13 @@ def test_github_workflow_change_includes_full_infra_dir(self, tmp_path: Path) -> dir_names = {p.name for p in result2} assert "infra" in dir_names - def test_unconditional_files_constants_have_correct_counts(self) -> None: - """_INFRA_UNCONDITIONAL_FILES has 13 entries; hooks has 3 entries. - - Pin the unconditional-files contract: a silent mutation to these - frozensets changes the always-run behavior, so any new entry must be - reflected here as well. - """ - assert len(_INFRA_UNCONDITIONAL_FILES) == 13 - assert len(_HOOKS_UNCONDITIONAL_FILES) == 3 + def test_unconditional_files_name_existing_test_modules(self) -> None: + """Every unconditional filename must identify a test module on disk.""" + tests_root = Path(__file__).parent + for filename in _INFRA_UNCONDITIONAL_FILES: + assert (tests_root / "infra" / filename).is_file(), filename + for filename in _HOOKS_UNCONDITIONAL_FILES: + assert (tests_root / "hooks" / filename).is_file(), filename def test_lint_gate_selection_contract_is_unconditional(self) -> None: """The lint-gate selection contract must run on every conservative changeset.